cloudron-box/src/apptask.js

#!/usr/bin/env node

'use strict';

exports = module.exports = {
    run,

    // exported for testing
    _createAppDir: createAppDir,
    _deleteAppDir: deleteAppDir,
    _verifyManifest: verifyManifest,
    _waitForDnsPropagation: waitForDnsPropagation
};

const apps = require('./apps.js'),
    assert = require('assert'),
    AuditSource = require('./auditsource.js'),
    backuptask = require('./backuptask.js'),
    BoxError = require('./boxerror.js'),
    collectd = require('./collectd.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:apptask'),
    df = require('@sindresorhus/df'),
    dns = require('./dns.js'),
    docker = require('./docker.js'),
    ejs = require('ejs'),
    fs = require('fs'),
    iputils = require('./iputils.js'),
    manifestFormat = require('cloudron-manifestformat'),
    mounts = require('./mounts.js'),
    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    promiseRetry = require('./promise-retry.js'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
    services = require('./services.js'),
    settings = require('./settings.js'),
    shell = require('./shell.js'),
    superagent = require('superagent'),
    sysinfo = require('./sysinfo.js'),
    _ = require('underscore');

const MV_VOLUME_CMD = path.join(__dirname, 'scripts/mvvolume.sh'),
    LOGROTATE_CONFIG_EJS = fs.readFileSync(__dirname + '/logrotate.ejs', { encoding: 'utf8' }),
    CONFIGURE_LOGROTATE_CMD = path.join(__dirname, 'scripts/configurelogrotate.sh');

// https://rootlesscontaine.rs/getting-started/common/cgroup2/#checking-whether-cgroup-v2-is-already-enabled
const CGROUP_VERSION = fs.existsSync('/sys/fs/cgroup/cgroup.controllers') ? '2' : '1';
const COLLECTD_CONFIG_EJS = fs.readFileSync(`${__dirname}/collectd/app_cgroup_v${CGROUP_VERSION}.ejs`, { encoding: 'utf8' });

function makeTaskError(error, app) {
    assert(error instanceof BoxError);
    assert.strictEqual(typeof app, 'object');

    // track a few variables which helps 'repair' restart the task (see also scheduleTask in apps.js)
    error.details.taskId = app.taskId;
    error.details.installationState = app.installationState;
    return error.toPlainObject();
}

// updates the app object and the database
async function updateApp(app, values) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof values, 'object');

    await apps.update(app.id, values);

    for (const value in values) {
        app[value] = values[value];
    }
}

async function allocateContainerIp(app) {
    assert.strictEqual(typeof app, 'object');

    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;

    await promiseRetry({ times: 10, interval: 0, debug }, async function () {
        const iprange = iputils.intFromIp('172.18.20.255') - iputils.intFromIp('172.18.16.1');
        let rnd = Math.floor(Math.random() * iprange);
        const containerIp = iputils.ipFromInt(iputils.intFromIp('172.18.16.1') + rnd);
        await updateApp(app, { containerIp });
    });
}

async function createContainer(app) {
    assert.strictEqual(typeof app, 'object');
    assert(!app.containerId); // otherwise, it will trigger volumeFrom

    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;

    debug('createContainer: creating container');

    const container = await docker.createContainer(app);

    await updateApp(app, { containerId: container.id });

    // re-generate configs that rely on container id
    await addLogrotateConfig(app);
    await addCollectdProfile(app);
}

async function deleteContainers(app, options) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');

    debug('deleteContainer: deleting app containers (app, scheduler)');

    // remove configs that rely on container id
    await removeCollectdProfile(app);
    await removeLogrotateConfig(app);
    await docker.stopContainers(app.id);
    await docker.deleteContainers(app.id, options);
    await updateApp(app, { containerId: null });
}

async function createAppDir(app) {
    assert.strictEqual(typeof app, 'object');

    const appDir = path.join(paths.APPS_DATA_DIR, app.id);
    const [error] = await safe(fs.promises.mkdir(appDir, { recursive: true }));
    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating directory: ${error.message}`, { appDir });
}

async function deleteAppDir(app, options) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');

    const appDataDir = path.join(paths.APPS_DATA_DIR, app.id);

    // resolve any symlinked data dir
    const stat = safe.fs.lstatSync(appDataDir);
    if (!stat) return;

    const resolvedAppDataDir = stat.isSymbolicLink() ? safe.fs.readlinkSync(appDataDir) : appDataDir;

    if (safe.fs.existsSync(resolvedAppDataDir)) {
        const entries = safe.fs.readdirSync(resolvedAppDataDir);
        if (!entries) throw new BoxError(BoxError.FS_ERROR, `Error listing ${resolvedAppDataDir}: ${safe.error.message}`);

        // remove only files. directories inside app dir are currently volumes managed by the addons
        // we cannot delete those dirs anyway because of perms
        entries.forEach(function (entry) {
            let stat = safe.fs.statSync(path.join(resolvedAppDataDir, entry));
            if (stat && !stat.isDirectory()) safe.fs.unlinkSync(path.join(resolvedAppDataDir, entry));
        });
    }

    // if this fails, it's probably because the localstorage/redis addons have not cleaned up properly
    if (options.removeDirectory) {
        if (stat.isSymbolicLink()) {
            if (!safe.fs.unlinkSync(appDataDir)) {
                if (safe.error.code !== 'ENOENT') throw new BoxError(BoxError.FS_ERROR, `Error unlinking dir ${appDataDir} : ${safe.error.message}`);
            }
        } else {
            if (!safe.fs.rmSync(appDataDir, { recursive: true })) {
                if (safe.error.code !== 'ENOENT') throw new BoxError(BoxError.FS_ERROR, `Error removing dir ${appDataDir} : ${safe.error.message}`);
            }
        }
    }
}

async function addCollectdProfile(app) {
    assert.strictEqual(typeof app, 'object');

    const appDataDir = await apps.getStorageDir(app);
    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir });
    await collectd.addProfile(app.id, collectdConf);
}

async function removeCollectdProfile(app) {
    assert.strictEqual(typeof app, 'object');

    await collectd.removeProfile(app.id);
}

async function addLogrotateConfig(app) {
    assert.strictEqual(typeof app, 'object');

    const result = await docker.inspect(app.containerId);

    const runVolume = result.Mounts.find(function (mount) { return mount.Destination === '/run'; });
    if (!runVolume) throw new BoxError(BoxError.DOCKER_ERROR, 'App does not have /run mounted');

    // logrotate configs can have arbitrary commands, so the config files must be owned by root
    const logrotateConf = ejs.render(LOGROTATE_CONFIG_EJS, { volumePath: runVolume.Source, appId: app.id });
    const tmpFilePath = path.join(os.tmpdir(), app.id + '.logrotate');

    safe.fs.writeFileSync(tmpFilePath, logrotateConf);
    if (safe.error) throw new BoxError(BoxError.FS_ERROR, `Error writing logrotate config: ${safe.error.message}`);

    const [error] = await safe(shell.promises.sudo('addLogrotateConfig', [ CONFIGURE_LOGROTATE_CMD, 'add', app.id, tmpFilePath ], {}));
    if (error) throw new BoxError(BoxError.LOGROTATE_ERROR, `Error adding logrotate config: ${error.message}`);
}

async function removeLogrotateConfig(app) {
    assert.strictEqual(typeof app, 'object');

    const [error] = await safe(shell.promises.sudo('removeLogrotateConfig', [ CONFIGURE_LOGROTATE_CMD, 'remove', app.id ], {}));
    if (error) throw new BoxError(BoxError.LOGROTATE_ERROR, `Error removing logrotate config: ${error.message}`);
}

async function cleanupLogs(app) {
    assert.strictEqual(typeof app, 'object');

    // note that redis container logs are cleaned up by the addon
    const [error] = await safe(fs.promises.rm(path.join(paths.LOG_DIR, app.id), { force: true, recursive: true }));
    if (error) debug('cleanupLogs: cannot cleanup logs:', error);
}

async function verifyManifest(manifest) {
    assert.strictEqual(typeof manifest, 'object');

    let error = manifestFormat.parse(manifest);
    if (error) throw new BoxError(BoxError.BAD_FIELD, `Manifest error: ${error.message}`);

    error = apps.checkManifestConstraints(manifest);
    if (error) throw new BoxError(BoxError.CONFLICT, `Manifest constraint check failed: ${error.message}`);
}

async function downloadIcon(app) {
    assert.strictEqual(typeof app, 'object');

    // nothing to download if we dont have an appStoreId
    if (!app.appStoreId) return;

    debug(`downloadIcon: Downloading icon of ${app.appStoreId}@${app.manifest.version}`);

    const iconUrl = settings.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';

    await promiseRetry({ times: 10, interval: 5000, debug }, async function () {
        const [networkError, response] = await safe(superagent.get(iconUrl)
            .buffer(true)
            .timeout(30 * 1000)
            .ok(() => true));

        if (networkError) throw new BoxError(BoxError.NETWORK_ERROR, `Network error downloading icon : ${networkError.message}`);
        if (response.status !== 200) return; // ignore error. this can also happen for apps installed with cloudron-cli

        await updateApp(app, { appStoreIcon: response.body });
    });
}

async function waitForDnsPropagation(app) {
    assert.strictEqual(typeof app, 'object');

    if (!constants.CLOUDRON) {
        debug('waitForDnsPropagation: Skipping dns propagation check for development');
        return;
    }

    const ipv4 = await sysinfo.getServerIPv4();
    const ipv6 = await sysinfo.getServerIPv6();

    let error;
    [error] = await safe(dns.waitForDnsRecord(app.subdomain, app.domain, 'A', ipv4, { times: 240 }));
    if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS A Record is not synced yet: ${error.message}`, { ipv4, subdomain: app.subdomain, domain: app.domain });
    if (ipv6) {
        [error] = await safe(dns.waitForDnsRecord(app.subdomain, app.domain, 'AAAA', ipv6, { times: 240 }));
        if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS AAAA Record is not synced yet: ${error.message}`, { ipv6, subdomain: app.subdomain, domain: app.domain });
    }

    // now wait for redirectDomains and aliasDomains, if any
    const allDomains = app.secondaryDomains.concat(app.redirectDomains).concat(app.aliasDomains);
    for (const domain of allDomains) {
        [error] = await safe(dns.waitForDnsRecord(domain.subdomain, domain.domain, 'A', ipv4, { times: 240 }));
        if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS A Record is not synced yet: ${error.message}`, { ipv4, subdomain: domain.subdomain, domain: domain.domain });
        if (ipv6) {
            [error] = await safe(dns.waitForDnsRecord(domain.subdomain, domain.domain, 'AAAA', ipv6, { times: 240 }));
            if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS AAAA Record is not synced yet: ${error.message}`, { ipv6, subdomain: domain.subdomain, domain: domain.domain });
        }
    }
}

async function moveDataDir(app, targetVolumeId, targetVolumePrefix) {
    assert.strictEqual(typeof app, 'object');
    assert(targetVolumeId === null || typeof targetVolumeId === 'string');
    assert(targetVolumePrefix === null || typeof targetVolumePrefix === 'string');

    const resolvedSourceDir = await apps.getStorageDir(app);
    const resolvedTargetDir = await apps.getStorageDir(_.extend({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix }));

    debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);

    if (resolvedSourceDir !== resolvedTargetDir) {
        const [error] = await safe(shell.promises.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
        if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`);
    }

    await updateApp(app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix });
}

async function downloadImage(manifest) {
    assert.strictEqual(typeof manifest, 'object');

    // skip for relay app
    if (manifest.id === constants.PROXY_APP_APPSTORE_ID) return;

    const info = await docker.info();
    const [dfError, diskUsage] = await safe(df.file(info.DockerRootDir));
    if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error getting file system info: ${dfError.message}`);

    if (diskUsage.available < (1024*1024*1024)) throw new BoxError(BoxError.DOCKER_ERROR, 'Not enough disk space to pull docker image', { diskUsage: diskUsage, dockerRootDir: info.DockerRootDir });

    await docker.downloadImage(manifest);
}

async function startApp(app) {
    debug('startApp: starting container');

    if (app.runState === apps.RSTATE_STOPPED) return;

    // skip for relay app
    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;

    await docker.startContainer(app.id);
}

async function install(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const restoreConfig = args.restoreConfig; // has to be set when restoring
    const overwriteDns = args.overwriteDns;
    const skipDnsSetup = args.skipDnsSetup;
    const oldManifest = args.oldManifest;

    // this protects against the theoretical possibility of an app being marked for install/restore from
    // a previous version of box code
    await verifyManifest(app.manifest);

    // teardown for re-installs
    await progressCallback({ percent: 10, message: 'Cleaning up old install' });
    await reverseProxy.unconfigureApp(app);
    await deleteContainers(app, { managedOnly: true });

    // when restoring, app does not require these addons anymore. remove carefully to preserve the db passwords
    let addonsToRemove;
    if (oldManifest) {
        addonsToRemove = _.omit(oldManifest.addons, Object.keys(app.manifest.addons));
    } else {
        addonsToRemove = app.manifest.addons;
    }
    await services.teardownAddons(app, addonsToRemove);

    if (!restoreConfig || restoreConfig.remotePath) { // in-place import should not delete data dir
        await deleteAppDir(app, { removeDirectory: false }); // do not remove any symlinked appdata dir
    }

    if (oldManifest && oldManifest.dockerImage === app.manifest.dockerImage) {
        await docker.deleteImage(oldManifest);
    }

    // allocating container ip here, lets the users "repair" an app if allocation fails at apps.add time
    await allocateContainerIp(app);

    await progressCallback({ percent: 20, message: 'Downloading icon' });
    await downloadIcon(app);

    if (!skipDnsSetup) {
        await progressCallback({ percent: 30, message: 'Registering subdomains' });

        await dns.registerLocations([ { subdomain: app.subdomain, domain: app.domain }].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback);
    }

    await progressCallback({ percent: 40, message: 'Downloading image' });
    await downloadImage(app.manifest);

    await progressCallback({ percent: 50, message: 'Creating app data directory' });
    await createAppDir(app);

    if (!restoreConfig) { // install
        await progressCallback({ percent: 60, message: 'Setting up addons' });
        await services.setupAddons(app, app.manifest.addons);
    } else if (app.installationState === apps.ISTATE_PENDING_IMPORT && !restoreConfig.remotePath) { // in-place import
        await progressCallback({ percent: 60, message: 'Importing addons in-place' });
        await services.setupAddons(app, app.manifest.addons);
        await services.clearAddons(app, _.omit(app.manifest.addons, 'localstorage'));
        await apps.restoreConfig(app);
        await services.restoreAddons(app, app.manifest.addons);
    } else if (app.installationState === apps.ISTATE_PENDING_IMPORT) { // import
        await progressCallback({ percent: 65, message: 'Downloading backup and restoring addons' });
        await services.setupAddons(app, app.manifest.addons);
        await services.clearAddons(app, app.manifest.addons);
        const backupConfig = restoreConfig.backupConfig; // can be null
        let mountObject = null;
        if (backupConfig && mounts.isManagedProvider(backupConfig.provider)) {
            await progressCallback({ percent: 70, message: 'Setting up mount for importing' });
            mountObject = {  // keep this in sync with importApp in apps.js
                name:  `appimport-${app.id}`,
                hostPath: `/mnt/appimport-${app.id}`,
                mountType: backupConfig.provider,
                mountOptions: backupConfig.mountOptions
            };
            await mounts.tryAddMount(mountObject, { timeout: 10 });
        }
        await backuptask.downloadApp(app, restoreConfig, (progress) => { progressCallback({ percent: 75, message: progress.message }); });
        await apps.restoreConfig(app);
        if (mountObject) await mounts.removeMount(mountObject);
        await progressCallback({ percent: 75, message: 'Restoring addons' });
        await services.restoreAddons(app, app.manifest.addons);
    } else { // clone and restore
        await progressCallback({ percent: 65, message: 'Downloading backup and restoring addons' });
        await services.setupAddons(app, app.manifest.addons);
        await services.clearAddons(app, app.manifest.addons);
        await backuptask.downloadApp(app, restoreConfig, (progress) => { progressCallback({ percent: 65, message: progress.message }); });
        await progressCallback({ percent: 70, message: 'Restoring addons' });
        await services.restoreAddons(app, app.manifest.addons);
    }

    await progressCallback({ percent: 80, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    if (!skipDnsSetup) {
        await progressCallback({ percent: 85, message: 'Waiting for DNS propagation' });
        await exports._waitForDnsPropagation(app);
    }

    await progressCallback({ percent: 95, message: 'Configuring reverse proxy' });
    await reverseProxy.configureApp(app, AuditSource.APPTASK);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function backup(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 10, message: 'Backing up' });
    const backupId = await backuptask.backupApp(app, { snapshotOnly: !!args.snapshotOnly }, (progress) => {
        progressCallback({ percent: 30, message: progress.message });
    });

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null });
    return backupId;
}

async function create(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 10, message: 'Cleaning up old install' });
    await deleteContainers(app, { managedOnly: true });

    // FIXME: re-setup addons only because sendmail addon to re-inject env vars on mailboxName change
    await progressCallback({ percent: 30, message: 'Setting up addons' });
    await services.setupAddons(app, app.manifest.addons);

    await progressCallback({ percent: 60, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function changeLocation(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const oldConfig = args.oldConfig;
    const locationChanged = oldConfig.fqdn !== app.fqdn;
    const skipDnsSetup = args.skipDnsSetup;
    const overwriteDns = args.overwriteDns;

    await progressCallback({ percent: 10, message: 'Cleaning up old install' });
    await reverseProxy.unconfigureApp(app);
    await deleteContainers(app, { managedOnly: true });

    // unregister old domains
    let obsoleteDomains = [];

    if (oldConfig.secondaryDomains) {
        obsoleteDomains = obsoleteDomains.concat(oldConfig.secondaryDomains.filter(function (o) {
            return !app.secondaryDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
        }));
    }

    if (oldConfig.redirectDomains) {
        obsoleteDomains = obsoleteDomains.concat(oldConfig.redirectDomains.filter(function (o) {
            return !app.redirectDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
        }));
    }

    if (oldConfig.aliasDomains) {
        obsoleteDomains = obsoleteDomains.concat(oldConfig.aliasDomains.filter(function (o) {
            return !app.aliasDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
        }));
    }

    if (locationChanged) obsoleteDomains.push({ subdomain: oldConfig.subdomain, domain: oldConfig.domain });

    if (obsoleteDomains.length !== 0) await dns.unregisterLocations(obsoleteDomains, progressCallback);

    // setup dns
    if (!skipDnsSetup) {
        await progressCallback({ percent: 30, message: 'Registering subdomains' });
        await dns.registerLocations([ { subdomain: app.subdomain, domain: app.domain }].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback);
    }

    // re-setup addons since they rely on the app's fqdn (e.g oauth)
    await progressCallback({ percent: 50, message: 'Setting up addons' });
    await services.setupAddons(app, app.manifest.addons);

    await progressCallback({ percent: 60, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    if (!skipDnsSetup) {
        await progressCallback({ percent: 80, message: 'Waiting for DNS propagation' });
        await exports._waitForDnsPropagation(app);
    }

    await progressCallback({ percent: 90, message: 'Configuring reverse proxy' });
    await reverseProxy.configureApp(app, AuditSource.APPTASK);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function migrateDataDir(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const { newStorageVolumeId, newStorageVolumePrefix } = args;
    assert(newStorageVolumeId === null || typeof newStorageVolumeId === 'string');
    assert(newStorageVolumePrefix === null || typeof newStorageVolumePrefix === 'string');

    await progressCallback({ percent: 10, message: 'Cleaning up old install' });
    await deleteContainers(app, { managedOnly: true });

    await progressCallback({ percent: 45, message: 'Ensuring app data directory' });
    await createAppDir(app);

    // re-setup addons since this creates the localStorage destination
    await progressCallback({ percent: 50, message: 'Setting up addons' });
    await services.setupAddons(_.extend({}, app, { storageVolumeId: newStorageVolumeId, storageVolumePrefix: newStorageVolumePrefix }), app.manifest.addons);

    await progressCallback({ percent: 60, message: 'Moving data dir' });
    await moveDataDir(app, newStorageVolumeId, newStorageVolumePrefix);

    await progressCallback({ percent: 90, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

// configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local"
async function configure(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 10, message: 'Cleaning up old install' });
    await reverseProxy.unconfigureApp(app);
    await deleteContainers(app, { managedOnly: true });

    await progressCallback({ percent: 20, message: 'Downloading icon' });
    await downloadIcon(app);

    await progressCallback({ percent: 40, message: 'Downloading image' });
    await downloadImage(app.manifest);

    await progressCallback({ percent: 45, message: 'Ensuring app data directory' });
    await createAppDir(app);

    // re-setup addons since they rely on the app's fqdn (e.g oauth)
    await progressCallback({ percent: 50, message: 'Setting up addons' });
    await services.setupAddons(app, app.manifest.addons);

    await progressCallback({ percent: 60, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    await progressCallback({ percent: 90, message: 'Configuring reverse proxy' });
    await reverseProxy.configureApp(app, AuditSource.APPTASK);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function update(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const updateConfig = args.updateConfig;

    // app does not want these addons anymore
    // FIXME: this does not handle option changes (like multipleDatabases)
    const unusedAddons = _.omit(app.manifest.addons, Object.keys(updateConfig.manifest.addons));
    const httpPortChanged = app.manifest.httpPort !== updateConfig.manifest.httpPort;
    const proxyAuthChanged = !_.isEqual(safe.query(app.manifest, 'addons.proxyAuth'), safe.query(updateConfig.manifest, 'addons.proxyAuth'));

    // this protects against the theoretical possibility of an app being marked for update from
    // a previous version of box code
    await progressCallback({ percent: 5, message: 'Verify manifest' });
    await verifyManifest(updateConfig.manifest);

    if (!updateConfig.skipBackup) {
        await progressCallback({ percent: 15, message: 'Backing up app' });
        // preserve update backups for 3 weeks
        const [error] = await safe(backuptask.backupApp(app, { preserveSecs: 3*7*24*60*60 }, (progress) => {
            progressCallback({ percent: 15, message: `Backup - ${progress.message}` });
        }));
        if (error) {
            error.backupError = true;
            throw error;
        }
    }

    // download new image before app is stopped. this is so we can reduce downtime
    // and also not remove the 'common' layers when the old image is deleted
    await progressCallback({ percent: 25, message: 'Downloading image' });
    await downloadImage(updateConfig.manifest);

    // note: we cleanup first and then backup. this is done so that the app is not running should backup fail
    // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
    await progressCallback({ percent: 35, message: 'Cleaning up old install' });
    await deleteContainers(app, { managedOnly: true });
    if (app.manifest.dockerImage !== updateConfig.manifest.dockerImage) await docker.deleteImage(app.manifest);

    // only delete unused addons after backup
    await services.teardownAddons(app, unusedAddons);

    // free unused ports
    const currentPorts = app.portBindings || {};
    const newTcpPorts = updateConfig.manifest.tcpPorts || {};
    const newUdpPorts = updateConfig.manifest.udpPorts || {};

    for (const portName of Object.keys(currentPorts)) {
        if (newTcpPorts[portName] || newUdpPorts[portName]) continue; // port still in use

        const [error] = await safe(apps.delPortBinding(currentPorts[portName], apps.PORT_TYPE_TCP));
        if (error && error.reason === BoxError.NOT_FOUND) debug('update: portbinding does not exist in database', error);
        else if (error) throw error;

        // also delete from app object for further processing (the db is updated in the next step)
        delete app.portBindings[portName];
    }

    await updateApp(app, _.pick(updateConfig, 'manifest', 'appStoreId', 'memoryLimit')); // switch over to the new config

    await progressCallback({ percent: 45, message: 'Downloading icon' });
    await downloadIcon(app);

    await progressCallback({ percent: 60, message: 'Updating addons' });
    await services.setupAddons(app, updateConfig.manifest.addons);

    await progressCallback({ percent: 70, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    await progressCallback({ percent: 90, message: 'Configuring reverse proxy' });
    if (proxyAuthChanged || httpPortChanged) {
        await reverseProxy.configureApp(app, AuditSource.APPTASK);
    }

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, updateTime: new Date() });
}

async function start(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 10, message: 'Starting app services' });
    await services.startAppServices(app);

    if (app.manifest.id !== constants.PROXY_APP_APPSTORE_ID) {
        await progressCallback({ percent: 35, message: 'Starting container' });
        await docker.startContainer(app.id);
    }

    await progressCallback({ percent: 60, message: 'Adding collectd profile' });
    await addCollectdProfile(app);

    // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
    await progressCallback({ percent: 80, message: 'Configuring reverse proxy' });
    await reverseProxy.configureApp(app, AuditSource.APPTASK);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function stop(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 20, message: 'Stopping container' });
    await reverseProxy.unconfigureApp(app); // removing nginx configs also means that we can auto-cleanup old certs since they are not referenced
    await docker.stopContainers(app.id);

    await progressCallback({ percent: 50, message: 'Stopping app services' });
    await services.stopAppServices(app);

    await progressCallback({ percent: 80, message: 'Removing collectd profile' });
    await removeCollectdProfile(app);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function restart(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    if (app.manifest.id !== constants.PROXY_APP_APPSTORE_ID) {
        await progressCallback({ percent: 20, message: 'Restarting container' });
        await docker.restartContainer(app.id);
    }

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function uninstall(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 20, message: 'Deleting container' });
    await reverseProxy.unconfigureApp(app);
    await deleteContainers(app, {});

    await progressCallback({ percent: 30, message: 'Teardown addons' });
    await services.teardownAddons(app, app.manifest.addons);

    await progressCallback({ percent: 40, message: 'Cleanup file manager' });

    await progressCallback({ percent: 50, message: 'Deleting app data directory' });
    await deleteAppDir(app, { removeDirectory: true });

    await progressCallback({ percent: 60, message: 'Deleting image' });
    await docker.deleteImage(app.manifest);

    await progressCallback({ percent: 70, message: 'Unregistering domains' });
    await dns.unregisterLocations([ { subdomain: app.subdomain, domain: app.domain } ].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), progressCallback);

    await progressCallback({ percent: 90, message: 'Cleanup logs' });
    await cleanupLogs(app);

    await progressCallback({ percent: 95, message: 'Remove app from database' });
    await apps.del(app.id);
}

async function run(appId, args, progressCallback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const app = await apps.get(appId);

    debug(`run: startTask installationState: ${app.installationState} runState: ${app.runState}`);

    let cmd;

    switch (app.installationState) {
    case apps.ISTATE_PENDING_INSTALL:
    case apps.ISTATE_PENDING_CLONE:
    case apps.ISTATE_PENDING_RESTORE:
    case apps.ISTATE_PENDING_IMPORT:
        cmd = install(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_CONFIGURE:
        cmd = configure(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_RECREATE_CONTAINER:
    case apps.ISTATE_PENDING_RESIZE:
    case apps.ISTATE_PENDING_DEBUG:
        cmd = create(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_LOCATION_CHANGE:
        cmd = changeLocation(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_DATA_DIR_MIGRATION:
        cmd = migrateDataDir(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_UNINSTALL:
        cmd = uninstall(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_UPDATE:
        cmd = update(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_BACKUP:
        cmd = backup(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_START:
        cmd = start(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_STOP:
        cmd = stop(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_RESTART:
        cmd = restart(app, args, progressCallback);
        break;
    case apps.ISTATE_INSTALLED: // can only happen when we have a bug in our code while testing/development
        cmd = updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
        break;
    default:
        debug('run: apptask launched with invalid command');
        throw new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState);
    }

    const [error, result] = await safe(cmd); // only some commands like backup return a result
    if (error) {
        debug(`run: app error for state ${app.installationState}:`, error);

        if (app.installationState === apps.ISTATE_PENDING_BACKUP) {
            // return to installed state intentionally. the error is stashed only in the task and not the app (the UI shows error state otherwise)
            await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }), { debug });
        } else if (app.installationState === apps.ISTATE_PENDING_UPDATE && error.backupError) {
            debug('run: update aborted because backup failed');
            await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, { debug }));
        } else {
            await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }), { debug });
        }

        throw error;
    }

    return result || null;
}