cloudron-box/src/apptask.js

#!/usr/bin/env node

'use strict';

exports = module.exports = {
    run,

    // exported for testing
    _createAppDir: createAppDir,
    _deleteAppDir: deleteAppDir,
    _verifyManifest: verifyManifest,
};

const apps = require('./apps.js'),
    appstore = require('./appstore.js'),
    assert = require('assert'),
    AuditSource = require('./auditsource.js'),
    backups = require('./backups.js'),
    backuptask = require('./backuptask.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:apptask'),
    df = require('./df.js'),
    dns = require('./dns.js'),
    docker = require('./docker.js'),
    ejs = require('ejs'),
    fs = require('fs'),
    iputils = require('./iputils.js'),
    manifestFormat = require('cloudron-manifestformat'),
    mounts = require('./mounts.js'),
    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    promiseRetry = require('./promise-retry.js'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
    services = require('./services.js'),
    shell = require('./shell.js')('apptask'),
    _ = require('./underscore.js');

const MV_VOLUME_CMD = path.join(__dirname, 'scripts/mvvolume.sh'),
    LOGROTATE_CONFIG_EJS = fs.readFileSync(__dirname + '/logrotate.ejs', { encoding: 'utf8' }),
    CONFIGURE_LOGROTATE_CMD = path.join(__dirname, 'scripts/configurelogrotate.sh');

function makeTaskError(error, app) {
    assert(error instanceof BoxError);
    assert.strictEqual(typeof app, 'object');

    // track a few variables which helps 'repair' restart the task (see also scheduleTask in apps.js)
    error.details.taskId = app.taskId;
    error.details.installationState = app.installationState;
    return error.toPlainObject();
}

// updates the app object and the database
async function updateApp(app, values) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof values, 'object');

    await apps.update(app.id, values);

    for (const value in values) {
        app[value] = values[value];
    }
}

async function allocateContainerIp(app) {
    assert.strictEqual(typeof app, 'object');

    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;

    await promiseRetry({ times: 10, interval: 0, debug }, async function () {
        const iprange = iputils.intFromIp(constants.APPS_IPv4_END) - iputils.intFromIp(constants.APPS_IPv4_START);
        const rnd = Math.floor(Math.random() * iprange);
        const containerIp = iputils.ipFromInt(iputils.intFromIp(constants.APPS_IPv4_START) + rnd);
        await updateApp(app, { containerIp });
    });
}

async function createContainer(app) {
    assert.strictEqual(typeof app, 'object');
    assert(!app.containerId); // otherwise, it will trigger volumeFrom

    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;

    debug('createContainer: creating container');

    const container = await docker.createContainer(app);

    await updateApp(app, { containerId: container.id });

    // re-generate configs that rely on container id
    await addLogrotateConfig(app);
}

async function deleteContainers(app, options) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');

    debug('deleteContainer: deleting app containers (app, scheduler)');

    // remove configs that rely on container id
    await removeLogrotateConfig(app);
    await docker.stopContainers(app.id);
    await docker.deleteContainers(app.id, options);
    await updateApp(app, { containerId: null });
}

async function createAppDir(app) {
    assert.strictEqual(typeof app, 'object');

    // we have to create app dir regardless of localstorage addon. this dir is used as a temp space for backup dumps
    const appDir = path.join(paths.APPS_DATA_DIR, app.id);
    const [error] = await safe(fs.promises.mkdir(appDir, { recursive: true }));
    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating directory: ${error.message}`);
}

async function deleteAppDir(app, options) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');

    const appDataDir = path.join(paths.APPS_DATA_DIR, app.id);

    // resolve any symlinked data dir
    const stat = safe.fs.lstatSync(appDataDir);
    if (!stat) return;

    const resolvedAppDataDir = stat.isSymbolicLink() ? safe.fs.readlinkSync(appDataDir) : appDataDir;

    if (safe.fs.existsSync(resolvedAppDataDir)) {
        const entries = safe.fs.readdirSync(resolvedAppDataDir);
        if (!entries) throw new BoxError(BoxError.FS_ERROR, `Error listing ${resolvedAppDataDir}: ${safe.error.message}`);

        // remove only files. directories inside app dir are currently volumes managed by the addons
        // we cannot delete those dirs anyway because of perms
        entries.forEach(function (entry) {
            const stat = safe.fs.statSync(path.join(resolvedAppDataDir, entry));
            if (stat && !stat.isDirectory()) safe.fs.unlinkSync(path.join(resolvedAppDataDir, entry));
        });
    }

    // if this fails, it's probably because the localstorage/redis addons have not cleaned up properly
    if (options.removeDirectory) {
        if (stat.isSymbolicLink()) {
            if (!safe.fs.unlinkSync(appDataDir)) {
                if (safe.error.code !== 'ENOENT') throw new BoxError(BoxError.FS_ERROR, `Error unlinking dir ${appDataDir} : ${safe.error.message}`);
            }
        } else {
            if (!safe.fs.rmSync(appDataDir, { recursive: true })) {
                if (safe.error.code !== 'ENOENT') throw new BoxError(BoxError.FS_ERROR, `Error removing dir ${appDataDir} : ${safe.error.message}`);
            }
        }
    }
}

async function addLogrotateConfig(app) {
    assert.strictEqual(typeof app, 'object');

    const result = await docker.inspect(app.containerId);

    const runVolume = result.Mounts.find(function (mount) { return mount.Destination === '/run'; });
    if (!runVolume) throw new BoxError(BoxError.DOCKER_ERROR, 'App does not have /run mounted');

    // logrotate configs can have arbitrary commands, so the config files must be owned by root
    const logrotateConf = ejs.render(LOGROTATE_CONFIG_EJS, { volumePath: runVolume.Source, appId: app.id });
    const tmpFilePath = path.join(os.tmpdir(), app.id + '.logrotate');

    safe.fs.writeFileSync(tmpFilePath, logrotateConf);
    if (safe.error) throw new BoxError(BoxError.FS_ERROR, `Error writing logrotate config: ${safe.error.message}`);

    const [error] = await safe(shell.promises.sudo([ CONFIGURE_LOGROTATE_CMD, 'add', app.id, tmpFilePath ], {}));
    if (error) throw new BoxError(BoxError.LOGROTATE_ERROR, `Error adding logrotate config: ${error.message}`);
}

async function removeLogrotateConfig(app) {
    assert.strictEqual(typeof app, 'object');

    const [error] = await safe(shell.promises.sudo([ CONFIGURE_LOGROTATE_CMD, 'remove', app.id ], {}));
    if (error) throw new BoxError(BoxError.LOGROTATE_ERROR, `Error removing logrotate config: ${error.message}`);
}

async function cleanupLogs(app) {
    assert.strictEqual(typeof app, 'object');

    // note that redis container logs are cleaned up by the addon
    const [error] = await safe(fs.promises.rm(path.join(paths.LOG_DIR, app.id), { force: true, recursive: true }));
    if (error) debug('cleanupLogs: cannot cleanup logs: %o', error);
}

async function verifyManifest(manifest) {
    assert.strictEqual(typeof manifest, 'object');

    let error = manifestFormat.parse(manifest);
    if (error) throw new BoxError(BoxError.BAD_FIELD, `Manifest error: ${error.message}`);

    error = await apps.checkManifest(manifest);
    if (error) throw new BoxError(BoxError.CONFLICT, `Manifest constraint check failed: ${error.message}`);
}

async function downloadIcon(app) {
    assert.strictEqual(typeof app, 'object');

    // nothing to download if we dont have an appStoreId
    if (!app.appStoreId) return;

    debug(`downloadIcon: Downloading icon of ${app.appStoreId}@${app.manifest.version}`);

    const appStoreIcon = await appstore.downloadIcon(app.appStoreId, app.manifest.version);
    await updateApp(app, { appStoreIcon });
}

async function moveDataDir(app, targetVolumeId, targetVolumePrefix) {
    assert.strictEqual(typeof app, 'object');
    assert.ok(app.manifest.addons.localstorage, 'should have local storage addon');
    assert(targetVolumeId === null || typeof targetVolumeId === 'string');
    assert(targetVolumePrefix === null || typeof targetVolumePrefix === 'string');

    const resolvedSourceDir = await apps.getStorageDir(app);
    const resolvedTargetDir = await apps.getStorageDir(Object.assign({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix }));

    debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);

    if (resolvedSourceDir !== resolvedTargetDir) {
        const [error] = await safe(shell.promises.sudo([ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
        if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`);
    }

    await updateApp(app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix });
}

async function downloadImage(manifest) {
    assert.strictEqual(typeof manifest, 'object');

    // skip for relay app
    if (manifest.id === constants.PROXY_APP_APPSTORE_ID) return;

    const info = await docker.info();
    const [dfError, diskUsage] = await safe(df.file(info.DockerRootDir));
    if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error getting file system info: ${dfError.message}`);

    if (diskUsage.available < (1024*1024*1024)) throw new BoxError(BoxError.DOCKER_ERROR, `Not enough disk space to pull docker image. available: ${diskUsage.available}`);

    await docker.downloadImage(manifest);
}

async function updateChecklist(app, newChecks, acknowledged = false) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof newChecks, 'object');
    assert.strictEqual(typeof acknowledged, 'boolean');

    // add new checklist items depending on sso state
    const checklist = app.checklist || {};
    for (const k in newChecks) {
        if (app.checklist[k]) continue;

        const item = {
            acknowledged: acknowledged,
            sso: newChecks[k].sso,
            appVersion: app.version,
            message: newChecks[k].message,
            changedAt: 0,
            changedBy: null, // username from audit log
        };

        if (typeof item.sso === 'undefined') checklist[k] = item;
        else if (item.sso && app.sso) checklist[k] = item;
        else if (!item.sso && !app.sso) checklist[k] = item;
    }

    await updateApp(app, { checklist });
}


async function startApp(app) {
    debug('startApp: starting container');

    if (app.runState === apps.RSTATE_STOPPED) return;

    // skip for relay app
    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;

    await docker.startContainer(app.id);
}

async function installCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const restoreConfig = args.restoreConfig; // has to be set when restoring
    const overwriteDns = args.overwriteDns;
    const skipDnsSetup = args.skipDnsSetup;
    const oldManifest = args.oldManifest;

    // this protects against the theoretical possibility of an app being marked for install/restore from
    // a previous version of box code
    await verifyManifest(app.manifest);

    // teardown for re-installs
    await progressCallback({ percent: 10, message: 'Deleting old containers' });
    await reverseProxy.unconfigureApp(app);
    await deleteContainers(app, { managedOnly: true });

    // when restoring, app does not require these addons anymore. remove carefully to preserve the db passwords
    let addonsToRemove;
    if (oldManifest) {
        addonsToRemove = _.omit(oldManifest.addons, Object.keys(app.manifest.addons));
    } else {
        addonsToRemove = app.manifest.addons;
    }
    await services.teardownAddons(app, addonsToRemove);

    if (!restoreConfig || restoreConfig.remotePath) { // in-place import should not delete data dir
        await deleteAppDir(app, { removeDirectory: false }); // do not remove any symlinked appdata dir
    }

    if (oldManifest && oldManifest.dockerImage !== app.manifest.dockerImage) {
        await docker.deleteImage(oldManifest.dockerImage);
    }

    // allocating container ip here, lets the users "repair" an app if allocation fails at apps.add time
    await allocateContainerIp(app);

    await progressCallback({ percent: 20, message: 'Downloading icon' });
    await downloadIcon(app);

    await progressCallback({ percent: 25, message: 'Updating checklist' });
    await updateChecklist(app, app.manifest.checklist || {}, restoreConfig ? true : false);

    if (!skipDnsSetup) {
        await progressCallback({ percent: 30, message: 'Registering subdomains' });

        await dns.registerLocations([ { subdomain: app.subdomain, domain: app.domain }].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback);
    }

    await progressCallback({ percent: 40, message: 'Downloading image' });
    await downloadImage(app.manifest);

    await progressCallback({ percent: 50, message: 'Creating app data directory' });
    await createAppDir(app);

    if (!restoreConfig) { // install
        await progressCallback({ percent: 60, message: 'Setting up addons' });
        await services.setupAddons(app, app.manifest.addons);
    } else if (app.installationState === apps.ISTATE_PENDING_IMPORT && !restoreConfig.remotePath) { // in-place import
        await progressCallback({ percent: 60, message: 'Importing addons in-place' });
        await services.setupAddons(app, app.manifest.addons);
        await services.clearAddons(app, _.omit(app.manifest.addons, ['localstorage']));
        await apps.loadConfig(app);
        await services.restoreAddons(app, app.manifest.addons);
    } else if (app.installationState === apps.ISTATE_PENDING_IMPORT) { // import
        await progressCallback({ percent: 65, message: 'Downloading backup and restoring addons' });
        await services.setupAddons(app, app.manifest.addons);
        await services.clearAddons(app, app.manifest.addons);
        const backupConfig = restoreConfig.backupConfig;
        const mountObject = await backups.setupManagedStorage(backupConfig, `/mnt/appimport-${app.id}`);
        if (mountObject) await progressCallback({ percent: 70, message: 'Setting up mount for importing' });
        backupConfig.rootPath = backups.getRootPath(backupConfig, `/mnt/appimport-${app.id}`);
        await backuptask.downloadApp(app, restoreConfig, (progress) => { progressCallback({ percent: 75, message: progress.message }); });
        await apps.loadConfig(app);
        if (mountObject) await mounts.removeMount(mountObject);
        await progressCallback({ percent: 75, message: 'Restoring addons' });
        await services.restoreAddons(app, app.manifest.addons);
    } else { // clone and restore
        await progressCallback({ percent: 65, message: 'Downloading backup and restoring addons' });
        await services.setupAddons(app, app.manifest.addons);
        await services.clearAddons(app, app.manifest.addons);
        await backuptask.downloadApp(app, restoreConfig, (progress) => { progressCallback({ percent: 65, message: progress.message }); });
        if (app.installationState === apps.ISTATE_PENDING_CLONE) {
            const customIcon = safe.fs.readFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/icon.png'));
            if (customIcon) await updateApp(app, { icon: customIcon });
        }
        await progressCallback({ percent: 70, message: 'Restoring addons' });
        await services.restoreAddons(app, app.manifest.addons);
    }

    await progressCallback({ percent: 80, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    if (!skipDnsSetup) {
        await progressCallback({ percent: 85, message: 'Waiting for DNS propagation' });
        await dns.waitForLocations([ { subdomain: app.subdomain, domain: app.domain }].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), progressCallback);
    }

    await progressCallback({ percent: 95, message: 'Configuring reverse proxy' });
    await reverseProxy.configureApp(app, AuditSource.APPTASK);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function backupCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 10, message: 'Backing up' });
    const backupId = await backuptask.backupApp(app, { snapshotOnly: !!args.snapshotOnly }, (progress) => {
        progressCallback({ percent: 30, message: progress.message });
    });

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null });
    return backupId;
}

async function createCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 10, message: 'Deleting old container' });
    await deleteContainers(app, { managedOnly: true });

    // FIXME: re-setup addons only because sendmail addon to re-inject env vars on mailboxName change
    await progressCallback({ percent: 30, message: 'Setting up addons' });
    await services.setupAddons(app, app.manifest.addons);

    await progressCallback({ percent: 60, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function changeLocationCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const oldConfig = args.oldConfig;
    const locationChanged = oldConfig.fqdn !== app.fqdn;
    const skipDnsSetup = args.skipDnsSetup;
    const overwriteDns = args.overwriteDns;

    await progressCallback({ percent: 10, message: 'Unregistering old domains' });
    await reverseProxy.unconfigureApp(app);
    await deleteContainers(app, { managedOnly: true });

    // unregister old domains
    let obsoleteDomains = [];

    if (oldConfig.secondaryDomains) {
        obsoleteDomains = obsoleteDomains.concat(oldConfig.secondaryDomains.filter(function (o) {
            return !app.secondaryDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
        }));
    }

    if (oldConfig.redirectDomains) {
        obsoleteDomains = obsoleteDomains.concat(oldConfig.redirectDomains.filter(function (o) {
            return !app.redirectDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
        }));
    }

    if (oldConfig.aliasDomains) {
        obsoleteDomains = obsoleteDomains.concat(oldConfig.aliasDomains.filter(function (o) {
            return !app.aliasDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
        }));
    }

    if (locationChanged) obsoleteDomains.push({ subdomain: oldConfig.subdomain, domain: oldConfig.domain });

    if (obsoleteDomains.length !== 0) await dns.unregisterLocations(obsoleteDomains, progressCallback);

    // setup dns
    if (!skipDnsSetup) {
        await progressCallback({ percent: 30, message: 'Registering subdomains' });
        await dns.registerLocations([ { subdomain: app.subdomain, domain: app.domain }].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback);
    }

    // re-setup addons since they rely on the app's fqdn (e.g oauth)
    await progressCallback({ percent: 50, message: 'Setting up addons' });
    await services.setupAddons(app, app.manifest.addons);

    await progressCallback({ percent: 60, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    if (!skipDnsSetup) {
        await progressCallback({ percent: 80, message: 'Waiting for DNS propagation' });
        await dns.waitForLocations([ { subdomain: app.subdomain, domain: app.domain }].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), progressCallback);
    }

    await progressCallback({ percent: 90, message: 'Configuring reverse proxy' });
    await reverseProxy.configureApp(app, AuditSource.APPTASK);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function changeServicesCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 10, message: 'Deleting old containers' });
    await deleteContainers(app, { managedOnly: true });

    const unusedAddons = {};
    if (app.manifest.addons?.turn && !app.enableTurn) unusedAddons.turn = app.manifest.addons.turn;
    if (app.manifest.addons?.sendmail && !app.enableMailbox) unusedAddons.sendmail = app.manifest.addons.sendmail;
    if (app.manifest.addons?.recvmail && !app.enableInbox) unusedAddons.recvmail = app.manifest.addons.recvmail;
    if (app.manifest.addons?.redis && !app.enableRedis) unusedAddons.redis = app.manifest.addons.redis;

    await progressCallback({ percent: 20, message: 'Removing unused addons' });
    await services.teardownAddons(app, unusedAddons);

    await progressCallback({ percent: 40, message: 'Setting up addons' });
    await services.setupAddons(app, app.manifest.addons);

    await progressCallback({ percent: 60, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function migrateDataDirCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const { newStorageVolumeId, newStorageVolumePrefix } = args;
    assert(newStorageVolumeId === null || typeof newStorageVolumeId === 'string');
    assert(newStorageVolumePrefix === null || typeof newStorageVolumePrefix === 'string');

    await progressCallback({ percent: 10, message: 'Deleting old containers' });
    await deleteContainers(app, { managedOnly: true });

    await progressCallback({ percent: 45, message: 'Ensuring app data directory' });
    await createAppDir(app);

    // re-setup addons since this creates the localStorage destination
    await progressCallback({ percent: 50, message: 'Setting up addons' });
    await services.setupAddons(Object.assign({}, app, { storageVolumeId: newStorageVolumeId, storageVolumePrefix: newStorageVolumePrefix }), app.manifest.addons);

    if (app.manifest.addons?.localstorage) {
        await progressCallback({ percent: 60, message: 'Moving data dir' });
        await moveDataDir(app, newStorageVolumeId, newStorageVolumePrefix);
    }

    await progressCallback({ percent: 90, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

// configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local"
async function configureCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 10, message: 'Deleting old containers' });
    await reverseProxy.unconfigureApp(app);
    await deleteContainers(app, { managedOnly: true });

    await progressCallback({ percent: 20, message: 'Downloading icon' });
    await downloadIcon(app);

    await progressCallback({ percent: 40, message: 'Downloading image' });
    await downloadImage(app.manifest);

    await progressCallback({ percent: 45, message: 'Ensuring app data directory' });
    await createAppDir(app);

    // re-setup addons since they rely on the app's fqdn (e.g oauth)
    await progressCallback({ percent: 50, message: 'Setting up addons' });
    await services.setupAddons(app, app.manifest.addons);

    await progressCallback({ percent: 60, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    await progressCallback({ percent: 90, message: 'Configuring reverse proxy' });
    await reverseProxy.configureApp(app, AuditSource.APPTASK);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function updateCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const updateConfig = args.updateConfig;

    // app does not want these addons anymore
    // FIXME: this does not handle option changes (like multipleDatabases)
    const unusedAddons = _.omit(app.manifest.addons, Object.keys(updateConfig.manifest.addons));
    const httpPortChanged = app.manifest.httpPort !== updateConfig.manifest.httpPort;
    const proxyAuthChanged = !_.isEqual(app.manifest.addons?.proxyAuth, updateConfig.manifest.addons?.proxyAuth);

    // this protects against the theoretical possibility of an app being marked for update from
    // a previous version of box code
    await progressCallback({ percent: 5, message: 'Verify manifest' });
    await verifyManifest(updateConfig.manifest);

    if (!updateConfig.skipBackup) {
        await progressCallback({ percent: 15, message: 'Backing up app' });
        // preserve update backups for 3 weeks
        const [error] = await safe(backuptask.backupApp(app, { preserveSecs: 3*7*24*60*60 }, (progress) => {
            progressCallback({ percent: 15, message: `Backup - ${progress.message}` });
        }));
        if (error) {
            error.backupError = true;
            throw error;
        }
    }

    await progressCallback({ percent: 20, message: 'Updating checklist' });
    await updateChecklist(app, app.manifest.checklist || {}, true /* new state acked */);

    // download new image before app is stopped. this is so we can reduce downtime
    // and also not remove the 'common' layers when the old image is deleted
    await progressCallback({ percent: 25, message: 'Downloading image' });
    await downloadImage(updateConfig.manifest);

    // note: we cleanup first and then backup. this is done so that the app is not running should backup fail
    // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
    await progressCallback({ percent: 35, message: 'Deleting old containers' });
    await deleteContainers(app, { managedOnly: true });
    if (app.manifest.dockerImage !== updateConfig.manifest.dockerImage) await docker.deleteImage(app.manifest.dockerImage);

    // only delete unused addons after backup
    await services.teardownAddons(app, unusedAddons);
    if (Object.keys(unusedAddons).includes('localstorage')) await updateApp(app, { storageVolumeId: null, storageVolumePrefix: null }); // lose reference

    // free unused ports. this is done after backup, so the app object is not in some inconsistent state should backup fail
    const newTcpPorts = updateConfig.manifest.tcpPorts || {};
    const newUdpPorts = updateConfig.manifest.udpPorts || {};
    const portBindings = { ...app.portBindings };

    for (const portName of Object.keys(portBindings)) {
        if (newTcpPorts[portName] || newUdpPorts[portName]) continue; // port still in use
        delete portBindings[portName];
    }

    // clear aliasDomains if needed based multiDomain change
    const aliasDomains = app.manifest.multiDomain && !updateConfig.manifest.multiDomain ? [] : app.aliasDomains;

    // clear unused secondaryDomains
    const secondaryDomains = [ ...app.secondaryDomains ];
    const newHttpPorts = updateConfig.manifest.httpPorts || {};
    for (let i = secondaryDomains.length-1; i >= 0; i--) {
        const { environmentVariable } = secondaryDomains[i];
        if (environmentVariable in newHttpPorts) continue; // domain still in use
        secondaryDomains.splice(i, 1); // remove domain
    }

    const values = {
        manifest: updateConfig.manifest,
        portBindings,
        // all domains have to be updated together
        subdomain: app.subdomain,
        domain: app.domain,
        aliasDomains,
        secondaryDomains,
        redirectDomains: app.redirectDomains
    };
    if ('memoryLimit' in updateConfig) values.memoryLimit = updateConfig.memoryLimit;
    if ('appStoreId' in updateConfig) values.appStoreId = updateConfig.appStoreId;

    await updateApp(app, values); // switch over to the new config

    await progressCallback({ percent: 45, message: 'Downloading icon' });
    await downloadIcon(app);

    await progressCallback({ percent: 60, message: 'Updating addons' });
    await services.setupAddons(app, updateConfig.manifest.addons);

    await progressCallback({ percent: 70, message: 'Creating container' });
    await createContainer(app);

    await startApp(app);

    await progressCallback({ percent: 90, message: 'Configuring reverse proxy' });
    if (proxyAuthChanged || httpPortChanged) {
        await reverseProxy.configureApp(app, AuditSource.APPTASK);
    }

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, updateTime: new Date() });
}

async function startCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 10, message: 'Starting app services' });
    await services.startAppServices(app);

    if (app.manifest.id !== constants.PROXY_APP_APPSTORE_ID) {
        await progressCallback({ percent: 35, message: 'Starting container' });
        await docker.startContainer(app.id);
    }

    // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
    await progressCallback({ percent: 80, message: 'Configuring reverse proxy' });
    await reverseProxy.configureApp(app, AuditSource.APPTASK);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function stopCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    // we don't delete the containers. app containers are created with the unless-stopped restart policy. there is no danger of apps getting restarted on reboot
    await progressCallback({ percent: 20, message: 'Stopping container' });
    await reverseProxy.unconfigureApp(app); // removing nginx configs also means that we can auto-cleanup old certs since they are not referenced
    await docker.stopContainers(app.id);

    await progressCallback({ percent: 50, message: 'Stopping app services' });
    await services.stopAppServices(app);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function restartCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    if (app.manifest.id !== constants.PROXY_APP_APPSTORE_ID) {
        await progressCallback({ percent: 10, message: 'Starting app services' });
        await services.startAppServices(app);

        await progressCallback({ percent: 20, message: 'Restarting container' });
        await docker.restartContainer(app.id);
    }

    // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
    await progressCallback({ percent: 80, message: 'Configuring reverse proxy' });
    await reverseProxy.configureApp(app, AuditSource.APPTASK);

    await progressCallback({ percent: 100, message: 'Done' });
    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
}

async function uninstallCommand(app, args, progressCallback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    await progressCallback({ percent: 20, message: 'Deleting container' });
    await reverseProxy.unconfigureApp(app);
    await deleteContainers(app, {});

    await progressCallback({ percent: 30, message: 'Teardown addons' });
    await services.teardownAddons(app, app.manifest.addons);

    await progressCallback({ percent: 40, message: 'Cleanup file manager' });

    await progressCallback({ percent: 50, message: 'Deleting app data directory' });
    await deleteAppDir(app, { removeDirectory: true });

    await progressCallback({ percent: 60, message: 'Deleting image' });
    await docker.deleteImage(app.manifest.dockerImage);

    await progressCallback({ percent: 70, message: 'Unregistering domains' });
    await dns.unregisterLocations([ { subdomain: app.subdomain, domain: app.domain } ].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), progressCallback);

    await progressCallback({ percent: 90, message: 'Cleanup logs' });
    await cleanupLogs(app);

    await progressCallback({ percent: 95, message: 'Remove app from database' });
    await apps.del(app.id);
}

async function run(appId, args, progressCallback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const app = await apps.get(appId);

    debug(`run: startTask installationState: ${app.installationState} runState: ${app.runState}`);

    let cmd;

    switch (app.installationState) {
    case apps.ISTATE_PENDING_INSTALL:
    case apps.ISTATE_PENDING_CLONE:
    case apps.ISTATE_PENDING_RESTORE:
    case apps.ISTATE_PENDING_IMPORT:
        cmd = installCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_CONFIGURE:
        cmd = configureCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_RECREATE_CONTAINER: // mounts/devices/env
    case apps.ISTATE_PENDING_RESIZE: // memory/cpu
    case apps.ISTATE_PENDING_DEBUG:
        cmd = createCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_LOCATION_CHANGE:
        cmd = changeLocationCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_SERVICES_CHANGE:
        cmd = changeServicesCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_DATA_DIR_MIGRATION:
        cmd = migrateDataDirCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_UNINSTALL:
        cmd = uninstallCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_UPDATE:
        cmd = updateCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_BACKUP:
        cmd = backupCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_START:
        cmd = startCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_STOP:
        cmd = stopCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_PENDING_RESTART:
        cmd = restartCommand(app, args, progressCallback);
        break;
    case apps.ISTATE_INSTALLED: // can only happen when we have a bug in our code while testing/development
        cmd = updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
        break;
    default:
        debug('run: apptask launched with invalid command');
        throw new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState);
    }

    const [error, result] = await safe(cmd); // only some commands like backup return a result
    if (error) {
        debug(`run: app error for state ${app.installationState}: %o`, error);

        if (app.installationState === apps.ISTATE_PENDING_BACKUP) {
            // return to installed state intentionally. the error is stashed only in the task and not the app (the UI shows error state otherwise)
            await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }), { debug });
        } else if (app.installationState === apps.ISTATE_PENDING_UPDATE && error.backupError) {
            debug('run: update aborted because backup failed');
            await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, { debug }));
        } else {
            await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }), { debug });
        }

        throw error;
    }

    return result || null;
}