cloudron-box/src/apps.js

'use strict';

exports = module.exports = {
    hasAccessTo,
    removeInternalFields,
    removeRestrictedFields,

    get,
    getByIpAddress,
    getByFqdn,
    getAll,
    getAllByUser,
    install,
    uninstall,

    setAccessRestriction,
    setLabel,
    setIcon,
    setTags,
    setMemoryLimit,
    setCpuShares,
    setMounts,
    setAutomaticBackup,
    setAutomaticUpdate,
    setReverseProxyConfig,
    setCertificate,
    setDebugMode,
    setEnvironment,
    setMailbox,
    setLocation,
    setDataDir,
    repair,

    restore,
    importApp,
    exportApp,
    clone,

    update,

    backup,
    listBackups,

    getLocalLogfilePaths,
    getLogs,

    getCertificate,

    start,
    stop,
    restart,

    exec,

    checkManifestConstraints,
    downloadManifest,

    canAutoupdateApp,
    autoupdateApps,

    restoreInstalledApps,
    configureInstalledApps,
    schedulePendingTasks,
    restartAppsUsingAddons,

    getDataDir,
    getIcon,
    getMemoryLimit,

    downloadFile,
    uploadFile,

    backupConfig,
    restoreConfig,

    PORT_TYPE_TCP: 'tcp',
    PORT_TYPE_UDP: 'udp',

    // task codes - the installation state is now a misnomer (keep in sync in UI)
    ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
    ISTATE_PENDING_CLONE: 'pending_clone', // clone
    ISTATE_PENDING_CONFIGURE: 'pending_configure', // infra update
    ISTATE_PENDING_RECREATE_CONTAINER: 'pending_recreate_container', // env change or addon change
    ISTATE_PENDING_LOCATION_CHANGE: 'pending_location_change',
    ISTATE_PENDING_DATA_DIR_MIGRATION: 'pending_data_dir_migration',
    ISTATE_PENDING_RESIZE: 'pending_resize',
    ISTATE_PENDING_DEBUG: 'pending_debug',
    ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
    ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
    ISTATE_PENDING_IMPORT: 'pending_import', // import from external backup
    ISTATE_PENDING_UPDATE: 'pending_update', // update from installed state preserving data
    ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app. this is state because it blocks other operations
    ISTATE_PENDING_START: 'pending_start',
    ISTATE_PENDING_STOP: 'pending_stop',
    ISTATE_PENDING_RESTART: 'pending_restart',
    ISTATE_ERROR: 'error', // error executing last pending_* command
    ISTATE_INSTALLED: 'installed', // app is installed

    // run states
    RSTATE_RUNNING: 'running',
    RSTATE_STOPPED: 'stopped', // app stopped by us

    // health states (keep in sync in UI)
    HEALTH_HEALTHY: 'healthy',
    HEALTH_UNHEALTHY: 'unhealthy',
    HEALTH_ERROR: 'error',
    HEALTH_DEAD: 'dead',

    // exported for testing
    _validatePortBindings: validatePortBindings,
    _validateAccessRestriction: validateAccessRestriction,
    _translatePortBindings: translatePortBindings,
};

const appdb = require('./appdb.js'),
    appstore = require('./appstore.js'),
    appTaskManager = require('./apptaskmanager.js'),
    assert = require('assert'),
    async = require('async'),
    backups = require('./backups.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    database = require('./database.js'),
    debug = require('debug')('box:apps'),
    docker = require('./docker.js'),
    domaindb = require('./domaindb.js'),
    domains = require('./domains.js'),
    eventlog = require('./eventlog.js'),
    fs = require('fs'),
    mail = require('./mail.js'),
    manifestFormat = require('cloudron-manifestformat'),
    once = require('once'),
    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
    semver = require('semver'),
    services = require('./services.js'),
    settings = require('./settings.js'),
    spawn = require('child_process').spawn,
    split = require('split'),
    superagent = require('superagent'),
    tasks = require('./tasks.js'),
    TransformStream = require('stream').Transform,
    users = require('./users.js'),
    util = require('util'),
    uuid = require('uuid'),
    validator = require('validator'),
    _ = require('underscore');

const NOOP_CALLBACK = function (error) { if (error) debug(error); };

// validate the port bindings
function validatePortBindings(portBindings, manifest) {
    assert.strictEqual(typeof portBindings, 'object');
    assert.strictEqual(typeof manifest, 'object');

    // keep the public ports in sync with firewall rules in setup/start/cloudron-firewall.sh
    // these ports are reserved even if we listen only on 127.0.0.1 because we setup HostIp to be 127.0.0.1
    // for custom tcp ports
    const RESERVED_PORTS = [
        22, /* ssh */
        25, /* smtp */
        80, /* http */
        143, /* imap */
        202, /* alternate ssh */
        222, /* proftd */
        443, /* https */
        465, /* smtps */
        587, /* submission */
        993, /* imaps */
        2003, /* graphite (lo) */
        2004, /* graphite (lo) */
        2514, /* cloudron-syslog (lo) */
        constants.PORT, /* app server (lo) */
        constants.AUTHWALL_PORT, /* protected sites */
        constants.INTERNAL_SMTP_PORT, /* internal smtp port (lo) */
        constants.LDAP_PORT,
        3306, /* mysql (lo) */
        3478, /* turn,stun */
        4190, /* managesieve */
        5349, /* turn,stun TLS */
        8000, /* ESXi monitoring */
        8417, /* graphite (lo) */
    ];

    const RESERVED_PORT_RANGES = [
        [50000, 51000] /* turn udp ports */
    ];

    const ALLOWED_PORTS = [
        53, // dns 53 is special and adblocker apps can use them
        853 // dns over tls
    ];

    if (!portBindings) return null;

    for (let portName in portBindings) {
        if (!/^[a-zA-Z0-9_]+$/.test(portName)) return new BoxError(BoxError.BAD_FIELD, `${portName} is not a valid environment variable`, { field: 'portBindings', portName: portName });

        const hostPort = portBindings[portName];
        if (!Number.isInteger(hostPort)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not an integer`, { field: 'portBindings', portName: portName });
        if (RESERVED_PORTS.indexOf(hostPort) !== -1) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
        if (RESERVED_PORT_RANGES.find(range => (hostPort >= range[0] && hostPort <= range[1]))) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
        if (ALLOWED_PORTS.indexOf(hostPort) === -1 && (hostPort <= 1023 || hostPort > 65535)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not in permitted range`, { field: 'portBindings', portName: portName });
    }

    // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
    // that the user wants the service disabled
    const tcpPorts = manifest.tcpPorts || { };
    const udpPorts = manifest.udpPorts || { };
    for (let portName in portBindings) {
        if (!(portName in tcpPorts) && !(portName in udpPorts)) return new BoxError(BoxError.BAD_FIELD, `Invalid portBindings ${portName}`, { field: 'portBindings', portName: portName });
    }

    return null;
}

function translatePortBindings(portBindings, manifest) {
    assert.strictEqual(typeof portBindings, 'object');
    assert.strictEqual(typeof manifest, 'object');

    if (!portBindings) return null;

    let result = {};
    const tcpPorts = manifest.tcpPorts || { };

    for (let portName in portBindings) {
        const portType = portName in tcpPorts ? exports.PORT_TYPE_TCP : exports.PORT_TYPE_UDP;
        result[portName] = { hostPort: portBindings[portName], type: portType };
    }
    return result;
}

function validateAccessRestriction(accessRestriction) {
    assert.strictEqual(typeof accessRestriction, 'object');

    if (accessRestriction === null) return null;

    if (accessRestriction.users) {
        if (!Array.isArray(accessRestriction.users)) return new BoxError(BoxError.BAD_FIELD, 'users array property required');
        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new BoxError(BoxError.BAD_FIELD, 'All users have to be strings');
    }

    if (accessRestriction.groups) {
        if (!Array.isArray(accessRestriction.groups)) return new BoxError(BoxError.BAD_FIELD, 'groups array property required');
        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new BoxError(BoxError.BAD_FIELD, 'All groups have to be strings');
    }

    // TODO: maybe validate if the users and groups actually exist
    return null;
}

function validateMemoryLimit(manifest, memoryLimit) {
    assert.strictEqual(typeof manifest, 'object');
    assert.strictEqual(typeof memoryLimit, 'number');

    var min = manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
    var max = os.totalmem() * 2; // this will overallocate since we don't allocate equal swap always (#466)

    // allow 0, which indicates that it is not set, the one from the manifest will be choosen but we don't commit any user value
    // this is needed so an app update can change the value in the manifest, and if not set by the user, the new value should be used
    if (memoryLimit === 0) return null;

    // a special value that indicates unlimited memory
    if (memoryLimit === -1) return null;

    if (memoryLimit < min) return new BoxError(BoxError.BAD_FIELD, 'memoryLimit too small');
    if (memoryLimit > max) return new BoxError(BoxError.BAD_FIELD, 'memoryLimit too large');

    return null;
}

function validateCpuShares(cpuShares) {
    assert.strictEqual(typeof cpuShares, 'number');

    if (cpuShares < 2 || cpuShares > 1024) return new BoxError(BoxError.BAD_FIELD, 'cpuShares has to be between 2 and 1024');

    return null;
}

function validateDebugMode(debugMode) {
    assert.strictEqual(typeof debugMode, 'object');

    if (debugMode === null) return null;
    if ('cmd' in debugMode && debugMode.cmd !== null && !Array.isArray(debugMode.cmd)) return new BoxError(BoxError.BAD_FIELD, 'debugMode.cmd must be an array or null' );
    if ('readonlyRootfs' in debugMode && typeof debugMode.readonlyRootfs !== 'boolean') return new BoxError(BoxError.BAD_FIELD, 'debugMode.readonlyRootfs must be a boolean' );

    return null;
}

function validateRobotsTxt(robotsTxt) {
    if (robotsTxt === null) return null;

    // this is the nginx limit on inline strings. if we really hit this, we have to generate a file
    if (robotsTxt.length > 4096) return new BoxError(BoxError.BAD_FIELD, 'robotsTxt must be less than 4096', { field: 'robotsTxt' });

    // TODO: validate the robots file? we escape the string when templating the nginx config right now

    return null;
}

function validateCsp(csp) {
    if (csp === null) return null;

    if (csp.length > 4096) return new BoxError(BoxError.BAD_FIELD, 'CSP must be less than 4096', { field: 'csp' });

    if (csp.includes('"')) return new BoxError(BoxError.BAD_FIELD, 'CSP cannot contains double quotes', { field: 'csp' });

    return null;
}

function validateBackupFormat(format) {
    assert.strictEqual(typeof format, 'string');

    if (format === 'tgz' || format == 'rsync') return null;

    return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format');
}

function validateLabel(label) {
    if (label === null) return null;

    if (label.length > 128) return new BoxError(BoxError.BAD_FIELD, 'label must be less than 128', { field: 'label' });

    return null;
}

function validateTags(tags) {
    if (tags.length > 64) return new BoxError(BoxError.BAD_FIELD, 'Can only set up to 64 tags', { field: 'tags' });

    if (tags.some(tag => tag.length == 0)) return new BoxError(BoxError.BAD_FIELD, 'tag cannot be empty', { field: 'tags' });
    if (tags.some(tag => tag.length > 128)) return new BoxError(BoxError.BAD_FIELD, 'tag must be less than 128', { field: 'tags' });

    return null;
}

function validateEnv(env) {
    for (let key in env) {
        if (key.length > 512) return new BoxError(BoxError.BAD_FIELD, 'Max env var key length is 512', { field: 'env', env: env });
        // http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap08.html
        if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) return new BoxError(BoxError.BAD_FIELD, `"${key}" is not a valid environment variable`, { field: 'env', env: env });
    }

    return null;
}

function validateDataDir(dataDir) {
    if (dataDir === null) return null;

    if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`, { field: 'dataDir' });
    if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`, { field: 'dataDir' });
    if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`, { field: 'dataDir' });

    // nfs shares will have the directory mounted already
    let stat = safe.fs.lstatSync(dataDir);
    if (stat) {
        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`, { field: 'dataDir' });
        let entries = safe.fs.readdirSync(dataDir);
        if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`, { field: 'dataDir' });
        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`, { field: 'dataDir' });
    }

    // backup logic relies on paths not overlapping (because it recurses)
    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`, { field: 'dataDir' });

    // if we made it this far, it cannot start with any of these realistically
    const fhs = [ '/bin', '/boot', '/etc', '/lib', '/lib32', '/lib64', '/proc', '/run', '/sbin', '/tmp', '/usr' ];
    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`, { field: 'dataDir' });

    return null;
}

function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, portBindings) {
    assert.strictEqual(typeof errorMessage, 'string');
    assert(Array.isArray(locations));
    assert.strictEqual(typeof domainObjectMap, 'object');
    assert.strictEqual(typeof portBindings, 'object');

    var match = errorMessage.match(/ER_DUP_ENTRY: Duplicate entry '(.*)' for key '(.*)'/);
    if (!match) {
        debug('Unexpected SQL error message.', errorMessage);
        return new BoxError(BoxError.DATABASE_ERROR, errorMessage);
    }

    // check if a location conflicts
    if (match[2] === 'subdomain') {
        for (let i = 0; i < locations.length; i++) {
            const { subdomain, domain } = locations[i];
            if  (match[1] !== `${subdomain}-${domain}`) continue;

            return new BoxError(BoxError.ALREADY_EXISTS, `Domain '${domains.fqdn(subdomain, domainObjectMap[domain])}' is in use`, { subdomain, domain });
        }
    }

    // check if any of the port bindings conflict
    for (let portName in portBindings) {
        if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is in use`, { portName });
    }

    if (match[2] === 'dataDir') {
        return new BoxError(BoxError.BAD_FIELD, `Data directory ${match[1]} is in use`, { field: 'dataDir' });
    }

    return new BoxError(BoxError.ALREADY_EXISTS, `${match[2]} '${match[1]}' is in use`);
}

function getDataDir(app, dataDir) {
    assert(dataDir === null || typeof dataDir === 'string');

    return dataDir || path.join(paths.APPS_DATA_DIR, app.id, 'data');
}

function removeInternalFields(app) {
    return _.pick(app,
        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId',
        'location', 'domain', 'fqdn', 'mailboxName', 'mailboxDomain',
        'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares',
        'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
        'label', 'alternateDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts', 'enableMailbox');
}

// non-admins can only see these
function removeRestrictedFields(app) {
    return _.pick(app,
        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'accessRestriction', 'alternateDomains', 'aliasDomains', 'sso',
        'location', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup');
}

function getIcon(app, options, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

    appdb.getIcons(app.id, function (error, icons) {
        if (error) return callback(error);

        if (!options.original && icons.icon) return callback(null, icons.icon);

        if (icons.appStoreIcon) return callback(null, icons.appStoreIcon);

        callback(new BoxError(BoxError.NOT_FOUND, 'No icon'));
    });
}

function getMemoryLimit(app) {
    assert.strictEqual(typeof app, 'object');

    let memoryLimit = app.memoryLimit || app.manifest.memoryLimit || 0;

    if (memoryLimit === -1) { // unrestricted
        memoryLimit = 0;
    } else if (memoryLimit === 0 || memoryLimit < constants.DEFAULT_MEMORY_LIMIT) { // ensure we never go below minimum (in case we change the default)
        memoryLimit = constants.DEFAULT_MEMORY_LIMIT;
    }

    return memoryLimit;
}

function postProcess(app, domainObjectMap) {
    let result = {};
    for (let portName in app.portBindings) {
        result[portName] = app.portBindings[portName].hostPort;
    }
    app.portBindings = result;
    app.iconUrl = app.hasIcon || app.hasAppStoreIcon  ? `/api/v1/apps/${app.id}/icon` : null;
    app.fqdn = domains.fqdn(app.location, domainObjectMap[app.domain]);
    app.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
    app.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
}

function hasAccessTo(app, user, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof user, 'object');
    assert.strictEqual(typeof callback, 'function');

    if (app.accessRestriction === null) return callback(null, true);

    // check user access
    if (app.accessRestriction.users.some(function (e) { return e === user.id; })) return callback(null, true);

    if (users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) return callback(null, true); // admins can always access any app

    if (!app.accessRestriction.groups) return callback(null, false);

    if (app.accessRestriction.groups.some(function (gid) { return Array.isArray(user.groupIds) && user.groupIds.indexOf(gid) !== -1; })) return callback(null, true);

    callback(null, false);
}

function getDomainObjectMap(callback) {
    assert.strictEqual(typeof callback, 'function');

    domaindb.getAll(function (error, domainObjects) {
        if (error) return callback(error);

        let domainObjectMap = {};
        for (let d of domainObjects) { domainObjectMap[d.domain] = d; }

        callback(null, domainObjectMap);
    });
}

function get(appId, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof callback, 'function');

    getDomainObjectMap(function (error, domainObjectMap) {
        if (error) return callback(error);

        appdb.get(appId, function (error, app) {
            if (error) return callback(error);

            postProcess(app, domainObjectMap);

            callback(null, app);
        });
    });
}

// returns the app associated with this IP (app or scheduler)
function getByIpAddress(ip, callback) {
    assert.strictEqual(typeof ip, 'string');
    assert.strictEqual(typeof callback, 'function');

    appdb.getByIpAddress(ip, function (error, app) {
        if (error) return callback(error);

        getDomainObjectMap(function (error, domainObjectMap) {
            if (error) return callback(error);

            postProcess(app, domainObjectMap);

            callback(null, app);
        });
    });
}

function getByFqdn(fqdn, callback) {
    assert.strictEqual(typeof fqdn, 'string');
    assert.strictEqual(typeof callback, 'function');

    getAll(function (error, result) {
        if (error) return callback(error);

        var app = result.find(function (a) { return a.fqdn === fqdn; });
        if (!app) return callback(new BoxError(BoxError.NOT_FOUND, 'No such app'));

        callback(null, app);
    });
}

function getAll(callback) {
    assert.strictEqual(typeof callback, 'function');

    getDomainObjectMap(function (error, domainObjectMap) {
        if (error) return callback(error);

        appdb.getAll(function (error, apps) {
            if (error) return callback(error);

            apps.forEach((app) => postProcess(app, domainObjectMap));

            callback(null, apps);
        });
    });
}

function getAllByUser(user, callback) {
    assert.strictEqual(typeof user, 'object');
    assert.strictEqual(typeof callback, 'function');

    getAll(function (error, result) {
        if (error) return callback(error);

        async.filter(result, function (app, iteratorDone) {
            hasAccessTo(app, user, iteratorDone);
        }, callback);
    });
}

function downloadManifest(appStoreId, manifest, callback) {
    if (!appStoreId && !manifest) return callback(new BoxError(BoxError.BAD_FIELD, 'Neither manifest nor appStoreId provided'));

    if (!appStoreId) return callback(null, '', manifest);

    var parts = appStoreId.split('@');

    var url = settings.apiServerOrigin() + '/api/v1/apps/' + parts[0] + (parts[1] ? '/versions/' + parts[1] : '');

    debug('downloading manifest from %s', url);

    superagent.get(url).timeout(30 * 1000).end(function (error, result) {
        if (error && !error.response) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Network error downloading manifest:' + error.message));

        if (result.statusCode !== 200) return callback(new BoxError(BoxError.NOT_FOUND, util.format('Failed to get app info from store.', result.statusCode, result.text)));

        if (!result.body.manifest || typeof result.body.manifest !== 'object') return callback(new BoxError(BoxError.NOT_FOUND, util.format('Missing manifest. Failed to get app info from store.', result.statusCode, result.text)));

        callback(null, parts[0], result.body.manifest);
    });
}

function mailboxNameForLocation(location, manifest) {
    if (location) return `${location}.app`;
    if (manifest.title) return manifest.title.toLowerCase().replace(/[^a-zA-Z0-9]/g, '') + '.app';
    return 'noreply.app';
}

function scheduleTask(appId, installationState, taskId, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof installationState, 'string');
    assert.strictEqual(typeof taskId, 'string');
    assert.strictEqual(typeof callback, 'function');

    settings.getBackupConfig(function (error, backupConfig) {
        if (error) return callback(error);

        let memoryLimit = 400;
        if (installationState === exports.ISTATE_PENDING_BACKUP || installationState === exports.ISTATE_PENDING_CLONE || installationState === exports.ISTATE_PENDING_RESTORE
            || installationState === exports.ISTATE_PENDING_IMPORT || installationState === exports.ISTATE_PENDING_UPDATE) {
            memoryLimit = 'memoryLimit' in backupConfig ? Math.max(backupConfig.memoryLimit/1024/1024, 400) : 400;
        }

        const options = { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit };

        appTaskManager.scheduleTask(appId, taskId, options, function (error) {
            debug(`scheduleTask: task ${taskId} of ${appId} completed`);
            if (error && (error.code === tasks.ECRASHED || error.code === tasks.ESTOPPED)) { // if task crashed, update the error
                debug(`Apptask crashed/stopped: ${error.message}`);
                let boxError = new BoxError(BoxError.TASK_ERROR, error.message);
                boxError.details.crashed = error.code === tasks.ECRASHED;
                boxError.details.stopped = error.code === tasks.ESTOPPED;
                // see also apptask makeTaskError
                boxError.details.taskId = taskId;
                boxError.details.installationState = installationState;
                appdb.update(appId, { installationState: exports.ISTATE_ERROR, error: boxError.toPlainObject(), taskId: null }, callback.bind(null, error));
            } else if (!(installationState === exports.ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall
                appdb.update(appId, { taskId: null }, callback.bind(null, error));
            } else {
                callback(error);
            }
        });
    });
}

function addTask(appId, installationState, task, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof installationState, 'string');
    assert.strictEqual(typeof task, 'object'); // { args, values }
    assert.strictEqual(typeof callback, 'function');

    const { args, values } = task;
    // TODO: match the SQL logic to match checkAppState. this means checking the error.installationState and installationState. Unfortunately, former is JSON right now
    const requiredState = null; // 'requiredState' in task ? task.requiredState : exports.ISTATE_INSTALLED;
    const scheduleNow = 'scheduleNow' in task ? task.scheduleNow : true;
    const requireNullTaskId = 'requireNullTaskId' in task ? task.requireNullTaskId : true;

    const tasksAdd = util.callbackify(tasks.add);

    tasksAdd(tasks.TASK_APP, [ appId, args ], function (error, taskId) {
        if (error) return callback(error);

        appdb.setTask(appId, _.extend({ installationState, taskId, error: null }, values), { requiredState, requireNullTaskId }, function (error) {
            if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_STATE, 'Another task is scheduled for this app')); // could be because app went away OR a taskId exists
            if (error) return callback(error);

            if (scheduleNow) scheduleTask(appId, installationState, taskId, task.onFinished || NOOP_CALLBACK);

            callback(null, { taskId });
        });
    });
}

function checkAppState(app, state) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof state, 'string');

    if (app.taskId) return new BoxError(BoxError.BAD_STATE, `Locked by task ${app.taskId} : ${app.installationState} / ${app.runState}`);

    if (app.installationState === exports.ISTATE_ERROR) {
        // allow task to be called again if that was the errored task
        if (app.error.installationState === state) return null;

        // allow uninstall from any state
        if (state !== exports.ISTATE_PENDING_UNINSTALL && state !== exports.ISTATE_PENDING_RESTORE && state !== exports.ISTATE_PENDING_IMPORT) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
    }

    if (app.runState === exports.RSTATE_STOPPED) {
        // can't backup or restore since app addons are down. can't update because migration scripts won't run
        if (state === exports.ISTATE_PENDING_UPDATE || state === exports.ISTATE_PENDING_BACKUP || state === exports.ISTATE_PENDING_RESTORE || state === exports.ISTATE_PENDING_IMPORT)  return new BoxError(BoxError.BAD_STATE, 'Not allowed in stopped state');
    }

    return null;
}

function validateLocations(locations, callback) {
    assert(Array.isArray(locations));
    assert.strictEqual(typeof callback, 'function');

    getDomainObjectMap(function (error, domainObjectMap) {
        if (error) return callback(error);

        for (let location of locations) {
            if (!(location.domain in domainObjectMap)) return callback(new BoxError(BoxError.BAD_FIELD, 'No such domain', { field: 'location', domain: location.domain, subdomain: location.subdomain }));

            let subdomain = location.subdomain;
            if (location.type === 'alias' && subdomain.startsWith('*')) {
                if (subdomain === '*') continue;
                subdomain = subdomain.replace(/^\*\./, ''); // remove *.
            }

            error = domains.validateHostname(subdomain, domainObjectMap[location.domain]);
            if (error) return callback(new BoxError(BoxError.BAD_FIELD, 'Bad location: ' + error.message, { field: 'location', domain: location.domain, subdomain: location.subdomain }));
        }

        callback(null, domainObjectMap);
    });
}

function hasMailAddon(manifest) {
    return manifest.addons.sendmail || manifest.addons.recvmail;
}

function install(data, auditSource, callback) {
    assert(data && typeof data === 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    assert.strictEqual(typeof data.manifest, 'object'); // manifest is already downloaded

    let location = data.location.toLowerCase(),
        domain = data.domain.toLowerCase(),
        portBindings = data.portBindings || null,
        accessRestriction = data.accessRestriction || null,
        icon = data.icon || null,
        memoryLimit = data.memoryLimit || 0,
        sso = 'sso' in data ? data.sso : null,
        debugMode = data.debugMode || null,
        enableBackup = 'enableBackup' in data ? data.enableBackup : true,
        enableAutomaticUpdate = 'enableAutomaticUpdate' in data ? data.enableAutomaticUpdate : true,
        alternateDomains = data.alternateDomains || [],
        aliasDomains = data.aliasDomains || [],
        env = data.env || {},
        label = data.label || null,
        tags = data.tags || [],
        overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false,
        skipDnsSetup = 'skipDnsSetup' in data ? data.skipDnsSetup : false,
        appStoreId = data.appStoreId,
        enableMailbox = 'enabledMailbox' in data ? data.enableMailbox : true,
        manifest = data.manifest;

    let error = manifestFormat.parse(manifest);
    if (error) return callback(new BoxError(BoxError.BAD_FIELD, 'Manifest error: ' + error.message));

    error = checkManifestConstraints(manifest);
    if (error) return callback(error);

    error = validatePortBindings(portBindings, manifest);
    if (error) return callback(error);

    error = validateAccessRestriction(accessRestriction);
    if (error) return callback(error);

    error = validateMemoryLimit(manifest, memoryLimit);
    if (error) return callback(error);

    error = validateDebugMode(debugMode);
    if (error) return callback(error);

    error = validateLabel(label);
    if (error) return callback(error);

    error = validateTags(tags);
    if (error) return callback(error);

    if ('sso' in data && !('optionalSso' in manifest)) return callback(new BoxError(BoxError.BAD_FIELD, 'sso can only be specified for apps with optionalSso'));
    // if sso was unspecified, enable it by default if possible
    if (sso === null) sso = !!manifest.addons['ldap'] || !!manifest.addons['proxyAuth'];

    error = validateEnv(env);
    if (error) return callback(error);

    if (settings.isDemo() && constants.DEMO_BLACKLISTED_APPS.includes(appStoreId)) return callback(new BoxError(BoxError.BAD_FIELD, 'This app is blacklisted in the demo'));

    const mailboxName = hasMailAddon(manifest) ? mailboxNameForLocation(location, manifest) : null;
    const mailboxDomain = hasMailAddon(manifest) ? domain : null;
    const appId = uuid.v4();

    if (icon) {
        if (!validator.isBase64(icon)) return callback(new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' }));
        icon = Buffer.from(icon, 'base64');
    }

    const locations = [{ subdomain: location, domain, type: 'primary' }]
        .concat(alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
        .concat(aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));

    validateLocations(locations, function (error, domainObjectMap) {
        if (error) return callback(error);

        debug('Will install app with id : ' + appId);

        const data = {
            accessRestriction,
            memoryLimit,
            sso,
            debugMode,
            mailboxName,
            mailboxDomain,
            enableBackup,
            enableAutomaticUpdate,
            alternateDomains,
            aliasDomains,
            env,
            label,
            tags,
            icon,
            enableMailbox,
            runState: exports.RSTATE_RUNNING,
            installationState: exports.ISTATE_PENDING_INSTALL
        };

        appdb.add(appId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) {
            if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, locations, domainObjectMap, portBindings));
            if (error) return callback(error);

            purchaseApp({ appId: appId, appstoreId: appStoreId, manifestId: manifest.id || 'customapp' }, function (error) {
                if (error) return callback(error);

                const task = {
                    args: { restoreConfig: null, skipDnsSetup, overwriteDns },
                    values: { },
                    requiredState: data.installationState
                };

                addTask(appId, data.installationState, task, function (error, result) {
                    if (error) return callback(error);

                    const newApp = _.extend({}, _.omit(data, 'icon'), { appStoreId, manifest, location, domain, portBindings });
                    newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
                    newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
                    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });

                    eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId, app: newApp, taskId: result.taskId });

                    callback(null, { id : appId, taskId: result.taskId });
                });
            });
        });
    });
}

function setAccessRestriction(app, accessRestriction, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof accessRestriction, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = validateAccessRestriction(accessRestriction);
    if (error) return callback(error);

    appdb.update(appId, { accessRestriction }, function (error) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, accessRestriction });

        callback();
    });
}

function setLabel(app, label, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof label, 'string');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = validateLabel(label);
    if (error) return callback(error);

    appdb.update(appId, { label }, function (error) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, label });

        callback();
    });
}

function setTags(app, tags, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert(Array.isArray(tags));
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = validateTags(tags);
    if (error) return callback(error);

    appdb.update(appId, { tags }, function (error) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, tags });

        callback();
    });
}

function setIcon(app, icon, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert(icon === null || typeof icon === 'string');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;

    if (icon) {
        if (!validator.isBase64(icon)) return callback(new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' }));
        icon = Buffer.from(icon, 'base64');
    }

    appdb.update(appId, { icon }, function (error) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, iconChanged: true });
        callback();
    });
}

function setMemoryLimit(app, memoryLimit, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof memoryLimit, 'number');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RESIZE);
    if (error) return callback(error);

    error = validateMemoryLimit(app.manifest, memoryLimit);
    if (error) return callback(error);

    const task = {
        args: {},
        values: { memoryLimit }
    };
    addTask(appId, exports.ISTATE_PENDING_RESIZE, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, memoryLimit, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function setCpuShares(app, cpuShares, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof cpuShares, 'number');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RESIZE);
    if (error) return callback(error);

    error = validateCpuShares(cpuShares);
    if (error) return callback(error);

    const task = {
        args: {},
        values: { cpuShares }
    };
    addTask(appId, exports.ISTATE_PENDING_RESIZE, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, cpuShares, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function setMounts(app, mounts, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert(Array.isArray(mounts));
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER);
    if (error) return callback(error);

    const task = {
        args: {},
        values: { mounts }
    };
    addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task, function (error, result) {
        if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(new BoxError(BoxError.CONFLICT, 'Duplicate mount points'));
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mounts, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function setEnvironment(app, env, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof env, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER);
    if (error) return callback(error);

    error = validateEnv(env);
    if (error) return callback(error);

    const task = {
        args: {},
        values: { env }
    };
    addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, env, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function setDebugMode(app, debugMode, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof debugMode, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_DEBUG);
    if (error) return callback(error);

    error = validateDebugMode(debugMode);
    if (error) return callback(error);

    const task = {
        args: {},
        values: { debugMode }
    };
    addTask(appId, exports.ISTATE_PENDING_DEBUG, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, debugMode, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function setMailbox(app, data, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const { enable, mailboxDomain } = data;
    let mailboxName = data.mailboxName;
    assert.strictEqual(typeof enable, 'boolean');
    assert(mailboxName === null || typeof mailboxName === 'string');
    assert.strictEqual(typeof mailboxDomain, 'string');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER);
    if (error) return callback(error);

    if (!hasMailAddon(app.manifest)) return callback(new BoxError(BoxError.BAD_FIELD, 'App does not use mail addons'));

    const getDomainFunc = util.callbackify(mail.getDomain);

    getDomainFunc(mailboxDomain, function (error) {
        if (error) return callback(error);

        if (mailboxName) {
            error = mail.validateName(mailboxName);
            if (error) return callback(new BoxError(BoxError.BAD_FIELD, error.message, { field: 'mailboxName' }));
        } else {
            mailboxName = mailboxNameForLocation(app.location, app.domain, app.manifest);
        }

        const task = {
            args: {},
            values: { enableMailbox: enable, mailboxName, mailboxDomain }
        };
        addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task, function (error, result) {
            if (error) return callback(error);

            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mailboxName, taskId: result.taskId });

            callback(null, { taskId: result.taskId });
        });
    });
}

function setAutomaticBackup(app, enable, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof enable, 'boolean');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    appdb.update(appId, { enableBackup: enable }, function (error) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, enableBackup: enable });

        callback();
    });
}

function setAutomaticUpdate(app, enable, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof enable, 'boolean');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    appdb.update(appId, { enableAutomaticUpdate: enable }, function (error) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, enableAutomaticUpdate: enable });

        callback();
    });
}

function setReverseProxyConfig(app, reverseProxyConfig, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof reverseProxyConfig, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    reverseProxyConfig = _.extend({ robotsTxt: null, csp: null }, reverseProxyConfig);

    const appId = app.id;
    let error = validateCsp(reverseProxyConfig.csp);
    if (error) return callback(error);

    error = validateRobotsTxt(reverseProxyConfig.robotsTxt);
    if (error) return callback(error);

    reverseProxy.writeAppConfig(_.extend({}, app, { reverseProxyConfig }), function (error) {
        if (error) return callback(error);

        appdb.update(appId, { reverseProxyConfig }, function (error) {
            if (error) return callback(error);

            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, reverseProxyConfig });

            callback();
        });
    });
}

function setCertificate(app, data, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert(data && typeof data === 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    const { location, domain, cert, key } = data;

    domains.get(domain, function (error, domainObject) {
        if (error) return callback(error);

        if (cert && key) {
            error = reverseProxy.validateCertificate(location, domainObject, { cert, key });
            if (error) return callback(error);
        }

        reverseProxy.setAppCertificateSync(location, domainObject, { cert, key }, function (error) {
            if (error) return callback(error);

            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, cert, key });

            callback();
        });
    });
}

function setLocation(app, data, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_LOCATION_CHANGE);
    if (error) return callback(error);

    let values = {
        location: data.location.toLowerCase(),
        domain: data.domain.toLowerCase(),
        // these are intentionally reset, if not set
        portBindings: null,
        alternateDomains: [],
        aliasDomains: []
    };

    if ('portBindings' in data) {
        error = validatePortBindings(data.portBindings, app.manifest);
        if (error) return callback(error);

        values.portBindings = translatePortBindings(data.portBindings || null, app.manifest);
    }

    // move the mailbox name to match the new location
    if (hasMailAddon(app.manifest) && app.mailboxName.endsWith('.app')) {
        values.mailboxName = mailboxNameForLocation(values.location, app.manifest);
        values.mailboxDomain = values.domain;
    }

    if ('alternateDomains' in data) {
        values.alternateDomains = data.alternateDomains;
    }

    if ('aliasDomains' in data) {
        values.aliasDomains = data.aliasDomains;
    }

    const locations = [{ subdomain: values.location, domain: values.domain, type: 'primary' }]
        .concat(values.alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
        .concat(values.aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));

    validateLocations(locations, function (error, domainObjectMap) {
        if (error) return callback(error);

        const task = {
            args: {
                oldConfig: _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'aliasDomains', 'portBindings'),
                skipDnsSetup: !!data.skipDnsSetup,
                overwriteDns: !!data.overwriteDns
            },
            values
        };
        addTask(appId, exports.ISTATE_PENDING_LOCATION_CHANGE, task, function (error, result) {
            if (error && error.reason === BoxError.ALREADY_EXISTS) error = getDuplicateErrorDetails(error.message, locations, domainObjectMap, data.portBindings);
            if (error) return callback(error);

            values.fqdn = domains.fqdn(values.location, domainObjectMap[values.domain]);
            values.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
            values.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });

            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, _.extend({ appId, app, taskId: result.taskId }, values));

            callback(null, { taskId: result.taskId });
        });
    });
}

function setDataDir(app, dataDir, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert(dataDir === null || typeof dataDir === 'string');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_DATA_DIR_MIGRATION);
    if (error) return callback(error);

    error = validateDataDir(dataDir);
    if (error) return callback(error);

    const task = {
        args: { newDataDir: dataDir },
        values: { },
        onFinished: (error) => {
            if (!error) services.rebuildService('sftp', NOOP_CALLBACK);
        }
    };
    addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, dataDir, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function update(app, data, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert(data && typeof data === 'object');
    assert(data.manifest && typeof data.manifest === 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const skipBackup = !!data.skipBackup,
        appId = app.id,
        manifest = data.manifest,
        appStoreId = data.appStoreId;

    let values = {};

    if (app.runState === exports.RSTATE_STOPPED) return callback(new BoxError(BoxError.BAD_STATE, 'Stopped apps cannot be updated'));

    let error = checkAppState(app, exports.ISTATE_PENDING_UPDATE);
    if (error) return callback(error);

    error = manifestFormat.parse(manifest);
    if (error) return callback(new BoxError(BoxError.BAD_FIELD, 'Manifest error:' + error.message));

    error = checkManifestConstraints(manifest);
    if (error) return callback(error);

    var updateConfig = { skipBackup, manifest, appStoreId }; // this will clear appStoreId when updating from a repo and set it if passed in for update route

    // prevent user from installing a app with different manifest id over an existing app
    // this allows cloudron install -f --app <appid> for an app installed from the appStore
    if (app.manifest.id !== updateConfig.manifest.id) {
        if (!data.force) return callback(new BoxError(BoxError.BAD_FIELD, 'manifest id does not match. force to override'));
    }

    // suffix '0' if prerelease is missing for semver.lte to work as expected
    const currentVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
    const updateVersion = semver.prerelease(updateConfig.manifest.version) ? updateConfig.manifest.version : `${updateConfig.manifest.version}-0`;
    if (app.appStoreId !== '' && semver.lte(updateVersion, currentVersion)) {
        if (!data.force) return callback(new BoxError(BoxError.BAD_FIELD, 'Downgrades are not permitted for apps installed from AppStore. force to override'));
    }

    if ('icon' in data) {
        if (data.icon) {
            if (!validator.isBase64(data.icon)) return callback(new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' }));
            data.icon = Buffer.from(data.icon, 'base64');
        }
        values.icon = data.icon;
    }

    // do not update apps in debug mode
    if (app.debugMode && !data.force) return callback(new BoxError(BoxError.BAD_STATE, 'debug mode enabled. force to override'));

    // Ensure we update the memory limit in case the new app requires more memory as a minimum
    // 0 and -1 are special updateConfig for memory limit indicating unset and unlimited
    if (app.memoryLimit > 0 && updateConfig.manifest.memoryLimit && app.memoryLimit < updateConfig.manifest.memoryLimit) {
        updateConfig.memoryLimit = updateConfig.manifest.memoryLimit;
    }

    if (!hasMailAddon(manifest)) { // clear if the update removed addon
        values.mailboxName = values.mailboxDomain = null;
    } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since restore added the addon
        values.mailboxName = mailboxNameForLocation(app.location, manifest);
        values.mailboxDomain = app.domain;
    }

    const task = {
        args: { updateConfig },
        values,
        onFinished: (error) => {
            eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditSource, { app, success: !error, errorMessage: error ? error.message : null });
        }
    };
    addTask(appId, exports.ISTATE_PENDING_UPDATE, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId, app, skipBackup, toManifest: manifest, fromManifest: app.manifest, force: data.force, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function getLocalLogfilePaths(app) {
    assert.strictEqual(typeof app, 'object');

    const appId = app.id;

    var filePaths = [];
    filePaths.push(path.join(paths.LOG_DIR, appId, 'apptask.log'));
    filePaths.push(path.join(paths.LOG_DIR, appId, 'app.log'));
    if (app.manifest.addons && app.manifest.addons.redis) filePaths.push(path.join(paths.LOG_DIR, `redis-${appId}/app.log`));

    return filePaths;
}

function getLogs(app, options, callback) {
    assert.strictEqual(typeof app, 'object');
    assert(options && typeof options === 'object');
    assert.strictEqual(typeof callback, 'function');

    assert.strictEqual(typeof options.lines, 'number');
    assert.strictEqual(typeof options.format, 'string');
    assert.strictEqual(typeof options.follow, 'boolean');

    const appId = app.id;

    var lines = options.lines === -1 ? '+1' : options.lines,
        format = options.format || 'json',
        follow = options.follow;

    assert.strictEqual(typeof format, 'string');

    var args = [ '--lines=' + lines ];
    if (follow) args.push('--follow', '--retry', '--quiet'); // same as -F. to make it work if file doesn't exist, --quiet to not output file headers, which are no logs

    var cp = spawn('/usr/bin/tail', args.concat(getLocalLogfilePaths(app)));

    var transformStream = split(function mapper(line) {
        if (format !== 'json') return line + '\n';

        var data = line.split(' '); // logs are <ISOtimestamp> <msg>
        var timestamp = (new Date(data[0])).getTime();
        if (isNaN(timestamp)) timestamp = 0;
        var message = line.slice(data[0].length+1);

        // ignore faulty empty logs
        if (!timestamp && !message) return;

        return JSON.stringify({
            realtimeTimestamp: timestamp * 1000,
            message: message,
            source: appId
        }) + '\n';
    });

    transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process

    cp.stdout.pipe(transformStream);

    return callback(null, transformStream);
}

async function getCertificate(subdomain, domain) {
    assert.strictEqual(typeof subdomain, 'string');
    assert.strictEqual(typeof domain, 'string');

    const result = await database.query('SELECT certificateJson FROM subdomains WHERE subdomain=? AND domain=?', [ subdomain, domain ]);
    if (result.length === 0) return null;
    return JSON.parse(result[0].certificateJson);
}

// does a re-configure when called from most states. for install/clone errors, it re-installs with an optional manifest
// re-configure can take a dockerImage but not a manifest because re-configure does not clean up addons
function repair(app, data, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object'); // { manifest }
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let errorState = (app.error && app.error.installationState) || exports.ISTATE_PENDING_CONFIGURE;

    const task = {
        args: {},
        values: {},
        requiredState: null
    };

    // maybe split this into a separate route like reinstall?
    if (errorState === exports.ISTATE_PENDING_INSTALL || errorState === exports.ISTATE_PENDING_CLONE) {
        task.args = { skipDnsSetup: false, overwriteDns: true };
        if (data.manifest) {
            let error = manifestFormat.parse(data.manifest);
            if (error) return callback(new BoxError(BoxError.BAD_FIELD, `manifest error: ${error.message}`));

            error = checkManifestConstraints(data.manifest);
            if (error) return callback(error);

            if (!hasMailAddon(data.manifest)) { // clear if repair removed addon
                task.values.mailboxName = task.values.mailboxDomain = null;
            } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since repair added the addon
                task.values.mailboxName = mailboxNameForLocation(app.location, data.manifest);
                task.values.mailboxDomain = app.domain;
            }

            task.values.manifest = data.manifest;
            task.args.oldManifest = app.manifest;
        }
    } else {
        errorState = exports.ISTATE_PENDING_CONFIGURE;
        if (data.dockerImage) {
            let newManifest = _.extend({}, app.manifest, { dockerImage: data.dockerImage });
            task.values.manifest = newManifest;
        }
    }

    addTask(appId, errorState, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { app, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

async function restore(app, backupId, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof backupId, 'string');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;

    let error = checkAppState(app, exports.ISTATE_PENDING_RESTORE);
    if (error) throw error;

    // for empty or null backupId, use existing manifest to mimic a reinstall
    const backupInfo = backupId ? await backups.get(backupId) : { manifest: app.manifest };

    if (!backupInfo.manifest) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Could not get restore manifest');
    if (backupInfo.encryptionVersion === 1) throw new BoxError(BoxError.BAD_FIELD, 'This encrypted backup was created with an older Cloudron version and has to be restored using the CLI tool');

    // re-validate because this new box version may not accept old configs
    error = checkManifestConstraints(backupInfo.manifest);
    if (error) throw error;

    let values = { manifest: backupInfo.manifest };
    if (!hasMailAddon(backupInfo.manifest)) { // clear if restore removed addon
        values.mailboxName = values.mailboxDomain = null;
    } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since restore added the addon
        values.mailboxName = mailboxNameForLocation(app.location, backupInfo.manifest);
        values.mailboxDomain = app.domain;
    }

    const restoreConfig = { backupId, backupFormat: backupInfo.format };

    const task = {
        args: {
            restoreConfig,
            oldManifest: app.manifest,
            skipDnsSetup: !!backupId, // if this is a restore, just skip dns setup. only re-installs should setup dns
            overwriteDns: true
        },
        values
    };

    return new Promise((resolve, reject) => {
        addTask(appId, exports.ISTATE_PENDING_RESTORE, task, function (error, result) {
            if (error) return reject(error);

            eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId: backupInfo.id, fromManifest: app.manifest, toManifest: backupInfo.manifest, taskId: result.taskId });

            resolve({ taskId: result.taskId });
        });
    });
}

function importApp(app, data, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;

    // all fields are optional
    data.backupId = data.backupId || null;
    data.backupFormat = data.backupFormat || null;
    data.backupConfig = data.backupConfig || null;
    const { backupId, backupFormat, backupConfig } = data;

    let error = backupFormat ? validateBackupFormat(backupFormat) : null;
    if (error) return callback(error);

    error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
    if (error) return callback(error);

    // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
    const testBackupConfig = backupConfig ? backups.testProviderConfig.bind(null, backupConfig) : (next) => next();

    testBackupConfig(function (error) {
        if (error) return callback(error);

        if (backupConfig) {
            if ('password' in backupConfig) {
                backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.password);
                delete backupConfig.password;
            } else {
                backupConfig.encryption = null;
            }
        }

        const restoreConfig = { backupId, backupFormat, backupConfig };

        const task = {
            args: {
                restoreConfig,
                oldManifest: app.manifest,
                skipDnsSetup: false,
                overwriteDns: true
            },
            values: {}
        };
        addTask(appId, exports.ISTATE_PENDING_IMPORT, task, function (error, result) {
            if (error) return callback(error);

            eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app: app, backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId: result.taskId });

            callback(null, { taskId: result.taskId });
        });
    });
}

function exportApp(app, data, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;

    let error = checkAppState(app, exports.ISTATE_PENDING_BACKUP);
    if (error) return callback(error);

    const task = {
        args: { snapshotOnly: true },
        values: {}
    };
    addTask(appId, exports.ISTATE_PENDING_BACKUP, task, (error, result) => {
        if (error) return callback(error);

        callback(null, { taskId: result.taskId });
    });
}

function purchaseApp(data, callback) {
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof callback, 'function');

    appstore.purchaseApp(data, function (error) {
        if (!error) return callback();

        // if purchase failed, rollback the appdb record
        appdb.del(data.appId, function (delError) {
            if (delError) debug('install: Failed to rollback app installation.', delError);

            callback(error);
        });
    });
}

function clone(app, data, user, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert(user && typeof user === 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const location = data.location.toLowerCase(),
        domain = data.domain.toLowerCase(),
        portBindings = data.portBindings || null,
        backupId = data.backupId,
        overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false,
        skipDnsSetup = 'skipDnsSetup' in data ? data.skipDnsSetup : false,
        appId = app.id;

    assert.strictEqual(typeof backupId, 'string');
    assert.strictEqual(typeof location, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof portBindings, 'object');

    const locations = [{ subdomain: location, domain, type: 'primary' }];
    validateLocations(locations, async function (error, domainObjectMap) {
        if (error) return callback(error);

        const [backupsError, backupInfo] = await safe(backups.get(backupId));
        if (backupsError) return callback(backupsError);

        if (!backupInfo.manifest) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Could not get restore config'));
        if (backupInfo.encryptionVersion === 1) return callback(new BoxError(BoxError.BAD_FIELD, 'This encrypted backup was created with an older Cloudron version and cannot be cloned'));

        const manifest = backupInfo.manifest, appStoreId = app.appStoreId;

        // re-validate because this new box version may not accept old configs
        error = checkManifestConstraints(manifest);
        if (error) return callback(error);

        error = validatePortBindings(portBindings, manifest);
        if (error) return callback(error);

        // should we copy the original app's mailbox settings instead?
        let mailboxName = hasMailAddon(manifest) ? mailboxNameForLocation(location, manifest) : null;
        let mailboxDomain = hasMailAddon(manifest) ? domain : null;

        const newAppId = uuid.v4();

        appdb.getIcons(app.id, function (error, icons) {
            if (error) return callback(error);

            const data = {
                installationState: exports.ISTATE_PENDING_CLONE,
                runState: exports.RSTATE_RUNNING,
                memoryLimit: app.memoryLimit,
                cpuShares: app.cpuShares,
                accessRestriction: app.accessRestriction,
                sso: !!app.sso,
                mailboxName: mailboxName,
                mailboxDomain: mailboxDomain,
                enableBackup: app.enableBackup,
                reverseProxyConfig: app.reverseProxyConfig,
                env: app.env,
                alternateDomains: [],
                aliasDomains: [],
                servicesConfig: app.servicesConfig,
                label: app.label ? `${app.label}-clone` : '',
                tags: app.tags,
                enableAutomaticUpdate: app.enableAutomaticUpdate,
                icon: icons.icon,
                enableMailbox: app.enableMailbox
            };

            appdb.add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) {
                if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, locations, domainObjectMap, portBindings));
                if (error) return callback(error);

                purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' }, function (error) {
                    if (error) return callback(error);

                    const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format };
                    const task = {
                        args: { restoreConfig, overwriteDns, skipDnsSetup, oldManifest: null },
                        values: {},
                        requiredState: exports.ISTATE_PENDING_CLONE
                    };
                    addTask(newAppId, exports.ISTATE_PENDING_CLONE, task, function (error, result) {
                        if (error) return callback(error);

                        const newApp = _.extend({}, _.omit(data, 'icon'), { appStoreId, manifest, location, domain, portBindings });
                        newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
                        newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
                        newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });

                        eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, oldApp: app, newApp: newApp, taskId: result.taskId });

                        callback(null, { id: newAppId, taskId: result.taskId });
                    });
                });
            });
        });
    });
}

function uninstall(app, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_UNINSTALL);
    if (error) return callback(error);

    appstore.unpurchaseApp(appId, { appstoreId: app.appStoreId, manifestId: app.manifest.id || 'customapp' }, function (error) {
        if (error) return callback(error);

        const task = {
            args: {},
            values: {},
            requiredState: null // can run in any state, as long as no task is active
        };
        addTask(appId, exports.ISTATE_PENDING_UNINSTALL, task, function (error, result) {
            if (error) return callback(error);

            eventlog.add(eventlog.ACTION_APP_UNINSTALL, auditSource, { appId, app, taskId: result.taskId });

            callback(null, { taskId: result.taskId });
        });
    });
}

function start(app, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_START);
    if (error) return callback(error);

    const task = {
        args: {},
        values: { runState: exports.RSTATE_RUNNING }
    };
    addTask(appId, exports.ISTATE_PENDING_START, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_START, auditSource, { appId, app, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function stop(app, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_STOP);
    if (error) return callback(error);

    const task = {
        args: {},
        values: { runState: exports.RSTATE_STOPPED }
    };
    addTask(appId, exports.ISTATE_PENDING_STOP, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_STOP, auditSource, { appId, app, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function restart(app, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RESTART);
    if (error) return callback(error);

    const task = {
        args: {},
        values: { runState: exports.RSTATE_RUNNING }
    };
    addTask(appId, exports.ISTATE_PENDING_RESTART, task, function (error, result) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_APP_RESTART, auditSource, { appId, app, taskId: result.taskId });

        callback(null, { taskId: result.taskId });
    });
}

function checkManifestConstraints(manifest) {
    assert(manifest && typeof manifest === 'object');

    if (manifest.manifestVersion > 2) return new BoxError(BoxError.BAD_FIELD, 'Manifest version must be <= 2');

    if (!manifest.dockerImage) return new BoxError(BoxError.BAD_FIELD, 'Missing dockerImage'); // dockerImage is optional in manifest

    if (semver.valid(manifest.maxBoxVersion) && semver.gt(constants.VERSION, manifest.maxBoxVersion)) {
        return new BoxError(BoxError.BAD_FIELD, 'Box version exceeds Apps maxBoxVersion');
    }

    if (semver.valid(manifest.minBoxVersion) && semver.gt(manifest.minBoxVersion, constants.VERSION)) {
        return new BoxError(BoxError.BAD_FIELD, 'App version requires a new platform version');
    }

    return null;
}

function exec(app, options, callback) {
    assert.strictEqual(typeof app, 'object');
    assert(options && typeof options === 'object');
    assert.strictEqual(typeof callback, 'function');

    let cmd = options.cmd || [ '/bin/bash' ];
    assert(Array.isArray(cmd) && cmd.length > 0);

    if (app.installationState !== exports.ISTATE_INSTALLED || app.runState !== exports.RSTATE_RUNNING) {
        return callback(new BoxError(BoxError.BAD_STATE, 'App not installed or running'));
    }

    var execOptions = {
        AttachStdin: true,
        AttachStdout: true,
        AttachStderr: true,
        // A pseudo tty is a terminal which processes can detect (for example, disable colored output)
        // Creating a pseudo terminal also assigns a terminal driver which detects control sequences
        // When passing binary data, tty must be disabled. In addition, the stdout/stderr becomes a single
        // unified stream because of the nature of a tty (see https://github.com/docker/docker/issues/19696)
        Tty: options.tty,
        Cmd: cmd
    };

    var startOptions = {
        Detach: false,
        Tty: options.tty,
        // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
        // we can work with half-close connections (not defined in http). this way, the client
        // can properly signal that stdin is EOF by closing it's side of the socket. In http,
        // the whole connection will be dropped when stdin get EOF.
        // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
        hijack: true,
        stream: true,
        stdin: true,
        stdout: true,
        stderr: true
    };

    docker.execContainer(app.containerId, { execOptions, startOptions, rows: options.rows, columns: options.columns }, function (error, stream) {
        if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running
        if (error) return callback(error);

        callback(null, stream);
    });
}

function canAutoupdateApp(app, updateInfo) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof updateInfo, 'object');

    const manifest = updateInfo.manifest;

    if (!app.enableAutomaticUpdate) return false;

    // for invalid subscriptions the appstore does not return a dockerImage
    if (!manifest.dockerImage) return false;

    if (updateInfo.unstable) return false; // only manual update allowed for unstable updates

    if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(manifest.version))) return false; // major changes are blocking

    if (app.runState === exports.RSTATE_STOPPED) return false; // stopped apps won't run migration scripts and shouldn't be updated

    const newTcpPorts = manifest.tcpPorts || { };
    const newUdpPorts = manifest.udpPorts || { };
    const portBindings = app.portBindings; // this is never null

    for (let portName in portBindings) {
        if (!(portName in newTcpPorts) && !(portName in newUdpPorts)) return false; // portName was in use but new update removes it
    }

    // it's fine if one or more (unused) keys got removed
    return true;
}

function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is { appId -> { manifest } }
    assert.strictEqual(typeof updateInfo, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    async.eachSeries(Object.keys(updateInfo), function iterator(appId, iteratorDone) {
        get(appId, function (error, app) {
            if (error) {
                debug('Cannot autoupdate app %s : %s', appId, error.message);
                return iteratorDone();
            }

            if (!canAutoupdateApp(app, updateInfo[appId])) {
                debug(`app ${app.fqdn} requires manual update`);
                return iteratorDone();
            }

            var data = {
                manifest: updateInfo[appId].manifest,
                force: false
            };

            update(app, data, auditSource, function (error) {
                if (error) debug('Error initiating autoupdate of %s. %s', appId, error.message);

                iteratorDone(null);
            });
        });
    }, callback);
}

function backup(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appId = app.id;

    let error = checkAppState(app, exports.ISTATE_PENDING_BACKUP);
    if (error) return callback(error);

    const task = {
        args: {},
        values: {}
    };
    addTask(appId, exports.ISTATE_PENDING_BACKUP, task, (error, result) => {
        if (error) return callback(error);

        callback(null, { taskId: result.taskId });
    });
}

async function listBackups(app, page, perPage) {
    assert.strictEqual(typeof app, 'object');
    assert(typeof page === 'number' && page > 0);
    assert(typeof perPage === 'number' && perPage > 0);

    return await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, page, perPage);
}

function restoreInstalledApps(options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

    const addTaskAsync = util.promisify(addTask);

    getAll(async function (error, apps) {
        if (error) return callback(error);

        apps = apps.filter(app => app.installationState !== exports.ISTATE_ERROR); // remove errored apps. let them be 'repaired' by hand
        apps = apps.filter(app => app.installationState !== exports.ISTATE_PENDING_RESTORE); // safeguard against tasks being created non-stop if we crash on startup

        for (const app of apps) {
            const [error, results] = await safe(backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1));
            let installationState, restoreConfig, oldManifest;
            if (!error && results.length) {
                installationState = exports.ISTATE_PENDING_RESTORE;
                restoreConfig = { backupId: results[0].id, backupFormat: results[0].format };
                oldManifest = app.manifest;
            } else {
                installationState = exports.ISTATE_PENDING_INSTALL;
                restoreConfig = null;
                oldManifest = null;
            }

            const task = {
                args: { restoreConfig, skipDnsSetup: options.skipDnsSetup, overwriteDns: true, oldManifest },
                values: {},
                scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                requireNullTaskId: false // ignore existing stale taskId
            };

            debug(`restoreInstalledApps: marking ${app.fqdn} for restore using restore config ${JSON.stringify(restoreConfig)}`);

            const [addTaskError, result] = await safe(addTaskAsync(app.id, installationState, task));
            if (addTaskError) debug(`restoreInstalledApps: error marking ${app.fqdn} for restore: ${JSON.stringify(addTaskError)}`);
            else debug(`restoreInstalledApps: marked ${app.id} for restore with taskId ${result.taskId}`);
        }

        callback(null);
    });
}

function configureInstalledApps(callback) {
    assert.strictEqual(typeof callback, 'function');

    getAll(function (error, apps) {
        if (error) return callback(error);

        apps = apps.filter(app => app.installationState !== exports.ISTATE_ERROR); // remove errored apps. let them be 'repaired' by hand
        apps = apps.filter(app => app.installationState !== exports.ISTATE_PENDING_CONFIGURE); // safeguard against tasks being created non-stop if we crash on startup

        async.eachSeries(apps, function (app, iteratorDone) {
            debug(`configureInstalledApps: marking ${app.fqdn} for reconfigure`);

            const task = {
                args: {},
                values: {},
                scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                requireNullTaskId: false // ignore existing stale taskId
            };

            addTask(app.id, exports.ISTATE_PENDING_CONFIGURE, task,  function (error, result) {
                if (error) debug(`configureInstalledApps: error marking ${app.fqdn} for configure: ${JSON.stringify(error)}`);
                else debug(`configureInstalledApps: marked ${app.id} for re-configure with taskId ${result.taskId}`);

                iteratorDone(); // ignore error
            });
        }, callback);
    });
}

function restartAppsUsingAddons(changedAddons, callback) {
    assert(Array.isArray(changedAddons));
    assert.strictEqual(typeof callback, 'function');

    getAll(function (error, apps) {
        if (error) return callback(error);

        apps = apps.filter(app => app.manifest.addons && _.intersection(Object.keys(app.manifest.addons), changedAddons).length !== 0);
        apps = apps.filter(app => app.installationState !== exports.ISTATE_ERROR); // remove errored apps. let them be 'repaired' by hand
        apps = apps.filter(app => app.installationState !== exports.ISTATE_PENDING_RESTART); // safeguard against tasks being created non-stop restart if we crash on startup
        apps = apps.filter(app => app.runState !== exports.RSTATE_STOPPED); // don't start stopped apps

        async.eachSeries(apps, function (app, iteratorDone) {
            debug(`restartAppsUsingAddons: marking ${app.fqdn} for restart`);

            const task = {
                args: {},
                values: { runState: exports.RSTATE_RUNNING }
            };

            // stop apps before updating the databases because postgres will "lock" them preventing import
            docker.stopContainers(app.id, function (error) {
                if (error) debug(`restartAppsUsingAddons: error stopping ${app.fqdn}`, error);

                addTask(app.id, exports.ISTATE_PENDING_RESTART, task, function (error, result) {
                    if (error) debug(`restartAppsUsingAddons: error marking ${app.fqdn} for restart: ${JSON.stringify(error)}`);
                    else debug(`restartAppsUsingAddons: marked ${app.id} for restart with taskId ${result.taskId}`);

                    iteratorDone(); // ignore error
                });
            });
        }, callback);
    });
}

// auto-restart app tasks after a crash
function schedulePendingTasks(callback) {
    assert.strictEqual(typeof callback, 'function');

    debug('schedulePendingTasks: scheduling app tasks');

    getAll(function (error, result) {
        if (error) return callback(error);

        result.forEach(function (app) {
            if (!app.taskId) return; // if not in any pending state, do nothing

            debug(`schedulePendingTasks: schedule task for ${app.fqdn} ${app.id}: state=${app.installationState},taskId=${app.taskId}`);

            scheduleTask(app.id, app.installationState, app.taskId, NOOP_CALLBACK);
        });

        callback(null);
    });
}

function downloadFile(app, filePath, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof filePath, 'string');
    assert.strictEqual(typeof callback, 'function');

    exec(app, { cmd: [ 'stat', '--printf=%F-%s', filePath ], tty: true }, function (error, stream) {
        if (error) return callback(error);

        var data = '';
        stream.setEncoding('utf8');
        stream.on('data', function (d) { data += d; });
        stream.on('end', function () {
            var parts = data.split('-');
            if (parts.length !== 2) return callback(new BoxError(BoxError.NOT_FOUND, 'file does not exist'));

            var type = parts[0], filename, cmd, size;

            if (type === 'regular file') {
                cmd = [ 'cat', filePath ];
                size = parseInt(parts[1], 10);
                filename = path.basename(filePath);
                if (isNaN(size)) return callback(new BoxError(BoxError.NOT_FOUND, 'file does not exist'));
            } else if (type === 'directory') {
                cmd = ['tar', 'zcf', '-', '-C', filePath, '.'];
                filename = path.basename(filePath) + '.tar.gz';
                size = 0; // unknown
            } else {
                return callback(new BoxError(BoxError.NOT_FOUND, 'only files or dirs can be downloaded'));
            }

            exec(app, { cmd: cmd , tty: false }, function (error, stream) {
                if (error) return callback(error);

                var stdoutStream = new TransformStream({
                    transform: function (chunk, ignoredEncoding, callback) {
                        this._buffer = this._buffer ? Buffer.concat([this._buffer, chunk]) : chunk;

                        for (;;) {
                            if (this._buffer.length < 8) break; // header is 8 bytes

                            var type = this._buffer.readUInt8(0);
                            var len = this._buffer.readUInt32BE(4);

                            if (this._buffer.length < (8 + len)) break; // not enough

                            var payload = this._buffer.slice(8, 8 + len);

                            this._buffer = this._buffer.slice(8+len); // consumed

                            if (type === 1) this.push(payload);
                        }

                        callback();
                    }
                });

                stream.pipe(stdoutStream);

                callback(null, stdoutStream, { filename: filename, size: size });
            });
        });
    });
}

function uploadFile(app, sourceFilePath, destFilePath, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof sourceFilePath, 'string');
    assert.strictEqual(typeof destFilePath, 'string');
    assert.strictEqual(typeof callback, 'function');

    const done = once(function (error) {
        safe.fs.unlinkSync(sourceFilePath); // remove file in /tmp
        if (error) return callback(new BoxError(BoxError.FS_ERROR, error.message)); // blame it on filesystem for now
        callback(null);
    });

    // the built-in bash printf understands "%q" but not /usr/bin/printf.
    // ' gets replaced with '\'' . the first closes the quote and last one starts a new one
    const escapedDestFilePath = safe.child_process.execSync(`printf %q '${destFilePath.replace(/'/g, '\'\\\'\'')}'`, { shell: '/bin/bash', encoding: 'utf8' });
    debug(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`);

    exec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false }, function (error, stream) {
        if (error) return done(error);

        var readFile = fs.createReadStream(sourceFilePath);
        readFile.on('error', done);

        stream.on('error', done);
        stream.on('finish', done);

        readFile.pipe(stream);
    });
}

function backupConfig(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

    if (!safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json'), JSON.stringify(app))) {
        return callback(new BoxError(BoxError.FS_ERROR, 'Error creating config.json: ' + safe.error.message));
    }

    appdb.getIcons(app.id, function (error, icons) {
        if (!error && icons.icon) safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/icon.png'), icons.icon);

        callback(null);
    });
}

function restoreConfig(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

    const appConfig = safe.JSON.parse(safe.fs.readFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json')));
    let data = {};
    if (appConfig) {
        data = _.pick(appConfig, 'memoryLimit', 'cpuShares', 'enableBackup', 'reverseProxyConfig', 'env', 'servicesConfig', 'label', 'tags', 'enableAutomaticUpdate');
    }

    const icon = safe.fs.readFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/icon.png'));
    if (icon) data.icon = icon;

    appdb.update(app.id, data, callback);
}