cloudron-box/src/apps.js

'use strict';

exports = module.exports = {
    canAccess,
    isOperator,
    accessLevel,
    removeInternalFields,
    removeRestrictedFields,

    // database crud
    add,
    update,
    setHealth,
    del,
    delPortBinding,

    get,
    getByIpAddress,
    getByFqdn,
    list,
    listByUser,

    // user actions
    install,
    uninstall,

    setAccessRestriction,
    setOperators,
    setCrontab,
    setLabel,
    setIcon,
    setTags,
    setMemoryLimit,
    setCpuShares,
    setMounts,
    setAutomaticBackup,
    setAutomaticUpdate,
    setReverseProxyConfig,
    setCertificate,
    setDebugMode,
    setEnvironment,
    setMailbox,
    setInbox,
    setLocation,
    setDataDir,
    repair,

    restore,
    importApp,
    exportApp,
    clone,

    updateApp,

    backup,
    listBackups,

    getTask,
    getLocalLogfilePaths,
    getLogs,

    getCertificate,

    start,
    stop,
    restart,

    exec,

    checkManifestConstraints,
    downloadManifest,

    canAutoupdateApp,
    autoupdateApps,

    restoreInstalledApps,
    configureInstalledApps,
    schedulePendingTasks,
    restartAppsUsingAddons,

    getDataDir,
    getIcon,
    getMemoryLimit,
    getLimits,
    getSchedulerConfig,

    listEventlog,

    downloadFile,
    uploadFile,

    backupConfig,
    restoreConfig,

    PORT_TYPE_TCP: 'tcp',
    PORT_TYPE_UDP: 'udp',

    // task codes - the installation state is now a misnomer (keep in sync in UI)
    ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
    ISTATE_PENDING_CLONE: 'pending_clone', // clone
    ISTATE_PENDING_CONFIGURE: 'pending_configure', // infra update
    ISTATE_PENDING_RECREATE_CONTAINER: 'pending_recreate_container', // env change or addon change
    ISTATE_PENDING_LOCATION_CHANGE: 'pending_location_change',
    ISTATE_PENDING_DATA_DIR_MIGRATION: 'pending_data_dir_migration',
    ISTATE_PENDING_RESIZE: 'pending_resize',
    ISTATE_PENDING_DEBUG: 'pending_debug',
    ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
    ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
    ISTATE_PENDING_IMPORT: 'pending_import', // import from external backup
    ISTATE_PENDING_UPDATE: 'pending_update', // update from installed state preserving data
    ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app. this is state because it blocks other operations
    ISTATE_PENDING_START: 'pending_start',
    ISTATE_PENDING_STOP: 'pending_stop',
    ISTATE_PENDING_RESTART: 'pending_restart',
    ISTATE_ERROR: 'error', // error executing last pending_* command
    ISTATE_INSTALLED: 'installed', // app is installed

    // run states
    RSTATE_RUNNING: 'running',
    RSTATE_STOPPED: 'stopped', // app stopped by us

    // health states (keep in sync in UI)
    HEALTH_HEALTHY: 'healthy',
    HEALTH_UNHEALTHY: 'unhealthy',
    HEALTH_ERROR: 'error',
    HEALTH_DEAD: 'dead',

    // subdomain table types
    SUBDOMAIN_TYPE_PRIMARY: 'primary',
    SUBDOMAIN_TYPE_REDIRECT: 'redirect',
    SUBDOMAIN_TYPE_ALIAS: 'alias',

    // exported for testing
    _validatePortBindings: validatePortBindings,
    _validateAccessRestriction: validateAccessRestriction,
    _translatePortBindings: translatePortBindings,
    _parseCrontab: parseCrontab,
    _clear: clear
};

const appstore = require('./appstore.js'),
    appTaskManager = require('./apptaskmanager.js'),
    assert = require('assert'),
    AuditSource = require('./auditsource.js'),
    backups = require('./backups.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    CronJob = require('cron').CronJob,
    database = require('./database.js'),
    debug = require('debug')('box:apps'),
    dns = require('./dns.js'),
    docker = require('./docker.js'),
    domains = require('./domains.js'),
    eventlog = require('./eventlog.js'),
    fs = require('fs'),
    mail = require('./mail.js'),
    manifestFormat = require('cloudron-manifestformat'),
    mounts = require('./mounts.js'),
    once = require('once'),
    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
    semver = require('semver'),
    services = require('./services.js'),
    settings = require('./settings.js'),
    spawn = require('child_process').spawn,
    split = require('split'),
    superagent = require('superagent'),
    system = require('./system.js'),
    tasks = require('./tasks.js'),
    TransformStream = require('stream').Transform,
    users = require('./users.js'),
    util = require('util'),
    uuid = require('uuid'),
    validator = require('validator'),
    _ = require('underscore');

const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson', 'apps.operatorsJson',
    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', 'apps.crontab',
    'apps.creationTime', 'apps.updateTime', 'apps.enableAutomaticUpdate',
    'apps.enableMailbox', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain',
    'apps.dataDir', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');

// const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');

// validate the port bindings
function validatePortBindings(portBindings, manifest) {
    assert.strictEqual(typeof portBindings, 'object');
    assert.strictEqual(typeof manifest, 'object');

    // keep the public ports in sync with firewall rules in setup/start/cloudron-firewall.sh
    // these ports are reserved even if we listen only on 127.0.0.1 because we setup HostIp to be 127.0.0.1
    // for custom tcp ports
    const RESERVED_PORTS = [
        22, /* ssh */
        25, /* smtp */
        80, /* http */
        143, /* imap */
        202, /* alternate ssh */
        222, /* proftd */
        443, /* https */
        465, /* smtps */
        587, /* submission */
        993, /* imaps */
        2003, /* graphite (lo) */
        2004, /* graphite (lo) */
        2514, /* cloudron-syslog (lo) */
        constants.PORT, /* app server (lo) */
        constants.AUTHWALL_PORT, /* protected sites */
        constants.INTERNAL_SMTP_PORT, /* internal smtp port (lo) */
        constants.LDAP_PORT,
        3306, /* mysql (lo) */
        3478, /* turn,stun */
        4190, /* managesieve */
        5349, /* turn,stun TLS */
        8000, /* ESXi monitoring */
        8417, /* graphite (lo) */
    ];

    const RESERVED_PORT_RANGES = [
        [50000, 51000] /* turn udp ports */
    ];

    const ALLOWED_PORTS = [
        53, // dns 53 is special and adblocker apps can use them
        853 // dns over tls
    ];

    if (!portBindings) return null;

    for (let portName in portBindings) {
        if (!/^[a-zA-Z0-9_]+$/.test(portName)) return new BoxError(BoxError.BAD_FIELD, `${portName} is not a valid environment variable`, { field: 'portBindings', portName: portName });

        const hostPort = portBindings[portName];
        if (!Number.isInteger(hostPort)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not an integer`, { field: 'portBindings', portName: portName });
        if (RESERVED_PORTS.indexOf(hostPort) !== -1) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
        if (RESERVED_PORT_RANGES.find(range => (hostPort >= range[0] && hostPort <= range[1]))) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
        if (ALLOWED_PORTS.indexOf(hostPort) === -1 && (hostPort <= 1023 || hostPort > 65535)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not in permitted range`, { field: 'portBindings', portName: portName });
    }

    // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
    // that the user wants the service disabled
    const tcpPorts = manifest.tcpPorts || { };
    const udpPorts = manifest.udpPorts || { };
    for (let portName in portBindings) {
        if (!(portName in tcpPorts) && !(portName in udpPorts)) return new BoxError(BoxError.BAD_FIELD, `Invalid portBindings ${portName}`, { field: 'portBindings', portName: portName });
    }

    return null;
}

function translatePortBindings(portBindings, manifest) {
    assert.strictEqual(typeof portBindings, 'object');
    assert.strictEqual(typeof manifest, 'object');

    if (!portBindings) return null;

    let result = {};
    const tcpPorts = manifest.tcpPorts || { };

    for (let portName in portBindings) {
        const portType = portName in tcpPorts ? exports.PORT_TYPE_TCP : exports.PORT_TYPE_UDP;
        result[portName] = { hostPort: portBindings[portName], type: portType };
    }
    return result;
}

function parseCrontab(crontab) {
    assert(crontab === null || typeof crontab === 'string');

    const result = [];
    if (!crontab) return result;

    const lines = crontab.split('\n');
    for (let i = 0; i < lines.length; i++) {
        const line = lines[i].trim();
        if (!line || line.startsWith('#')) continue;
        const parts = /^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/.exec(line);
        if (!parts) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron configuration at line ${i+1}`);
        const schedule = parts.slice(1, 6).join(' ');
        const command = parts[6];

        try {
            new CronJob('00 ' + schedule, function() {}); // second is disallowed
        } catch (ex) {
            throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern at line ${i+1}`);
        }

        if (command.length === 0) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern. Command must not be empty at line ${i+1}`); // not possible with the regexp we have

        result.push({ schedule, command });
    }

    return result;
}

function getSchedulerConfig(app) {
    assert.strictEqual(typeof app, 'object');

    let schedulerConfig = app.manifest.addons?.scheduler || null;

    const crontab = parseCrontab(app.crontab);
    if (crontab.length === 0) return schedulerConfig;

    schedulerConfig = schedulerConfig || {};
    // put a '.' because it is not a valid name for schedule name in manifestformat
    crontab.forEach((c, idx) => schedulerConfig[`crontab.${idx}`] = c);

    return schedulerConfig;
}

// also validates operators
function validateAccessRestriction(accessRestriction) {
    assert.strictEqual(typeof accessRestriction, 'object');

    if (accessRestriction === null) return null;

    if (accessRestriction.users) {
        if (!Array.isArray(accessRestriction.users)) return new BoxError(BoxError.BAD_FIELD, 'users array property required');
        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new BoxError(BoxError.BAD_FIELD, 'All users have to be strings');
    }

    if (accessRestriction.groups) {
        if (!Array.isArray(accessRestriction.groups)) return new BoxError(BoxError.BAD_FIELD, 'groups array property required');
        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new BoxError(BoxError.BAD_FIELD, 'All groups have to be strings');
    }

    // TODO: maybe validate if the users and groups actually exist
    return null;
}

function validateMemoryLimit(manifest, memoryLimit) {
    assert.strictEqual(typeof manifest, 'object');
    assert.strictEqual(typeof memoryLimit, 'number');

    var min = manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
    var max = os.totalmem() * 2; // this will overallocate since we don't allocate equal swap always (#466)

    // allow 0, which indicates that it is not set, the one from the manifest will be choosen but we don't commit any user value
    // this is needed so an app update can change the value in the manifest, and if not set by the user, the new value should be used
    if (memoryLimit === 0) return null;

    // a special value that indicates unlimited memory
    if (memoryLimit === -1) return null;

    if (memoryLimit < min) return new BoxError(BoxError.BAD_FIELD, 'memoryLimit too small');
    if (memoryLimit > max) return new BoxError(BoxError.BAD_FIELD, 'memoryLimit too large');

    return null;
}

function validateCpuShares(cpuShares) {
    assert.strictEqual(typeof cpuShares, 'number');

    if (cpuShares < 2 || cpuShares > 1024) return new BoxError(BoxError.BAD_FIELD, 'cpuShares has to be between 2 and 1024');

    return null;
}

function validateDebugMode(debugMode) {
    assert.strictEqual(typeof debugMode, 'object');

    if (debugMode === null) return null;
    if ('cmd' in debugMode && debugMode.cmd !== null && !Array.isArray(debugMode.cmd)) return new BoxError(BoxError.BAD_FIELD, 'debugMode.cmd must be an array or null' );
    if ('readonlyRootfs' in debugMode && typeof debugMode.readonlyRootfs !== 'boolean') return new BoxError(BoxError.BAD_FIELD, 'debugMode.readonlyRootfs must be a boolean' );

    return null;
}

function validateRobotsTxt(robotsTxt) {
    if (robotsTxt === null) return null;

    // this is the nginx limit on inline strings. if we really hit this, we have to generate a file
    if (robotsTxt.length > 4096) return new BoxError(BoxError.BAD_FIELD, 'robotsTxt must be less than 4096', { field: 'robotsTxt' });

    // TODO: validate the robots file? we escape the string when templating the nginx config right now

    return null;
}

function validateCsp(csp) {
    if (csp === null) return null;

    if (csp.length > 4096) return new BoxError(BoxError.BAD_FIELD, 'CSP must be less than 4096', { field: 'csp' });

    if (csp.includes('"')) return new BoxError(BoxError.BAD_FIELD, 'CSP cannot contains double quotes', { field: 'csp' });

    return null;
}

function validateBackupFormat(format) {
    assert.strictEqual(typeof format, 'string');

    if (format === 'tgz' || format == 'rsync') return null;

    return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format');
}

function validateLabel(label) {
    if (label === null) return null;

    if (label.length > 128) return new BoxError(BoxError.BAD_FIELD, 'label must be less than 128', { field: 'label' });

    return null;
}

function validateTags(tags) {
    if (tags.length > 64) return new BoxError(BoxError.BAD_FIELD, 'Can only set up to 64 tags', { field: 'tags' });

    if (tags.some(tag => tag.length == 0)) return new BoxError(BoxError.BAD_FIELD, 'tag cannot be empty', { field: 'tags' });
    if (tags.some(tag => tag.length > 128)) return new BoxError(BoxError.BAD_FIELD, 'tag must be less than 128', { field: 'tags' });

    return null;
}

function validateEnv(env) {
    for (let key in env) {
        if (key.length > 512) return new BoxError(BoxError.BAD_FIELD, 'Max env var key length is 512', { field: 'env', env: env });
        // http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap08.html
        if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) return new BoxError(BoxError.BAD_FIELD, `"${key}" is not a valid environment variable`, { field: 'env', env: env });
    }

    return null;
}

function validateDataDir(dataDir) {
    if (dataDir === null) return null;

    if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`, { field: 'dataDir' });
    if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`, { field: 'dataDir' });
    if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`, { field: 'dataDir' });

    // nfs shares will have the directory mounted already
    let stat = safe.fs.lstatSync(dataDir);
    if (stat) {
        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`, { field: 'dataDir' });
        let entries = safe.fs.readdirSync(dataDir);
        if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`, { field: 'dataDir' });
        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`, { field: 'dataDir' });
    }

    // backup logic relies on paths not overlapping (because it recurses)
    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`, { field: 'dataDir' });

    // if we made it this far, it cannot start with any of these realistically
    const fhs = [ '/bin', '/boot', '/etc', '/lib', '/lib32', '/lib64', '/proc', '/run', '/sbin', '/tmp', '/usr' ];
    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`, { field: 'dataDir' });

    return null;
}

function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, portBindings) {
    assert.strictEqual(typeof errorMessage, 'string');
    assert(Array.isArray(locations));
    assert.strictEqual(typeof domainObjectMap, 'object');
    assert.strictEqual(typeof portBindings, 'object');

    var match = errorMessage.match(/ER_DUP_ENTRY: Duplicate entry '(.*)' for key '(.*)'/);
    if (!match) {
        debug('Unexpected SQL error message.', errorMessage);
        return new BoxError(BoxError.DATABASE_ERROR, errorMessage);
    }

    // check if a location conflicts
    if (match[2] === 'subdomain') {
        for (let i = 0; i < locations.length; i++) {
            const { subdomain, domain } = locations[i];
            if  (match[1] !== `${subdomain}-${domain}`) continue;

            return new BoxError(BoxError.ALREADY_EXISTS, `Domain '${dns.fqdn(subdomain, domainObjectMap[domain])}' is in use`, { subdomain, domain });
        }
    }

    // check if any of the port bindings conflict
    for (let portName in portBindings) {
        if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is in use`, { portName });
    }

    if (match[2] === 'dataDir') {
        return new BoxError(BoxError.BAD_FIELD, `Data directory ${match[1]} is in use`, { field: 'dataDir' });
    }

    return new BoxError(BoxError.ALREADY_EXISTS, `${match[2]} '${match[1]}' is in use`);
}

function getDataDir(app, dataDir) {
    assert(dataDir === null || typeof dataDir === 'string');

    return dataDir || path.join(paths.APPS_DATA_DIR, app.id, 'data');
}

function removeInternalFields(app) {
    return _.pick(app,
        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId',
        'location', 'domain', 'fqdn', 'crontab',
        'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'operators',
        'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
        'label', 'alternateDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts',
        'enableMailbox', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
}

// non-admins can only see these
function removeRestrictedFields(app) {
    return _.pick(app,
        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'accessRestriction', 'alternateDomains', 'aliasDomains', 'sso',
        'location', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup');
}

async function getIcon(app, options) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');

    const icons = await getIcons(app.id);
    if (!icons) throw new BoxError(BoxError.NOT_FOUND, 'No such app');

    if (!options.original && icons.icon) return icons.icon;
    if (icons.appStoreIcon) return icons.appStoreIcon;

    return null;
}

async function getLimits(app) {
    assert.strictEqual(typeof app, 'object');

    return {
        memory: await system.getMemory()
    };
}

function getMemoryLimit(app) {
    assert.strictEqual(typeof app, 'object');

    let memoryLimit = app.memoryLimit || app.manifest.memoryLimit || 0;

    if (memoryLimit === -1) { // unrestricted
        memoryLimit = 0;
    } else if (memoryLimit === 0 || memoryLimit < constants.DEFAULT_MEMORY_LIMIT) { // ensure we never go below minimum (in case we change the default)
        memoryLimit = constants.DEFAULT_MEMORY_LIMIT;
    }

    return memoryLimit;
}

function postProcess(result) {
    assert.strictEqual(typeof result, 'object');

    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
    result.manifest = safe.JSON.parse(result.manifestJson);
    delete result.manifestJson;

    assert(result.tagsJson === null || typeof result.tagsJson === 'string');
    result.tags = safe.JSON.parse(result.tagsJson) || [];
    delete result.tagsJson;

    assert(result.reverseProxyConfigJson === null || typeof result.reverseProxyConfigJson === 'string');
    result.reverseProxyConfig = safe.JSON.parse(result.reverseProxyConfigJson) || {};
    delete result.reverseProxyConfigJson;

    assert(result.hostPorts === null || typeof result.hostPorts === 'string');
    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');

    result.portBindings = { };
    let hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
    let environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
    let portTypes = result.portTypes === null ? [ ] : result.portTypes.split(',');

    delete result.hostPorts;
    delete result.environmentVariables;
    delete result.portTypes;

    for (let i = 0; i < environmentVariables.length; i++) {
        result.portBindings[environmentVariables[i]] = { hostPort: parseInt(hostPorts[i], 10), type: portTypes[i] };
    }

    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
    delete result.accessRestrictionJson;

    result.operators = safe.JSON.parse(result.operatorsJson);
    if (result.operators && !result.operators.users) result.operators.users = [];
    delete result.operatorsJson;

    result.sso = !!result.sso;
    result.enableBackup = !!result.enableBackup;
    result.enableAutomaticUpdate = !!result.enableAutomaticUpdate;
    result.enableMailbox = !!result.enableMailbox;
    result.enableInbox = !!result.enableInbox;
    result.proxyAuth = !!result.proxyAuth;
    result.hasIcon = !!result.hasIcon;
    result.hasAppStoreIcon = !!result.hasAppStoreIcon;

    assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
    result.debugMode = safe.JSON.parse(result.debugModeJson);
    delete result.debugModeJson;

    assert(result.servicesConfigJson === null || typeof result.servicesConfigJson === 'string');
    result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
    delete result.servicesConfigJson;

    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
    delete result.subdomains;
    delete result.domains;
    delete result.subdomainTypes;

    result.alternateDomains = [];
    result.aliasDomains = [];
    for (let i = 0; i < subdomainTypes.length; i++) {
        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
            result.location = subdomains[i];
            result.domain = domains[i];
        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
        }
    }

    let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
    delete result.envNames;
    delete result.envValues;
    result.env = {};
    for (let i = 0; i < envNames.length; i++) { // NOTE: envNames is [ null ] when env of an app is empty
        if (envNames[i]) result.env[envNames[i]] = envValues[i];
    }

    let volumeIds = JSON.parse(result.volumeIds);
    delete result.volumeIds;
    let volumeReadOnlys = JSON.parse(result.volumeReadOnlys);
    delete result.volumeReadOnlys;

    result.mounts = volumeIds[0] === null ? [] : volumeIds.map((v, idx) => { return { volumeId: v, readOnly: !!volumeReadOnlys[idx] }; }); // NOTE: volumeIds is [null] when volumes of an app is empty

    result.error = safe.JSON.parse(result.errorJson);
    delete result.errorJson;

    result.taskId = result.taskId ? String(result.taskId) : null;
}

function attachProperties(app, domainObjectMap) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof domainObjectMap, 'object');

    let result = {};
    for (let portName in app.portBindings) {
        result[portName] = app.portBindings[portName].hostPort;
    }
    app.portBindings = result;
    app.iconUrl = app.hasIcon || app.hasAppStoreIcon  ? `/api/v1/apps/${app.id}/icon` : null;
    app.fqdn = dns.fqdn(app.location, domainObjectMap[app.domain]);
    app.alternateDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
    app.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
}

function isAdmin(user) {
    assert.strictEqual(typeof user, 'object');

    return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0;
}

function isOperator(app, user) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof user, 'object');

    if (!app.operators) return isAdmin(user);

    if (app.operators.users.some(function (e) { return e === user.id; })) return true;
    if (!app.operators.groups) return isAdmin(user);
    if (app.operators.groups.some(function (gid) { return Array.isArray(user.groupIds) && user.groupIds.indexOf(gid) !== -1; })) return true;

    return isAdmin(user);
}

function canAccess(app, user) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof user, 'object');

    if (app.accessRestriction === null) return true;

    if (app.accessRestriction.users.some(function (e) { return e === user.id; })) return true;
    if (!app.accessRestriction.groups) return isOperator(app, user);
    if (app.accessRestriction.groups.some(function (gid) { return Array.isArray(user.groupIds) && user.groupIds.indexOf(gid) !== -1; })) return true;

    return isOperator(app, user);
}

function accessLevel(app, user) {
    if (isAdmin(user)) return 'admin';
    if (isOperator(app, user)) return 'operator';
    return canAccess(app, user) ? 'user' : null;
}

async function add(id, appStoreId, manifest, location, domain, portBindings, data) {
    assert.strictEqual(typeof id, 'string');
    assert.strictEqual(typeof appStoreId, 'string');
    assert(manifest && typeof manifest === 'object');
    assert.strictEqual(typeof manifest.version, 'string');
    assert.strictEqual(typeof location, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof portBindings, 'object');
    assert(data && typeof data === 'object');

    portBindings = portBindings || { };

    const manifestJson = JSON.stringify(manifest),
        accessRestriction = data.accessRestriction || null,
        accessRestrictionJson = JSON.stringify(accessRestriction),
        memoryLimit = data.memoryLimit || 0,
        cpuShares = data.cpuShares || 512,
        installationState = data.installationState,
        runState = data.runState,
        sso = 'sso' in data ? data.sso : null,
        debugModeJson = data.debugMode ? JSON.stringify(data.debugMode) : null,
        env = data.env || {},
        label = data.label || null,
        tagsJson = data.tags ? JSON.stringify(data.tags) : null,
        mailboxName = data.mailboxName || null,
        mailboxDomain = data.mailboxDomain || null,
        reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null,
        servicesConfigJson = data.servicesConfig ? JSON.stringify(data.servicesConfig) : null,
        enableMailbox = data.enableMailbox || false,
        icon = data.icon || null;

    const queries = [];

    queries.push({
        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
            + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, '
            + 'enableMailbox) '
            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox ]
    });

    queries.push({
        query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
        args: [ id, domain, location, exports.SUBDOMAIN_TYPE_PRIMARY ]
    });

    Object.keys(portBindings).forEach(function (env) {
        queries.push({
            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, type, appId) VALUES (?, ?, ?, ?)',
            args: [ env, portBindings[env].hostPort, portBindings[env].type, id ]
        });
    });

    Object.keys(env).forEach(function (name) {
        queries.push({
            query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
            args: [ id, name, env[name] ]
        });
    });

    if (data.alternateDomains) {
        data.alternateDomains.forEach(function (d) {
            queries.push({
                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]
            });
        });
    }

    if (data.aliasDomains) {
        data.aliasDomains.forEach(function (d) {
            queries.push({
                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
            });
        });
    }

    const [error] = await safe(database.transaction(queries));
    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, error.message);
    if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, 'no such domain');
    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
}

async function getIcons(id) {
    assert.strictEqual(typeof id, 'string');

    const results = await database.query('SELECT icon, appStoreIcon FROM apps WHERE id = ?', [ id ]);
    if (results.length === 0) return null;
    return { icon: results[0].icon, appStoreIcon: results[0].appStoreIcon };
}

async function updateWithConstraints(id, app, constraints) {
    assert.strictEqual(typeof id, 'string');
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof constraints, 'string');
    assert(!('portBindings' in app) || typeof app.portBindings === 'object');
    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
    assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
    assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
    assert(!('tags' in app) || Array.isArray(app.tags));
    assert(!('env' in app) || typeof app.env === 'object');

    const queries = [ ];

    if ('portBindings' in app) {
        var portBindings = app.portBindings || { };
        // replace entries by app id
        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
        Object.keys(portBindings).forEach(function (env) {
            var values = [ portBindings[env].hostPort, portBindings[env].type, env, id ];
            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, type, environmentVariable, appId) VALUES(?, ?, ?, ?)', args: values });
        });
    }

    if ('env' in app) {
        queries.push({ query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] });

        Object.keys(app.env).forEach(function (name) {
            queries.push({
                query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
                args: [ id, name, app.env[name] ]
            });
        });
    }

    if ('location' in app && 'domain' in app) { // must be updated together as they are unique together
        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ]}); // all locations of an app must be updated together
        queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, app.domain, app.location, exports.SUBDOMAIN_TYPE_PRIMARY ]});

        if ('alternateDomains' in app) {
            app.alternateDomains.forEach(function (d) {
                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
            });
        }

        if ('aliasDomains' in app) {
            app.aliasDomains.forEach(function (d) {
                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
            });
        }
    }

    if ('mounts' in app) {
        queries.push({ query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ]});
        app.mounts.forEach(function (m) {
            queries.push({ query: 'INSERT INTO appMounts (appId, volumeId, readOnly) VALUES (?, ?, ?)', args: [ id, m.volumeId, m.readOnly ]});
        });
    }

    const fields = [ ], values = [ ];
    for (const p in app) {
        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig' || p === 'operators') {
            fields.push(`${p}Json = ?`);
            values.push(JSON.stringify(app[p]));
        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
            fields.push(p + ' = ?');
            values.push(app[p]);
        }
    }

    if (values.length !== 0) {
        values.push(id);
        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
    }

    const [error, results] = await safe(database.transaction(queries));
    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, error.message);
    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
    if (results[results.length - 1].affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'App not found');
}

async function update(id, app) {
    // ts is useful as a versioning mechanism (for example, icon changed). update the timestamp explicity in code instead of db.
    // this way health and healthTime can be updated without changing ts
    app.ts = new Date();
    await updateWithConstraints(id, app, '');
}

async function setHealth(appId, health, healthTime) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof health, 'string');
    assert(util.types.isDate(healthTime));

    await updateWithConstraints(appId, { health, healthTime }, '');
}

async function setTask(appId, values, options) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof values, 'object');
    assert.strictEqual(typeof options, 'object');

    values.ts = new Date();

    if (!options.requireNullTaskId) return await updateWithConstraints(appId, values, '');

    if (options.requiredState === null) {
        await updateWithConstraints(appId, values, 'AND taskId IS NULL');
    } else {
        await updateWithConstraints(appId, values, `AND taskId IS NULL AND installationState = "${options.requiredState}"`);
    }
}

async function del(id) {
    assert.strictEqual(typeof id, 'string');

    const queries = [
        { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
        { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
        { query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ] },
        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
    ];

    const results = await database.transaction(queries);
    if (results[5].affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'App not found');
}

async function delPortBinding(hostPort, type) {
    assert.strictEqual(typeof hostPort, 'number');
    assert.strictEqual(typeof type, 'string');

    const result = await database.query('DELETE FROM appPortBindings WHERE hostPort=? AND type=?', [ hostPort, type ]);
    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'App not found');
}

async function clear() {
    await database.query('DELETE FROM subdomains');
    await database.query('DELETE FROM appPortBindings');
    await database.query('DELETE FROM appAddonConfigs');
    await database.query('DELETE FROM appEnvVars');
    await database.query('DELETE FROM apps');
}

async function getDomainObjectMap() {
    const domainObjects = await domains.list();
    let domainObjectMap = {};
    for (let d of domainObjects) { domainObjectMap[d.domain] = d; }
    return domainObjectMap;
}

// each query simply join apps table with another table by id. we then join the full result together
const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
    + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
    + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
    + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
    + ` LEFT JOIN (${MOUNTS_QUERY}) AS q4 on q4.id = apps.id`;

async function get(id) {
    assert.strictEqual(typeof id, 'string');

    const domainObjectMap = await getDomainObjectMap();

    const result = await database.query(`${APPS_QUERY} WHERE apps.id = ?`, [ id ]);
    if (result.length === 0) return null;

    postProcess(result[0]);
    attachProperties(result[0], domainObjectMap);

    return result[0];
}

// returns the app associated with this IP (app or scheduler)
async function getByIpAddress(ip) {
    assert.strictEqual(typeof ip, 'string');

    const domainObjectMap = await getDomainObjectMap();

    const result = await database.query(`${APPS_QUERY} WHERE apps.containerIp = ?`, [ ip ]);
    if (result.length === 0) return null;

    postProcess(result[0]);
    attachProperties(result[0], domainObjectMap);
    return result[0];
}

async function list() {
    const domainObjectMap = await getDomainObjectMap();

    const results = await database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ]);
    results.forEach(postProcess);
    results.forEach((app) => attachProperties(app, domainObjectMap));
    return results;
}

async function getByFqdn(fqdn) {
    assert.strictEqual(typeof fqdn, 'string');

    const result = await list();
    const app = result.find(function (a) { return a.fqdn === fqdn; });
    return app;
}

async function listByUser(user) {
    assert.strictEqual(typeof user, 'object');

    const result = await list();
    return result.filter((app) => canAccess(app, user));
}

async function getTask(app) {
    assert.strictEqual(typeof app, 'object');

    if (!app.taskId) return null;
    return await tasks.get(app.taskId);
}

async function downloadManifest(appStoreId, manifest) {
    if (!appStoreId && !manifest) throw new BoxError(BoxError.BAD_FIELD, 'Neither manifest nor appStoreId provided');

    if (!appStoreId) return { appStoreId: '', manifest };

    const parts = appStoreId.split('@');

    const url = settings.apiServerOrigin() + '/api/v1/apps/' + parts[0] + (parts[1] ? '/versions/' + parts[1] : '');

    debug('downloading manifest from %s', url);

    const [error, response] = await safe(superagent.get(url).timeout(30 * 1000).ok(() => true));

    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Network error downloading manifest:' + error.message);

    if (response.status !== 200) throw new BoxError(BoxError.NOT_FOUND, `Failed to get app info from store. status: ${response.status} text: ${response.text}`);

    if (!response.body.manifest || typeof response.body.manifest !== 'object') throw new BoxError(BoxError.NOT_FOUND, `Missing manifest. Failed to get app info from store. status: ${response.status} text: ${response.text}`);

    return { appStoreId: parts[0], manifest: response.body.manifest };
}

function mailboxNameForLocation(location, manifest) {
    if (location) return `${location}.app`;
    if (manifest.title) return manifest.title.toLowerCase().replace(/[^a-zA-Z0-9]/g, '') + '.app';
    return 'noreply.app';
}

async function onTaskFinished(appId, installationState, taskId, error) {
    const success = !error;
    const errorMessage = error?.message || null;

    const app = await get(appId);
    const task = await tasks.get(taskId);
    if (!app || !task) return;

    switch (installationState) {
    case exports.ISTATE_PENDING_DATA_DIR_MIGRATION:
        if (success) await safe(services.rebuildService('sftp', AuditSource.APPTASK), { debug });
        break;
    case exports.ISTATE_PENDING_UPDATE: {
        const fromManifest = success ? task.args[1].updateConfig.manifest : app.manifest;
        const toManifest = success ? app.manifest : task.args[1].updateConfig.manifest;

        await eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, AuditSource.APPTASK, { app, toManifest, fromManifest, success, errorMessage });
        break;
    }
    case exports.ISTATE_PENDING_BACKUP:
        await eventlog.add(eventlog.ACTION_APP_BACKUP_FINISH, AuditSource.APPTASK, { app, success, errorMessage, backupId: task.result });
        break;
    }
}

async function scheduleTask(appId, installationState, taskId) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof installationState, 'string');
    assert.strictEqual(typeof taskId, 'string');

    const backupConfig = await settings.getBackupConfig();

    let memoryLimit = 400;
    if (installationState === exports.ISTATE_PENDING_BACKUP || installationState === exports.ISTATE_PENDING_CLONE || installationState === exports.ISTATE_PENDING_RESTORE
        || installationState === exports.ISTATE_PENDING_IMPORT || installationState === exports.ISTATE_PENDING_UPDATE) {
        memoryLimit = 'memoryLimit' in backupConfig ? Math.max(backupConfig.memoryLimit/1024/1024, 400) : 400;
    } else if (installationState === exports.ISTATE_PENDING_DATA_DIR_MIGRATION) {
        memoryLimit = 1024; // cp takes more memory than we think
    }

    const options = { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit };

    appTaskManager.scheduleTask(appId, taskId, options, async function (error) {
        debug(`scheduleTask: task ${taskId} of ${appId} completed`);
        if (error && (error.code === tasks.ECRASHED || error.code === tasks.ESTOPPED)) { // if task crashed, update the error
            debug(`Apptask crashed/stopped: ${error.message}`);
            let boxError = new BoxError(BoxError.TASK_ERROR, error.message);
            boxError.details.crashed = error.code === tasks.ECRASHED;
            boxError.details.stopped = error.code === tasks.ESTOPPED;
            // see also apptask makeTaskError
            boxError.details.taskId = taskId;
            boxError.details.installationState = installationState;
            await safe(update(appId, { installationState: exports.ISTATE_ERROR, error: boxError.toPlainObject(), taskId: null }), { debug });
        } else if (!(installationState === exports.ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall
            await safe(update(appId, { taskId: null }), { debug });
        }

        await safe(onTaskFinished(appId, installationState, taskId, error), { debug }); // ignore error
    });
}

async function addTask(appId, installationState, task) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof installationState, 'string');
    assert.strictEqual(typeof task, 'object'); // { args, values }

    const { args, values } = task;
    // TODO: match the SQL logic to match checkAppState. this means checking the error.installationState and installationState. Unfortunately, former is JSON right now
    const requiredState = null; // 'requiredState' in task ? task.requiredState : exports.ISTATE_INSTALLED;
    const scheduleNow = 'scheduleNow' in task ? task.scheduleNow : true;
    const requireNullTaskId = 'requireNullTaskId' in task ? task.requireNullTaskId : true;

    const taskId = await tasks.add(tasks.TASK_APP, [ appId, args ]);

    const [updateError] = await safe(setTask(appId, _.extend({ installationState, taskId, error: null }, values), { requiredState, requireNullTaskId }));
    if (updateError && updateError.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.BAD_STATE, 'Another task is scheduled for this app'); // could be because app went away OR a taskId exists
    if (updateError) throw updateError;

    if (scheduleNow) await safe(scheduleTask(appId, installationState, taskId), { debug }); // ignore error

    return taskId;
}

function checkAppState(app, state) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof state, 'string');

    if (app.taskId) return new BoxError(BoxError.BAD_STATE, `Locked by task ${app.taskId} : ${app.installationState} / ${app.runState}`);

    if (app.installationState === exports.ISTATE_ERROR) {
        // allow task to be called again if that was the errored task
        if (app.error.installationState === state) return null;

        // allow uninstall from any state
        if (state !== exports.ISTATE_PENDING_UNINSTALL && state !== exports.ISTATE_PENDING_RESTORE && state !== exports.ISTATE_PENDING_IMPORT) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
    }

    if (app.runState === exports.RSTATE_STOPPED) {
        // can't backup or restore since app addons are down. can't update because migration scripts won't run
        if (state === exports.ISTATE_PENDING_UPDATE || state === exports.ISTATE_PENDING_BACKUP || state === exports.ISTATE_PENDING_RESTORE || state === exports.ISTATE_PENDING_IMPORT)  return new BoxError(BoxError.BAD_STATE, 'Not allowed in stopped state');
    }

    return null;
}

async function validateLocations(locations) {
    assert(Array.isArray(locations));

    const domainObjectMap = await getDomainObjectMap();

    for (let location of locations) {
        if (!(location.domain in domainObjectMap)) throw new BoxError(BoxError.BAD_FIELD, 'No such domain', { field: 'location', domain: location.domain, subdomain: location.subdomain });

        let subdomain = location.subdomain;
        if (location.type === 'alias' && subdomain.startsWith('*')) {
            if (subdomain === '*') continue;
            subdomain = subdomain.replace(/^\*\./, ''); // remove *.
        }

        const error = dns.validateHostname(subdomain, domainObjectMap[location.domain]);
        if (error) throw new BoxError(BoxError.BAD_FIELD, 'Bad location: ' + error.message, { field: 'location', domain: location.domain, subdomain: location.subdomain });
    }

    return domainObjectMap;
}

async function install(data, auditSource) {
    assert(data && typeof data === 'object');
    assert.strictEqual(typeof auditSource, 'object');

    assert.strictEqual(typeof data.manifest, 'object'); // manifest is already downloaded

    const location = data.location.toLowerCase(),
        domain = data.domain.toLowerCase(),
        portBindings = data.portBindings || null,
        accessRestriction = data.accessRestriction || null,
        memoryLimit = data.memoryLimit || 0,
        debugMode = data.debugMode || null,
        enableBackup = 'enableBackup' in data ? data.enableBackup : true,
        enableAutomaticUpdate = 'enableAutomaticUpdate' in data ? data.enableAutomaticUpdate : true,
        alternateDomains = data.alternateDomains || [],
        aliasDomains = data.aliasDomains || [],
        env = data.env || {},
        label = data.label || null,
        tags = data.tags || [],
        overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false,
        skipDnsSetup = 'skipDnsSetup' in data ? data.skipDnsSetup : false,
        appStoreId = data.appStoreId,
        manifest = data.manifest;

    let error = manifestFormat.parse(manifest);
    if (error) throw new BoxError(BoxError.BAD_FIELD, `Manifest error: ${error.message}`);

    error = checkManifestConstraints(manifest);
    if (error) throw error;

    error = validatePortBindings(portBindings, manifest);
    if (error) throw error;

    error = validateAccessRestriction(accessRestriction);
    if (error) throw error;

    error = validateMemoryLimit(manifest, memoryLimit);
    if (error) throw error;

    error = validateDebugMode(debugMode);
    if (error) throw error;

    error = validateLabel(label);
    if (error) throw error;

    error = validateTags(tags);
    if (error) throw error;

    let sso = 'sso' in data ? data.sso : null;
    if ('sso' in data && !('optionalSso' in manifest)) throw new BoxError(BoxError.BAD_FIELD, 'sso can only be specified for apps with optionalSso');
    // if sso was unspecified, enable it by default if possible
    if (sso === null) sso = !!manifest.addons['ldap'] || !!manifest.addons['proxyAuth'];

    error = validateEnv(env);
    if (error) throw error;

    if (settings.isDemo() && constants.DEMO_BLACKLISTED_APPS.includes(appStoreId)) throw new BoxError(BoxError.BAD_FIELD, 'This app is blacklisted in the demo');

    // sendmail is enabled by default
    const enableMailbox = 'enableMailbox' in data ? data.enableMailbox : true;
    const mailboxName = manifest.addons?.sendmail ? mailboxNameForLocation(location, manifest) : null;
    const mailboxDomain = manifest.addons?.sendmail ? domain : null;

    let icon = data.icon || null;
    if (icon) {
        if (!validator.isBase64(icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' });
        icon = Buffer.from(icon, 'base64');
    }

    const locations = [{ subdomain: location, domain, type: 'primary' }]
        .concat(alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
        .concat(aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));

    const domainObjectMap = await validateLocations(locations);

    const appId = uuid.v4();
    debug('Will install app with id : ' + appId);

    const app = {
        accessRestriction,
        memoryLimit,
        sso,
        debugMode,
        mailboxName,
        mailboxDomain,
        enableBackup,
        enableAutomaticUpdate,
        alternateDomains,
        aliasDomains,
        env,
        label,
        tags,
        icon,
        enableMailbox,
        runState: exports.RSTATE_RUNNING,
        installationState: exports.ISTATE_PENDING_INSTALL
    };

    const [addError] = await safe(add(appId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), app));
    if (addError && addError.reason === BoxError.ALREADY_EXISTS) throw getDuplicateErrorDetails(addError.message, locations, domainObjectMap, portBindings);
    if (addError) throw addError;

    await purchaseApp({ appId, appstoreId: appStoreId, manifestId: manifest.id || 'customapp' });

    const task = {
        args: { restoreConfig: null, skipDnsSetup, overwriteDns },
        values: { },
        requiredState: app.installationState
    };

    const taskId = await addTask(appId, app.installationState, task);

    const newApp = _.extend({}, _.omit(app, 'icon'), { appStoreId, manifest, location, domain, portBindings });
    newApp.fqdn = dns.fqdn(newApp.location, domainObjectMap[newApp.domain]);
    newApp.alternateDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });

    await eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId, app: newApp, taskId });

    return { id : appId, taskId };
}

async function setAccessRestriction(app, accessRestriction, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof accessRestriction, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = validateAccessRestriction(accessRestriction);
    if (error) throw error;

    await update(appId, { accessRestriction });
    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, accessRestriction });
}

async function setOperators(app, operators, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof operators, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    const error = validateAccessRestriction(operators); // not a typo. same structure for operators and accessRestriction
    if (error) throw error;

    await update(appId, { operators });
    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, operators });
}

async function setCrontab(app, crontab, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert(crontab === null || typeof crontab === 'string');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    parseCrontab(crontab);

    await update(appId, { crontab });
    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, crontab });
}

async function setLabel(app, label, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof label, 'string');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = validateLabel(label);
    if (error) throw error;

    await update(appId, { label });
    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, label });
}

async function setTags(app, tags, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert(Array.isArray(tags));
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = validateTags(tags);
    if (error) throw error;

    await update(appId, { tags });
    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, tags });
}

async function setIcon(app, icon, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert(icon === null || typeof icon === 'string');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;

    if (icon) {
        if (!validator.isBase64(icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' });
        icon = Buffer.from(icon, 'base64');
    }

    await update(appId, { icon });
    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, iconChanged: true });
}

async function setMemoryLimit(app, memoryLimit, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof memoryLimit, 'number');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RESIZE);
    if (error) throw error;

    error = validateMemoryLimit(app.manifest, memoryLimit);
    if (error) throw error;

    const task = {
        args: {},
        values: { memoryLimit }
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_RESIZE, task);

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, memoryLimit, taskId });

    return { taskId };
}

async function setCpuShares(app, cpuShares, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof cpuShares, 'number');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RESIZE);
    if (error) throw error;

    error = validateCpuShares(cpuShares);
    if (error) throw error;

    const task = {
        args: {},
        values: { cpuShares }
    };
    const taskId = await safe(addTask(appId, exports.ISTATE_PENDING_RESIZE, task));

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, cpuShares, taskId });

    return { taskId };
}

async function setMounts(app, mounts, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert(Array.isArray(mounts));
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER);
    if (error) throw error;

    const task = {
        args: {},
        values: { mounts }
    };
    const [taskError, taskId] = await safe(addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task));
    if (taskError && taskError.reason === BoxError.ALREADY_EXISTS) throw new BoxError(BoxError.CONFLICT, 'Duplicate mount points');
    if (taskError) throw taskError;

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mounts, taskId });

    return { taskId };
}

async function setEnvironment(app, env, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof env, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER);
    if (error) throw error;

    error = validateEnv(env);
    if (error) throw error;

    const task = {
        args: {},
        values: { env }
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task);

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, env, taskId });

    return { taskId };
}

async function setDebugMode(app, debugMode, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof debugMode, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_DEBUG);
    if (error) throw error;

    error = validateDebugMode(debugMode);
    if (error) throw error;

    const task = {
        args: {},
        values: { debugMode }
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_DEBUG, task);

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, debugMode, taskId });

    return { taskId };
}

async function setMailbox(app, data, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    assert.strictEqual(typeof data.enable, 'boolean');

    const enableMailbox = data.enable;

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER);
    if (error) throw error;

    if (!app.manifest.addons?.sendmail) throw new BoxError(BoxError.BAD_FIELD, 'App does not use sendmail');
    const optional = 'optional' in app.manifest.addons.sendmail ? app.manifest.addons.sendmail.optional : false;
    if (!optional && !enableMailbox) throw new BoxError(BoxError.BAD_FIELD, 'App requires sendmail to be enabled');

    let mailboxName = data.mailboxName || null;
    const mailboxDomain = data.mailboxDomain || null;

    if (enableMailbox) {
        await mail.getDomain(mailboxDomain); // check if domain exists

        if (mailboxName) {
            error = mail.validateName(mailboxName);
            if (error) throw new BoxError(BoxError.BAD_FIELD, error.message, { field: 'mailboxName' });
        } else {
            mailboxName = mailboxNameForLocation(app.location, app.domain, app.manifest);
        }
    }

    const task = {
        args: {},
        values: { enableMailbox, mailboxName, mailboxDomain }
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task);

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mailboxName, mailboxDomain, taskId });

    return { taskId };
}

async function setInbox(app, data, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    assert.strictEqual(typeof data.enable, 'boolean');

    const enableInbox = data.enable;
    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER);
    if (error) throw error;

    if (!app.manifest.addons?.recvmail) throw new BoxError(BoxError.BAD_FIELD, 'App does not use recvmail addon');

    const inboxName = data.inboxName || null;
    const inboxDomain = data.inboxDomain || null;
    if (enableInbox) {
        const domain = await mail.getDomain(data.inboxDomain); // check if domain exists
        if (!domain.enabled) throw new BoxError(BoxError.BAD_FIELD, 'Domain does not have incoming email enabled');

        error = mail.validateName(data.inboxName);
        if (error) throw new BoxError(BoxError.BAD_FIELD, error.message, { field: 'inboxName' });
    }

    const task = {
        args: {},
        values: { enableInbox, inboxName, inboxDomain }
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task);

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, enableInbox, inboxName, inboxDomain, taskId });

    return { taskId };
}

async function setAutomaticBackup(app, enable, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof enable, 'boolean');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    await update(appId, { enableBackup: enable });
    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, enableBackup: enable });
}

async function setAutomaticUpdate(app, enable, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof enable, 'boolean');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    await update(appId, { enableAutomaticUpdate: enable });
    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, enableAutomaticUpdate: enable });
}

async function setReverseProxyConfig(app, reverseProxyConfig, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof reverseProxyConfig, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    reverseProxyConfig = _.extend({ robotsTxt: null, csp: null }, reverseProxyConfig);

    const appId = app.id;
    let error = validateCsp(reverseProxyConfig.csp);
    if (error) throw error;

    error = validateRobotsTxt(reverseProxyConfig.robotsTxt);
    if (error) throw error;

    await reverseProxy.writeAppConfig(_.extend({}, app, { reverseProxyConfig }));

    await update(appId, { reverseProxyConfig });

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, reverseProxyConfig });
}

async function setCertificate(app, data, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert(data && typeof data === 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    const { location, domain, cert, key } = data;

    const domainObject = await domains.get(domain);

    if (cert && key) {
        const error = reverseProxy.validateCertificate(location, domainObject, { cert, key });
        if (error) throw error;
    }

    await reverseProxy.setAppCertificate(location, domainObject, { cert, key });
    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, cert, key });
}

async function setLocation(app, data, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_LOCATION_CHANGE);
    if (error) throw error;

    let values = {
        location: data.location.toLowerCase(),
        domain: data.domain.toLowerCase(),
        // these are intentionally reset, if not set
        portBindings: null,
        alternateDomains: [],
        aliasDomains: []
    };

    if ('portBindings' in data) {
        error = validatePortBindings(data.portBindings, app.manifest);
        if (error) throw error;

        values.portBindings = translatePortBindings(data.portBindings || null, app.manifest);
    }

    // rename the auto-created mailbox to match the new location
    if (app.manifest.addons?.sendmail && app.mailboxName.endsWith('.app')) {
        values.mailboxName = mailboxNameForLocation(values.location, app.manifest);
        values.mailboxDomain = values.domain;
    }

    if ('alternateDomains' in data) {
        values.alternateDomains = data.alternateDomains;
    }

    if ('aliasDomains' in data) {
        values.aliasDomains = data.aliasDomains;
    }

    const locations = [{ subdomain: values.location, domain: values.domain, type: 'primary' }]
        .concat(values.alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
        .concat(values.aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));

    const domainObjectMap = await validateLocations(locations);

    const task = {
        args: {
            oldConfig: _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'aliasDomains', 'portBindings'),
            skipDnsSetup: !!data.skipDnsSetup,
            overwriteDns: !!data.overwriteDns
        },
        values
    };
    let [taskError, taskId] = await safe(addTask(appId, exports.ISTATE_PENDING_LOCATION_CHANGE, task));
    if (taskError && taskError.reason === BoxError.ALREADY_EXISTS) taskError = getDuplicateErrorDetails(error.message, locations, domainObjectMap, data.portBindings);
    if (taskError) throw taskError;

    values.fqdn = dns.fqdn(values.location, domainObjectMap[values.domain]);
    values.alternateDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
    values.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, _.extend({ appId, app, taskId }, values));

    return { taskId };
}

async function setDataDir(app, dataDir, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert(dataDir === null || typeof dataDir === 'string');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_DATA_DIR_MIGRATION);
    if (error) throw error;

    error = validateDataDir(dataDir);
    if (error) throw error;

    const task = {
        args: { newDataDir: dataDir },
        values: {}
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task);

    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, dataDir, taskId });

    return { taskId };
}

async function updateApp(app, data, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert(data && typeof data === 'object');
    assert(data.manifest && typeof data.manifest === 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const skipBackup = !!data.skipBackup,
        appId = app.id,
        manifest = data.manifest,
        appStoreId = data.appStoreId;

    let values = {};

    if (app.runState === exports.RSTATE_STOPPED) throw new BoxError(BoxError.BAD_STATE, 'Stopped apps cannot be updated');

    let error = checkAppState(app, exports.ISTATE_PENDING_UPDATE);
    if (error) throw error;

    error = manifestFormat.parse(manifest);
    if (error) throw new BoxError(BoxError.BAD_FIELD, 'Manifest error:' + error.message);

    error = checkManifestConstraints(manifest);
    if (error) throw error;

    const updateConfig = { skipBackup, manifest, appStoreId }; // this will clear appStoreId when updating from a repo and set it if passed in for update route

    // prevent user from installing a app with different manifest id over an existing app
    // this allows cloudron install -f --app <appid> for an app installed from the appStore
    if (app.manifest.id !== updateConfig.manifest.id) {
        if (!data.force) throw new BoxError(BoxError.BAD_FIELD, 'manifest id does not match. force to override');
    }

    // suffix '0' if prerelease is missing for semver.lte to work as expected
    const currentVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
    const updateVersion = semver.prerelease(updateConfig.manifest.version) ? updateConfig.manifest.version : `${updateConfig.manifest.version}-0`;
    if (app.appStoreId !== '' && semver.lte(updateVersion, currentVersion)) {
        if (!data.force) throw new BoxError(BoxError.BAD_FIELD, 'Downgrades are not permitted for apps installed from AppStore. force to override');
    }

    if ('icon' in data) {
        if (data.icon) {
            if (!validator.isBase64(data.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' });
            data.icon = Buffer.from(data.icon, 'base64');
        }
        values.icon = data.icon;
    }

    // do not update apps in debug mode
    if (app.debugMode && !data.force) throw new BoxError(BoxError.BAD_STATE, 'debug mode enabled. force to override');

    // Ensure we update the memory limit in case the new app requires more memory as a minimum
    // 0 and -1 are special updateConfig for memory limit indicating unset and unlimited
    if (app.memoryLimit > 0 && updateConfig.manifest.memoryLimit && app.memoryLimit < updateConfig.manifest.memoryLimit) {
        updateConfig.memoryLimit = updateConfig.manifest.memoryLimit;
    }

    if (!manifest.addons?.sendmail) { // clear if the update removed addon
        values.mailboxName = values.mailboxDomain = null;
    } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since update added the addon
        values.mailboxName = mailboxNameForLocation(app.location, manifest);
        values.mailboxDomain = app.domain;
    }

    const task = {
        args: { updateConfig },
        values
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_UPDATE, task);

    await eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId, app, skipBackup, toManifest: manifest, fromManifest: app.manifest, force: data.force, taskId });

    return { taskId };
}

function getLocalLogfilePaths(app) {
    assert.strictEqual(typeof app, 'object');

    const appId = app.id;

    var filePaths = [];
    filePaths.push(path.join(paths.LOG_DIR, appId, 'apptask.log'));
    filePaths.push(path.join(paths.LOG_DIR, appId, 'app.log'));
    if (app.manifest.addons && app.manifest.addons.redis) filePaths.push(path.join(paths.LOG_DIR, `redis-${appId}/app.log`));

    return filePaths;
}

async function getLogs(app, options) {
    assert.strictEqual(typeof app, 'object');
    assert(options && typeof options === 'object');

    assert.strictEqual(typeof options.lines, 'number');
    assert.strictEqual(typeof options.format, 'string');
    assert.strictEqual(typeof options.follow, 'boolean');

    const appId = app.id;

    const lines = options.lines === -1 ? '+1' : options.lines,
        format = options.format || 'json',
        follow = options.follow;

    assert.strictEqual(typeof format, 'string');

    const args = [ '--lines=' + lines ];
    if (follow) args.push('--follow', '--retry', '--quiet'); // same as -F. to make it work if file doesn't exist, --quiet to not output file headers, which are no logs

    const cp = spawn('/usr/bin/tail', args.concat(getLocalLogfilePaths(app)));

    const transformStream = split(function mapper(line) {
        if (format !== 'json') return line + '\n';

        var data = line.split(' '); // logs are <ISOtimestamp> <msg>
        var timestamp = (new Date(data[0])).getTime();
        if (isNaN(timestamp)) timestamp = 0;
        var message = line.slice(data[0].length+1);

        // ignore faulty empty logs
        if (!timestamp && !message) return;

        return JSON.stringify({
            realtimeTimestamp: timestamp * 1000,
            message: message,
            source: appId
        }) + '\n';
    });

    transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process

    cp.stdout.pipe(transformStream);

    return transformStream;
}

async function getCertificate(subdomain, domain) {
    assert.strictEqual(typeof subdomain, 'string');
    assert.strictEqual(typeof domain, 'string');

    const result = await database.query('SELECT certificateJson FROM subdomains WHERE subdomain=? AND domain=?', [ subdomain, domain ]);
    if (result.length === 0) return null;
    return JSON.parse(result[0].certificateJson);
}

// does a re-configure when called from most states. for install/clone errors, it re-installs with an optional manifest
// re-configure can take a dockerImage but not a manifest because re-configure does not clean up addons
async function repair(app, data, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object'); // { manifest }
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let errorState = (app.error && app.error.installationState) || exports.ISTATE_PENDING_CONFIGURE;

    const task = {
        args: {},
        values: {},
        requiredState: null
    };

    // maybe split this into a separate route like reinstall?
    if (errorState === exports.ISTATE_PENDING_INSTALL || errorState === exports.ISTATE_PENDING_CLONE) {
        task.args = { skipDnsSetup: false, overwriteDns: true };
        if (data.manifest) {
            let error = manifestFormat.parse(data.manifest);
            if (error) throw new BoxError(BoxError.BAD_FIELD, `manifest error: ${error.message}`);

            error = checkManifestConstraints(data.manifest);
            if (error) throw error;

            if (!data.manifest.addons?.sendmail) { // clear if repair removed addon
                task.values.mailboxName = task.values.mailboxDomain = null;
            } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since repair added the addon
                task.values.mailboxName = mailboxNameForLocation(app.location, data.manifest);
                task.values.mailboxDomain = app.domain;
            }

            task.values.manifest = data.manifest;
            task.args.oldManifest = app.manifest;
        }
    } else {
        errorState = exports.ISTATE_PENDING_CONFIGURE;
        if (data.dockerImage) {
            let newManifest = _.extend({}, app.manifest, { dockerImage: data.dockerImage });
            task.values.manifest = newManifest;
        }
    }

    const taskId = await addTask(appId, errorState, task);

    await eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { app, taskId });

    return { taskId };
}

async function restore(app, backupId, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof backupId, 'string');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;

    let error = checkAppState(app, exports.ISTATE_PENDING_RESTORE);
    if (error) throw error;

    // for empty or null backupId, use existing manifest to mimic a reinstall
    const backupInfo = backupId ? await backups.get(backupId) : { manifest: app.manifest };

    if (!backupInfo.manifest) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Could not get restore manifest');
    if (backupInfo.encryptionVersion === 1) throw new BoxError(BoxError.BAD_FIELD, 'This encrypted backup was created with an older Cloudron version and has to be restored using the CLI tool');

    // re-validate because this new box version may not accept old configs
    error = checkManifestConstraints(backupInfo.manifest);
    if (error) throw error;

    let values = { manifest: backupInfo.manifest };
    if (!backupInfo.manifest.addons?.sendmail) { // clear if restore removed addon
        values.mailboxName = values.mailboxDomain = null;
    } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since restore added the addon
        values.mailboxName = mailboxNameForLocation(app.location, backupInfo.manifest);
        values.mailboxDomain = app.domain;
    }

    const restoreConfig = { backupId, backupFormat: backupInfo.format };

    const task = {
        args: {
            restoreConfig,
            oldManifest: app.manifest,
            skipDnsSetup: !!backupId, // if this is a restore, just skip dns setup. only re-installs should setup dns
            overwriteDns: true
        },
        values
    };

    const taskId = await addTask(appId, exports.ISTATE_PENDING_RESTORE, task);

    await eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId: backupInfo.id, fromManifest: app.manifest, toManifest: backupInfo.manifest, taskId });

    return { taskId };
}

async function importApp(app, data, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;

    // all fields are optional
    data.backupId = data.backupId || null;
    data.backupFormat = data.backupFormat || null;
    data.backupConfig = data.backupConfig || null;
    const { backupId, backupFormat, backupConfig } = data;

    let error = backupFormat ? validateBackupFormat(backupFormat) : null;
    if (error) throw error;

    error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
    if (error) throw error;

    // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
    if (backupConfig) {
        if (mounts.isMountProvider(backupConfig.provider)) {
            backupConfig.mountPoint = `/mnt/appimport-${app.id}`;
            error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions);
            if (error) throw error;

            const mountObject = { // keep this in sync with the import code in apptask
                name: `appimport-${app.id}`,
                hostPath: backupConfig.mountPoint,
                mountType: backupConfig.provider,
                mountOptions: backupConfig.mountOptions
            };
            await mounts.tryAddMount(mountObject, { timeout: 10 });

        }
        error = await backups.testProviderConfig(backupConfig);
        if (error) throw error;
    }

    if (backupConfig) {
        if ('password' in backupConfig) {
            backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.password);
            delete backupConfig.password;
        } else {
            backupConfig.encryption = null;
        }
    }

    const restoreConfig = { backupId, backupFormat, backupConfig };

    const task = {
        args: {
            restoreConfig,
            oldManifest: app.manifest,
            skipDnsSetup: false,
            overwriteDns: true
        },
        values: {}
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_IMPORT, task);

    await eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app: app, backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId });

    return { taskId };
}

async function exportApp(app, data, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;

    let error = checkAppState(app, exports.ISTATE_PENDING_BACKUP);
    if (error) throw error;

    const task = {
        args: { snapshotOnly: true },
        values: {}
    };

    const taskId = await addTask(appId, exports.ISTATE_PENDING_BACKUP, task);
    return { taskId };
}

async function purchaseApp(data) {
    assert.strictEqual(typeof data, 'object');

    const [purchaseError] = await safe(appstore.purchaseApp(data));
    if (!purchaseError) return;

    await del(data.appId);

    throw purchaseError;
}

async function clone(app, data, user, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof data, 'object');
    assert(user && typeof user === 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const location = data.location.toLowerCase(),
        domain = data.domain.toLowerCase(),
        portBindings = data.portBindings || null,
        backupId = data.backupId,
        overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false,
        skipDnsSetup = 'skipDnsSetup' in data ? data.skipDnsSetup : false,
        appId = app.id;

    assert.strictEqual(typeof backupId, 'string');
    assert.strictEqual(typeof location, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof portBindings, 'object');

    const locations = [{ subdomain: location, domain, type: 'primary' }];
    const domainObjectMap = await validateLocations(locations);
    const backupInfo = await backups.get(backupId);

    if (!backupInfo.manifest) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Could not get restore config');
    if (backupInfo.encryptionVersion === 1) throw new BoxError(BoxError.BAD_FIELD, 'This encrypted backup was created with an older Cloudron version and cannot be cloned');

    const manifest = backupInfo.manifest, appStoreId = app.appStoreId;

    // re-validate because this new box version may not accept old configs
    let error = checkManifestConstraints(manifest);
    if (error) throw error;

    error = validatePortBindings(portBindings, manifest);
    if (error) throw error;

    // should we copy the original app's mailbox settings instead?
    const mailboxName = manifest.addons?.sendmail ? mailboxNameForLocation(location, manifest) : null;
    const mailboxDomain = manifest.addons?.sendmail ? domain : null;

    const newAppId = uuid.v4();

    const icons = await getIcons(app.id);

    const obj = {
        installationState: exports.ISTATE_PENDING_CLONE,
        runState: exports.RSTATE_RUNNING,
        memoryLimit: app.memoryLimit,
        cpuShares: app.cpuShares,
        accessRestriction: app.accessRestriction,
        sso: !!app.sso,
        mailboxName: mailboxName,
        mailboxDomain: mailboxDomain,
        enableBackup: app.enableBackup,
        reverseProxyConfig: app.reverseProxyConfig,
        env: app.env,
        alternateDomains: [],
        aliasDomains: [],
        servicesConfig: app.servicesConfig,
        label: app.label ? `${app.label}-clone` : '',
        tags: app.tags,
        enableAutomaticUpdate: app.enableAutomaticUpdate,
        icon: icons.icon,
        enableMailbox: app.enableMailbox
    };

    const [addError] = await safe(add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), obj));
    if (addError && addError.reason === BoxError.ALREADY_EXISTS) throw getDuplicateErrorDetails(addError.message, locations, domainObjectMap, portBindings);
    if (addError) throw addError;

    await purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' });

    const restoreConfig = { backupId, backupFormat: backupInfo.format };
    const task = {
        args: { restoreConfig, overwriteDns, skipDnsSetup, oldManifest: null },
        values: {},
        requiredState: exports.ISTATE_PENDING_CLONE
    };
    const taskId = await addTask(newAppId, exports.ISTATE_PENDING_CLONE, task);

    const newApp = _.extend({}, _.omit(obj, 'icon'), { appStoreId, manifest, location, domain, portBindings });
    newApp.fqdn = dns.fqdn(newApp.location, domainObjectMap[newApp.domain]);
    newApp.alternateDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });

    await eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId, oldApp: app, newApp, taskId });

    return { id: newAppId, taskId };
}

async function uninstall(app, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_UNINSTALL);
    if (error) throw error;

    await appstore.unpurchaseApp(appId, { appstoreId: app.appStoreId, manifestId: app.manifest.id || 'customapp' });

    const task = {
        args: {},
        values: {},
        requiredState: null // can run in any state, as long as no task is active
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_UNINSTALL, task);
    await eventlog.add(eventlog.ACTION_APP_UNINSTALL, auditSource, { appId, app, taskId });

    return { taskId };
}

async function start(app, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_START);
    if (error) throw error;

    const task = {
        args: {},
        values: { runState: exports.RSTATE_RUNNING }
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_START, task);
    await eventlog.add(eventlog.ACTION_APP_START, auditSource, { appId, app, taskId });
    return { taskId };
}

async function stop(app, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_STOP);
    if (error) throw error;

    const task = {
        args: {},
        values: { runState: exports.RSTATE_STOPPED }
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_STOP, task);

    await eventlog.add(eventlog.ACTION_APP_STOP, auditSource, { appId, app, taskId });

    return { taskId };
}

async function restart(app, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;
    let error = checkAppState(app, exports.ISTATE_PENDING_RESTART);
    if (error) throw error;

    const task = {
        args: {},
        values: { runState: exports.RSTATE_RUNNING }
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_RESTART, task);

    await eventlog.add(eventlog.ACTION_APP_RESTART, auditSource, { appId, app, taskId });

    return { taskId };
}

function checkManifestConstraints(manifest) {
    assert(manifest && typeof manifest === 'object');

    if (manifest.manifestVersion > 2) return new BoxError(BoxError.BAD_FIELD, 'Manifest version must be <= 2');

    if (!manifest.dockerImage) return new BoxError(BoxError.BAD_FIELD, 'Missing dockerImage'); // dockerImage is optional in manifest

    if (semver.valid(manifest.maxBoxVersion) && semver.gt(constants.VERSION, manifest.maxBoxVersion)) {
        return new BoxError(BoxError.BAD_FIELD, 'Box version exceeds Apps maxBoxVersion');
    }

    if (semver.valid(manifest.minBoxVersion) && semver.gt(manifest.minBoxVersion, constants.VERSION)) {
        return new BoxError(BoxError.BAD_FIELD, 'App version requires a new platform version');
    }

    return null;
}

async function exec(app, options) {
    assert.strictEqual(typeof app, 'object');
    assert(options && typeof options === 'object');

    const cmd = options.cmd || [ '/bin/bash' ];
    assert(Array.isArray(cmd) && cmd.length > 0);

    if (app.installationState !== exports.ISTATE_INSTALLED || app.runState !== exports.RSTATE_RUNNING) {
        throw new BoxError(BoxError.BAD_STATE, 'App not installed or running');
    }

    const execOptions = {
        AttachStdin: true,
        AttachStdout: true,
        AttachStderr: true,
        // A pseudo tty is a terminal which processes can detect (for example, disable colored output)
        // Creating a pseudo terminal also assigns a terminal driver which detects control sequences
        // When passing binary data, tty must be disabled. In addition, the stdout/stderr becomes a single
        // unified stream because of the nature of a tty (see https://github.com/docker/docker/issues/19696)
        Tty: options.tty,
        Cmd: cmd
    };

    const startOptions = {
        Detach: false,
        Tty: options.tty,
        // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
        // we can work with half-close connections (not defined in http). this way, the client
        // can properly signal that stdin is EOF by closing it's side of the socket. In http,
        // the whole connection will be dropped when stdin get EOF.
        // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
        hijack: true,
        stream: true,
        stdin: true,
        stdout: true,
        stderr: true
    };

    const stream = await docker.execContainer(app.containerId, { execOptions, startOptions, rows: options.rows, columns: options.columns });
    return stream;
}

function canAutoupdateApp(app, updateInfo) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof updateInfo, 'object');

    const manifest = updateInfo.manifest;

    if (!app.enableAutomaticUpdate) return false;

    // for invalid subscriptions the appstore does not return a dockerImage
    if (!manifest.dockerImage) return false;

    if (updateInfo.unstable) return false; // only manual update allowed for unstable updates

    if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(manifest.version))) return false; // major changes are blocking

    if (app.runState === exports.RSTATE_STOPPED) return false; // stopped apps won't run migration scripts and shouldn't be updated

    const newTcpPorts = manifest.tcpPorts || { };
    const newUdpPorts = manifest.udpPorts || { };
    const portBindings = app.portBindings; // this is never null

    for (let portName in portBindings) {
        if (!(portName in newTcpPorts) && !(portName in newUdpPorts)) return false; // portName was in use but new update removes it
    }

    // it's fine if one or more (unused) keys got removed
    return true;
}

async function autoupdateApps(updateInfo, auditSource) { // updateInfo is { appId -> { manifest } }
    assert.strictEqual(typeof updateInfo, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    for (const appId of Object.keys(updateInfo)) {
        const [getError, app] = await safe(get(appId));
        if (getError) {
            debug(`Cannot autoupdate app ${appId}: ${getError.message}`);
            continue;
        }

        if (!canAutoupdateApp(app, updateInfo[appId])) {
            debug(`app ${app.fqdn} requires manual update`);
            continue;
        }

        const data = {
            manifest: updateInfo[appId].manifest,
            force: false
        };

        const [updateError] = await safe(updateApp(app, data, auditSource));
        if (updateError) debug(`Error initiating autoupdate of ${appId}. ${updateError.message}`);
    }
}

async function backup(app, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const appId = app.id;

    let error = checkAppState(app, exports.ISTATE_PENDING_BACKUP);
    if (error) throw error;

    const task = {
        args: {},
        values: {}
    };
    const taskId = await addTask(appId, exports.ISTATE_PENDING_BACKUP, task);
    await eventlog.add(eventlog.ACTION_APP_BACKUP, auditSource, { app, appId, taskId });

    return { taskId };
}

async function listBackups(app, page, perPage) {
    assert.strictEqual(typeof app, 'object');
    assert(typeof page === 'number' && page > 0);
    assert(typeof perPage === 'number' && perPage > 0);

    return await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, page, perPage);
}

async function restoreInstalledApps(options) {
    assert.strictEqual(typeof options, 'object');

    let apps = await list();
    apps = apps.filter(app => app.installationState !== exports.ISTATE_ERROR); // remove errored apps. let them be 'repaired' by hand
    apps = apps.filter(app => app.installationState !== exports.ISTATE_PENDING_RESTORE); // safeguard against tasks being created non-stop if we crash on startup

    for (const app of apps) {
        const [error, results] = await safe(backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1));
        let installationState, restoreConfig, oldManifest;
        if (!error && results.length) {
            installationState = exports.ISTATE_PENDING_RESTORE;
            restoreConfig = { backupId: results[0].id, backupFormat: results[0].format };
            oldManifest = app.manifest;
        } else {
            installationState = exports.ISTATE_PENDING_INSTALL;
            restoreConfig = null;
            oldManifest = null;
        }

        const task = {
            args: { restoreConfig, skipDnsSetup: options.skipDnsSetup, overwriteDns: true, oldManifest },
            values: {},
            scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
            requireNullTaskId: false // ignore existing stale taskId
        };

        debug(`restoreInstalledApps: marking ${app.fqdn} for restore using restore config ${JSON.stringify(restoreConfig)}`);

        const [addTaskError, taskId] = await safe(addTask(app.id, installationState, task));
        if (addTaskError) debug(`restoreInstalledApps: error marking ${app.fqdn} for restore: ${JSON.stringify(addTaskError)}`);
        else debug(`restoreInstalledApps: marked ${app.id} for restore with taskId ${taskId}`);
    }
}

async function configureInstalledApps() {
    let apps = await list();
    apps = apps.filter(app => app.installationState !== exports.ISTATE_ERROR); // remove errored apps. let them be 'repaired' by hand
    apps = apps.filter(app => app.installationState !== exports.ISTATE_PENDING_CONFIGURE); // safeguard against tasks being created non-stop if we crash on startup

    for (const app of apps) {
        debug(`configureInstalledApps: marking ${app.fqdn} for reconfigure`);

        const task = {
            args: {},
            values: {},
            scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
            requireNullTaskId: false // ignore existing stale taskId
        };

        const [addTaskError, taskId] = await safe(addTask(app.id, exports.ISTATE_PENDING_CONFIGURE, task));
        if (addTaskError) debug(`configureInstalledApps: error marking ${app.fqdn} for configure: ${JSON.stringify(addTaskError)}`);
        else debug(`configureInstalledApps: marked ${app.id} for re-configure with taskId ${taskId}`);
    }
}

async function restartAppsUsingAddons(changedAddons) {
    assert(Array.isArray(changedAddons));

    let apps = await list();
    apps = apps.filter(app => app.manifest.addons && _.intersection(Object.keys(app.manifest.addons), changedAddons).length !== 0);
    apps = apps.filter(app => app.installationState !== exports.ISTATE_ERROR); // remove errored apps. let them be 'repaired' by hand
    apps = apps.filter(app => app.installationState !== exports.ISTATE_PENDING_RESTART); // safeguard against tasks being created non-stop restart if we crash on startup
    apps = apps.filter(app => app.runState !== exports.RSTATE_STOPPED); // don't start stopped apps

    for (const app of apps) {
        debug(`restartAppsUsingAddons: marking ${app.fqdn} for restart`);

        const task = {
            args: {},
            values: { runState: exports.RSTATE_RUNNING }
        };

        // stop apps before updating the databases because postgres will "lock" them preventing import
        const [stopError] = await safe(docker.stopContainers(app.id));
        if (stopError) debug(`restartAppsUsingAddons: error stopping ${app.fqdn}`, stopError);

        const [addTaskError, taskId] = await safe(addTask(app.id, exports.ISTATE_PENDING_RESTART, task));
        if (addTaskError) debug(`restartAppsUsingAddons: error marking ${app.fqdn} for restart: ${JSON.stringify(addTaskError)}`);
        else debug(`restartAppsUsingAddons: marked ${app.id} for restart with taskId ${taskId}`);
    }
}

// auto-restart app tasks after a crash
async function schedulePendingTasks() {
    debug('schedulePendingTasks: scheduling app tasks');

    const result = await list();

    for (const app of result) {
        if (!app.taskId) continue; // if not in any pending state, do nothing

        debug(`schedulePendingTasks: schedule task for ${app.fqdn} ${app.id}: state=${app.installationState},taskId=${app.taskId}`);

        await safe(scheduleTask(app.id, app.installationState, app.taskId), { debug }); // ignore error
    }
}

async function listEventlog(app, page, perPage) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    const actions = [];
    const search = app.id;
    return await eventlog.listPaged(actions, search, page, perPage);
}

function downloadFile(app, filePath, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof filePath, 'string');
    assert.strictEqual(typeof callback, 'function');

    exec(app, { cmd: [ 'stat', '--printf=%F-%s', filePath ], tty: true }, function (error, stream) {
        if (error) return callback(error);

        var data = '';
        stream.setEncoding('utf8');
        stream.on('data', function (d) { data += d; });
        stream.on('end', function () {
            var parts = data.split('-');
            if (parts.length !== 2) return callback(new BoxError(BoxError.NOT_FOUND, 'file does not exist'));

            var type = parts[0], filename, cmd, size;

            if (type === 'regular file') {
                cmd = [ 'cat', filePath ];
                size = parseInt(parts[1], 10);
                filename = path.basename(filePath);
                if (isNaN(size)) return callback(new BoxError(BoxError.NOT_FOUND, 'file does not exist'));
            } else if (type === 'directory') {
                cmd = ['tar', 'zcf', '-', '-C', filePath, '.'];
                filename = path.basename(filePath) + '.tar.gz';
                size = 0; // unknown
            } else {
                return callback(new BoxError(BoxError.NOT_FOUND, 'only files or dirs can be downloaded'));
            }

            exec(app, { cmd: cmd , tty: false }, function (error, stream) {
                if (error) return callback(error);

                var stdoutStream = new TransformStream({
                    transform: function (chunk, ignoredEncoding, callback) {
                        this._buffer = this._buffer ? Buffer.concat([this._buffer, chunk]) : chunk;

                        for (;;) {
                            if (this._buffer.length < 8) break; // header is 8 bytes

                            var type = this._buffer.readUInt8(0);
                            var len = this._buffer.readUInt32BE(4);

                            if (this._buffer.length < (8 + len)) break; // not enough

                            var payload = this._buffer.slice(8, 8 + len);

                            this._buffer = this._buffer.slice(8+len); // consumed

                            if (type === 1) this.push(payload);
                        }

                        callback();
                    }
                });

                stream.pipe(stdoutStream);

                callback(null, stdoutStream, { filename: filename, size: size });
            });
        });
    });
}

function uploadFile(app, sourceFilePath, destFilePath, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof sourceFilePath, 'string');
    assert.strictEqual(typeof destFilePath, 'string');
    assert.strictEqual(typeof callback, 'function');

    const done = once(function (error) {
        safe.fs.unlinkSync(sourceFilePath); // remove file in /tmp
        if (error) return callback(new BoxError(BoxError.FS_ERROR, error.message)); // blame it on filesystem for now
        callback(null);
    });

    // the built-in bash printf understands "%q" but not /usr/bin/printf.
    // ' gets replaced with '\'' . the first closes the quote and last one starts a new one
    const escapedDestFilePath = safe.child_process.execSync(`printf %q '${destFilePath.replace(/'/g, '\'\\\'\'')}'`, { shell: '/bin/bash', encoding: 'utf8' });
    debug(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`);

    exec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false }, function (error, stream) {
        if (error) return done(error);

        var readFile = fs.createReadStream(sourceFilePath);
        readFile.on('error', done);

        stream.on('error', done);
        stream.on('finish', done);

        readFile.pipe(stream);
    });
}

async function backupConfig(app) {
    assert.strictEqual(typeof app, 'object');

    if (!safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json'), JSON.stringify(app))) {
        throw new BoxError(BoxError.FS_ERROR, 'Error creating config.json: ' + safe.error.message);
    }

    const [error, icons] = await safe(getIcons(app.id));
    if (!error && icons.icon) safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/icon.png'), icons.icon);
}

async function restoreConfig(app) {
    assert.strictEqual(typeof app, 'object');

    const appConfig = safe.JSON.parse(safe.fs.readFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json')));
    let data = {};
    if (appConfig) {
        data = _.pick(appConfig, 'memoryLimit', 'cpuShares', 'enableBackup', 'reverseProxyConfig', 'env', 'servicesConfig', 'label', 'tags', 'enableAutomaticUpdate');
    }

    const icon = safe.fs.readFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/icon.png'));
    if (icon) data.icon = icon;

    await update(app.id, data);
}