return 422 on instance id mismatch

the ui redirects otherwise
Revert "rename ami to aws-mp"
2019-12-11 15:13:38 -08:00 · 2019-12-11 12:56:30 -08:00 · 2019-12-11 12:27:55 -08:00 · 2019-12-11 10:05:45 -08:00 · 2019-12-11 14:02:40 +01:00 · 2019-12-10 18:01:07 +01:00
44 changed files with 697 additions and 515 deletions
@@ -1735,3 +1735,15 @@
 * Add 'New Apps' section to Appstore view
 * Fix issue where graphs of some apps were not appearing

+[4.4.0]
+* Show swap in graphs
+* Make avatars customizable
+* Hide access tokens from logs
+* Add missing '@' sign for email address in app mailbox
+* Add app fqdn to backup progress message
+* import: add option to import app in-place
+* import: add option to import app from arbitrary backup config
+* Show download progress for rsync backups
+* Fix various repair workflows
+* acme2: Implement post-as-get
+
@@ -9,7 +9,7 @@ exports.up = function(db, callback) {
            if (!mailbox.membersJson) return iteratorDone();

            let members = JSON.parse(mailbox.membersJson);
-            members = members.map((m) => m.indexOf('@') === -1 ? `${m}@${mailbox.domain}` : m); // only because we don't do things in a xction
+            members = members.map((m) => m && m.indexOf('@') === -1 ? `${m}@${mailbox.domain}` : m); // only because we don't do things in a xction

            db.runSql('UPDATE mailboxes SET membersJson=? WHERE name=? AND domain=?', [ JSON.stringify(members), mailbox.name, mailbox.domain ], iteratorDone);
        }, callback);
@@ -22,7 +22,7 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
 mkdir -p platformdata/addons/mail platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks

 # put cert
@@ -51,6 +51,7 @@ mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
 mkdir -p "${PLATFORM_DATA_DIR}/update"

 mkdir -p "${BOX_DATA_DIR}/appicons"
+mkdir -p "${BOX_DATA_DIR}/profileicons"
 mkdir -p "${BOX_DATA_DIR}/certs"
 mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
 mkdir -p "${BOX_DATA_DIR}/mail/dkim"
@@ -40,7 +40,6 @@ var accesscontrol = require('./accesscontrol.js'),
    crypto = require('crypto'),
    debug = require('debug')('box:addons'),
    docker = require('./docker.js'),
-    dockerConnection = docker.connection,
    fs = require('fs'),
    graphs = require('./graphs.js'),
    hat = require('./hat.js'),
@@ -232,33 +231,10 @@ function dumpPath(addon, appId) {
    }
 }

-function restartContainer(serviceName, callback) {
-    assert.strictEqual(typeof serviceName, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    assert(KNOWN_SERVICES[serviceName], `Unknown service ${serviceName}`);
-
-    docker.stopContainer(serviceName, function (error) {
-        if (error) return callback(error);
-
-        docker.startContainer(serviceName, function (error) {
-            if (error && error.reason === BoxError.NOT_FOUND) {
-                callback(null); // callback early since rebuilding takes long
-                return rebuildService(serviceName, function (error) { if (error) console.error(`Unable to rebuild service ${serviceName}`, error); });
-            }
-            if (error) return callback(error);
-
-            callback(null);
-        });
-    });
-}
-
 function rebuildService(serviceName, callback) {
    assert.strictEqual(typeof serviceName, 'string');
    assert.strictEqual(typeof callback, 'function');

-    assert(KNOWN_SERVICES[serviceName], `Unknown service ${serviceName}`);
-
    // this attempts to recreate the service docker container if they don't exist but platform infra version is unchanged
    // passing an infra version of 'none' will not attempt to purge existing data, not sure if this is good or bad
    if (serviceName === 'mongodb') return startMongodb({ version: 'none' }, callback);
@@ -271,6 +247,24 @@ function rebuildService(serviceName, callback) {
    callback();
 }

+function restartContainer(serviceName, callback) {
+    assert.strictEqual(typeof serviceName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.stopContainer(serviceName, function (error) {
+        if (error) return callback(error);
+
+        docker.startContainer(serviceName, function (error) {
+            if (error && error.reason === BoxError.NOT_FOUND) {
+                callback(null); // callback early since rebuilding takes long
+                return rebuildService(serviceName, function (error) { if (error) console.error(`Unable to rebuild service ${serviceName}`, error); });
+            }
+
+            callback(error);
+        });
+    });
+}
+
 function getServiceDetails(containerName, tokenEnvName, callback) {
    assert.strictEqual(typeof containerName, 'string');
    assert.strictEqual(typeof tokenEnvName, 'string');
@@ -280,15 +274,15 @@ function getServiceDetails(containerName, tokenEnvName, callback) {
        if (error) return callback(error);

        const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
-        if (!ip) return callback(new BoxError(BoxError.INACTIVE, `Error getting ${containerName} container ip`));
+        if (!ip) return callback(new BoxError(BoxError.INACTIVE, `Error getting IP of ${containerName} service`));

        // extract the cloudron token for auth
        const env = safe.query(result, 'Config.Env', null);
-        if (!env) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} env`));
+        if (!env) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error inspecting environment of ${containerName} service`));
        const tmp = env.find(function (e) { return e.indexOf(tokenEnvName) === 0; });
-        if (!tmp) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} cloudron token env var`));
+        if (!tmp) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`));
        const token = tmp.slice(tokenEnvName.length + 1); // +1 for the = sign
-        if (!token)  return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} cloudron token`));
+        if (!token)  return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`));

        callback(null, { ip: ip, token: token, state: result.State });
    });
@@ -483,8 +477,8 @@ function waitForService(containerName, tokenEnvName, callback) {

        async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
            request.get(`https://${result.ip}:3000/healthcheck?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
-                if (error) return retryCallback(new Error(`Error waiting for ${containerName}: ${error.message}`));
-                if (response.statusCode !== 200 || !response.body.status) return retryCallback(new Error(`Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Network error waiting for ${containerName}: ${error.message}`));
+                if (response.statusCode !== 200 || !response.body.status) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`));

                retryCallback(null);
            });
@@ -502,7 +496,7 @@ function setupAddons(app, addons, callback) {
    debugApp(app, 'setupAddons: Setting up %j', Object.keys(addons));

    async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));

        debugApp(app, 'Setting up addon %s with options %j', addon, addons[addon]);

@@ -520,7 +514,7 @@ function teardownAddons(app, addons, callback) {
    debugApp(app, 'teardownAddons: Tearing down %j', Object.keys(addons));

    async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));

        debugApp(app, 'Tearing down addon %s with options %j', addon, addons[addon]);

@@ -540,7 +534,7 @@ function backupAddons(app, addons, callback) {
    debugApp(app, 'backupAddons: Backing up %j', Object.keys(addons));

    async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));

        KNOWN_ADDONS[addon].backup(app, addons[addon], iteratorCallback);
    }, callback);
@@ -558,7 +552,7 @@ function clearAddons(app, addons, callback) {
    debugApp(app, 'clearAddons: clearing %j', Object.keys(addons));

    async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));

        KNOWN_ADDONS[addon].clear(app, addons[addon], iteratorCallback);
    }, callback);
@@ -576,7 +570,7 @@ function restoreAddons(app, addons, callback) {
    debugApp(app, 'restoreAddons: restoring %j', Object.keys(addons));

    async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));

        KNOWN_ADDONS[addon].restore(app, addons[addon], iteratorCallback);
    }, callback);
@@ -587,7 +581,7 @@ function importAppDatabase(app, addon, callback) {
    assert.strictEqual(typeof addon, 'string');
    assert.strictEqual(typeof callback, 'function');

-    if (!(addon in KNOWN_ADDONS)) return callback(new Error(`No such addon: ${addon}`));
+    if (!(addon in KNOWN_ADDONS)) return callback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));

    async.series([
        KNOWN_ADDONS[addon].setup.bind(null, app, app.manifest.addons[addon]),
@@ -1052,8 +1046,8 @@ function setupMySql(app, options, callback) {
            if (error) return callback(error);

            request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up mysql: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mysql: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mysql. Status code: ${response.statusCode} message: ${response.body.message}`));

                const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';

@@ -1090,9 +1084,10 @@ function clearMySql(app, options, callback) {
    getServiceDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) {
        if (error) return callback(error);

-        request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mysql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+
            callback();
        });
    });
@@ -1109,9 +1104,9 @@ function teardownMySql(app, options, callback) {
    getServiceDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) {
        if (error) return callback(error);

-        request.delete(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mysql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql. Status code: ${response.statusCode} message: ${response.body.message}`));

            appdb.unsetAddonConfig(app.id, 'mysql', callback);
        });
@@ -1130,14 +1125,15 @@ function pipeRequestToFile(url, filename, callback) {
        callback(error);
    });

-    writeStream.on('error', done);
+    writeStream.on('error', (error) => done(new BoxError(BoxError.FS_ERROR, `Error writing to ${filename}: ${error.message}`)));
+
    writeStream.on('open', function () {
        // note: do not attach to post callback handler because this will buffer the entire reponse!
        // see https://github.com/request/request/issues/2270
        const req = request.post(url, { rejectUnauthorized: false });
-        req.on('error', done); // network error, dns error, request errored in middle etc
+        req.on('error', (error) => done(new BoxError(BoxError.NETWORK_ERROR, `Request error writing to ${filename}: ${error.message}`))); // network error, dns error, request errored in middle etc
        req.on('response', function (response) {
-            if (response.statusCode !== 200) return done(new Error(`Unexpected response code: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`));
+            if (response.statusCode !== 200) return done(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code when piping ${url}: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`));

            response.pipe(writeStream).on('finish', done); // this is hit after data written to disk
        });
@@ -1176,11 +1172,11 @@ function restoreMySql(app, options, callback) {
        if (error) return callback(error);

        var input = fs.createReadStream(dumpPath('mysql', app.id));
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mysql: ${error.message}`)));

-        const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from mysql addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql. Status code: ${response.statusCode} message: ${response.body.message}`));

            callback(null);
        });
@@ -1265,8 +1261,8 @@ function setupPostgreSql(app, options, callback) {
            if (error) return callback(error);

            request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up postgresql: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up postgresql: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));

                const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';

@@ -1298,9 +1294,9 @@ function clearPostgreSql(app, options, callback) {
    getServiceDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) {
        if (error) return callback(error);

-        request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing postgresql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));

            callback(null);
        });
@@ -1317,9 +1313,9 @@ function teardownPostgreSql(app, options, callback) {
    getServiceDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) {
        if (error) return callback(error);

-        request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error tearing down postgresql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error tearing down postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error tearing down postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));

            appdb.unsetAddonConfig(app.id, 'postgresql', callback);
        });
@@ -1358,11 +1354,11 @@ function restorePostgreSql(app, options, callback) {
        if (error) return callback(error);

        var input = fs.createReadStream(dumpPath('postgresql', app.id));
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring postgresql: ${error.message}`)));

-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from postgresql addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));

            callback(null);
        });
@@ -1441,8 +1437,8 @@ function setupMongoDb(app, options, callback) {
            if (error) return callback(error);

            request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up mongodb: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up mongodb. Status code: ${response.statusCode}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mongodb: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));

                const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';

@@ -1476,9 +1472,9 @@ function clearMongodb(app, options, callback) {
    getServiceDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) {
        if (error) return callback(error);

-        request.post(`https://${result.ip}:3000/databases/${app.id}/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mongodb: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/databases/${app.id}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));

            callback();
        });
@@ -1495,9 +1491,9 @@ function teardownMongoDb(app, options, callback) {
    getServiceDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) {
        if (error) return callback(error);

-        request.delete(`https://${result.ip}:3000/databases/${app.id}?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error tearing down mongodb: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error tearing down mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/databases/${app.id}?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));

            appdb.unsetAddonConfig(app.id, 'mongodb', callback);
        });
@@ -1532,11 +1528,11 @@ function restoreMongoDb(app, options, callback) {
        if (error) return callback(error);

        const readStream = fs.createReadStream(dumpPath('mongodb', app.id));
-        readStream.on('error', callback);
+        readStream.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mongodb: ${error.message}`)));

-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${app.id}/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from mongodb addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/databases/${app.id}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));

            callback(null);
        });
@@ -1654,9 +1650,9 @@ function clearRedis(app, options, callback) {
    getServiceDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) {
        if (error) return callback(error);

-        request.post(`https://${result.ip}:3000/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing redis: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing redis. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing redis: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing redis. Status code: ${response.statusCode} message: ${response.body.message}`));

            callback(null);
        });
@@ -1668,18 +1664,11 @@ function teardownRedis(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var container = dockerConnection.getContainer('redis-' + app.id);
-
-    var removeOptions = {
-        force: true, // kill container if it's running
-        v: true // removes volumes associated with the container
-    };
-
-    container.remove(removeOptions, function (error) {
-        if (error && error.statusCode !== 404) return callback(new Error('Error removing container:' + error));
+    docker.deleteContainer(`redis-${app.id}`, function (error) {
+        if (error) return callback(error);

        shell.sudo('removeVolume', [ RMADDONDIR_CMD, 'redis', app.id ], {}, function (error) {
-            if (error) return callback(new Error('Error removing redis data:' + error));
+            if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`));

            rimraf(path.join(paths.LOG_DIR, `redis-${app.id}`), function (error) {
                if (error) debugApp(app, 'cannot cleanup logs: %s', error);
@@ -1712,6 +1701,8 @@ function restoreRedis(app, options, callback) {

    debugApp(app, 'Restoring redis');

+    callback = once(callback); // protect from multiple returns with streams
+
    getServiceDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) {
        if (error) return callback(error);

@@ -1722,11 +1713,11 @@ function restoreRedis(app, options, callback) {
        } else { // old location of dumps
            input = fs.createReadStream(path.join(paths.APPS_DATA_DIR, app.id, 'redis/dump.rdb'));
        }
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring redis: ${error.message}`)));

-        const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from redis addon: ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis. Status code: ${response.statusCode} message: ${response.body.message}`));

            callback(null);
        });
@@ -1811,7 +1802,7 @@ function statusGraphite(callback) {
        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, { status: exports.SERVICE_STATUS_STOPPED });
        if (error) return callback(error);

-        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { timeout: 3000 }, function (error, response) {
+        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { json: true, timeout: 3000 }, function (error, response) {
            if (error) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite: ${error.message}` });
            if (response.statusCode !== 200) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite. Status code: ${response.statusCode} message: ${response.body.message}` });

@@ -622,8 +622,8 @@ function addTask(appId, installationState, task, callback) {
    assert.strictEqual(typeof callback, 'function');

    const { args, values } = task;
-    // by default, a task can only run on installed state. if it's null, it can be run on any state
-    const requiredState = 'requiredState' in task ? task.requiredState : exports.ISTATE_INSTALLED;
+    // TODO: match the SQL logic to match checkAppState. this means checking the error.installationState and installationState. Unfortunately, former is JSON right now
+    const requiredState = null; // 'requiredState' in task ? task.requiredState : exports.ISTATE_INSTALLED;
    const scheduleNow = 'scheduleNow' in task ? task.scheduleNow : true;
    const requireNullTaskId = 'requireNullTaskId' in task ? task.requireNullTaskId : true;

@@ -645,9 +645,13 @@ function checkAppState(app, state) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof state, 'string');

-    if (app.taskId) return new BoxError(BoxError.BAD_STATE, `Not allowed in this app state : ${app.installationState} / ${app.runState}`);
+    if (app.taskId) return new BoxError(BoxError.BAD_STATE, `Locked by task ${app.taskId} : ${app.installationState} / ${app.runState}`);

    if (app.installationState === exports.ISTATE_ERROR) {
+        // allow task to be called again if that was the errored task
+        if (app.error.installationState === state) return null;
+
+        // allow uninstall from any state
        if (state !== exports.ISTATE_PENDING_UNINSTALL) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
    }

@@ -1196,13 +1200,13 @@ function setDataDir(appId, dataDir, auditSource, callback) {
        if (error) return callback(error);

        const task = {
-            args: { oldDataDir: app.dataDir },
-            values: { dataDir: dataDir }
+            args: { newDataDir: dataDir },
+            values: { }
        };
        addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, function (error, result) {
            if (error) return callback(error);

-            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId, app: app, dataDir: dataDir, taskId: result.taskId });
+            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId, app: app, newDataDir: dataDir, taskId: result.taskId });

            callback(null, { taskId: result.taskId });
        });
@@ -1344,9 +1348,10 @@ function getLogs(appId, options, callback) {
    });
 }

+// does a re-configure when called from most states. for install/clone errors, it re-installs with an optional manifest
 function repair(appId, data, auditSource, callback) {
    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof data, 'object'); // { manifest }
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

@@ -1355,42 +1360,42 @@ function repair(appId, data, auditSource, callback) {
    get(appId, function (error, app) {
        if (error) return callback(error);

-        const appError = app.error || {}; // repair can always be called
-        const newState = appError.installationState ? appError.installationState : exports.ISTATE_PENDING_CONFIGURE;
+        const errorState = (app.error && app.error.installationState) || exports.ISTATE_PENDING_CONFIGURE;

-        debug(`Repairing app with error: ${JSON.stringify(error)} and state: ${newState}`);
+        const task = {
+            args: {},
+            values: {},
+            requiredState: null
+        };

-        let values = _.pick(data, 'location', 'domain', 'alternateDomains');
+        // maybe split this into a separate route like reinstall?
+        if (errorState === exports.ISTATE_PENDING_INSTALL || errorState === exports.ISTATE_PENDING_CLONE) {
+            task.args = { overwriteDns: true };
+            if (data.manifest) {
+                error = manifestFormat.parse(data.manifest);
+                if (error) return callback(new BoxError(BoxError.BAD_FIELD, `manifest error: ${error.message}`));

-        const locations = (values.location ? [ { subdomain: values.location, domain: values.domain } ] : []).concat(values.alternateDomains || []);
-        validateLocations(locations, function (error, domainObjectMap) {
+                error = checkManifestConstraints(data.manifest);
+                if (error) return callback(error);
+
+                task.values.manifest = data.manifest;
+                task.args.oldManifest = app.manifest;
+            }
+        }
+
+        addTask(appId, errorState, task, function (error, result) {
            if (error) return callback(error);

-            tasks.get(appError.taskId || '', function (error, task) {
-                let args = !error ? task.args[1] : {}; // pick args for the failed task. the first argument is the app id
+            eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { taskId: result.taskId, app });

-                if ('backupId' in data) {
-                    args.restoreConfig = data.backupId ? { backupId: data.backupId, backupFormat: data.backupFormat, oldManifest: app.manifest } : null; // when null, apptask simply reinstalls
-                }
-                args.overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false;
-
-                // create a new task instead of updating the old one, since it helps tracking
-                addTask(appId, newState, { args, values, requiredState: null }, function (error, result) {
-                    if (error && error.reason === BoxError.ALREADY_EXISTS) error = getDuplicateErrorDetails(error.message, locations, domainObjectMap, { /* portBindings */});
-                    if (error) return callback(error);
-
-                    eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { taskId: result.taskId, app, newState });
-
-                    callback(null, { taskId: result.taskId });
-                });
-            });
+            callback(null, { taskId: result.taskId });
        });
    });
 }

-function restore(appId, data, auditSource, callback) {
+function restore(appId, backupId, auditSource, callback) {
    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof backupId, 'string');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

@@ -1403,7 +1408,7 @@ function restore(appId, data, auditSource, callback) {
        if (error) return callback(error);

        // for empty or null backupId, use existing manifest to mimic a reinstall
-        var func = data.backupId ? backups.get.bind(null, data.backupId) : function (next) { return next(null, { manifest: app.manifest }); };
+        var func = backupId ? backups.get.bind(null, backupId) : function (next) { return next(null, { manifest: app.manifest }); };

        func(function (error, backupInfo) {
            if (error) return callback(error);
@@ -1414,11 +1419,12 @@ function restore(appId, data, auditSource, callback) {
            error = checkManifestConstraints(backupInfo.manifest);
            if (error) return callback(error);

-            const restoreConfig = { backupId: data.backupId, backupFormat: backupInfo.format, oldManifest: app.manifest };
+            const restoreConfig = { backupId, backupFormat: backupInfo.format };

            const task = {
                args: {
                    restoreConfig,
+                    oldManifest: app.manifest,
                    overwriteDns: true
                },
                values: {
@@ -1447,28 +1453,41 @@ function importApp(appId, data, auditSource, callback) {
    get(appId, function (error, app) {
        if (error) return callback(error);

-        error = validateBackupFormat(data.backupFormat);
+        // all fields are optional
+        data.backupId = data.backupId || null;
+        data.backupFormat = data.backupFormat || null;
+        data.backupConfig = data.backupConfig || null;
+        const { backupId, backupFormat, backupConfig } = data;
+
+        error = backupFormat ? validateBackupFormat(backupFormat) : null;
        if (error) return callback(error);

        error = checkAppState(app, exports.ISTATE_PENDING_RESTORE);
        if (error) return callback(error);

-        // TODO: check if the file exists in the storage backend
-        const restoreConfig = { backupId: data.backupId, backupFormat: data.backupFormat, oldManifest: app.manifest };
+        // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
+        const testBackupConfig = backupConfig ? backups.testProviderConfig.bind(null, backupConfig) : (next) => next();

-        const task = {
-            args: {
-                restoreConfig,
-                overwriteDns: true
-            },
-            values: {}
-        };
-        addTask(appId, exports.ISTATE_PENDING_RESTORE, task, function (error, result) {
+        testBackupConfig(function (error) {
            if (error) return callback(error);

-            eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId: data.backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId: result.taskId });
+            const restoreConfig = { backupId, backupFormat, backupConfig };

-            callback(null, { taskId: result.taskId });
+            const task = {
+                args: {
+                    restoreConfig,
+                    oldManifest: app.manifest,
+                    overwriteDns: true
+                },
+                values: {}
+            };
+            addTask(appId, exports.ISTATE_PENDING_RESTORE, task, function (error, result) {
+                if (error) return callback(error);
+
+                eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId: result.taskId });
+
+                callback(null, { taskId: result.taskId });
+            });
        });
    });
 }
@@ -1554,9 +1573,9 @@ function clone(appId, data, user, auditSource, callback) {
                    purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' }, function (error) {
                        if (error) return callback(error);

-                        const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format, oldManifest: null };
+                        const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format };
                        const task = {
-                            args: { restoreConfig, overwriteDns },
+                            args: { restoreConfig, overwriteDns, oldManifest: null },
                            values: {},
                            requiredState: exports.ISTATE_PENDING_CLONE
                        };
@@ -1690,8 +1709,6 @@ function exec(appId, options, callback) {
            return callback(new BoxError(BoxError.BAD_STATE, 'App not installed or running'));
        }

-        var container = docker.connection.getContainer(app.containerId);
-
        var execOptions = {
            AttachStdin: true,
            AttachStdout: true,
@@ -1704,55 +1721,44 @@ function exec(appId, options, callback) {
            Cmd: cmd
        };

-        container.exec(execOptions, function (error, exec) {
+        var startOptions = {
+            Detach: false,
+            Tty: options.tty,
+            // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
+            // we can work with half-close connections (not defined in http). this way, the client
+            // can properly signal that stdin is EOF by closing it's side of the socket. In http,
+            // the whole connection will be dropped when stdin get EOF.
+            // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
+            hijack: true,
+            stream: true,
+            stdin: true,
+            stdout: true,
+            stderr: true
+        };
+
+        docker.execContainer(app.containerId, { execOptions, startOptions, rows: options.rows, columns: options.columns }, function (error, stream) {
            if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running
            if (error) return callback(error);

-            var startOptions = {
-                Detach: false,
-                Tty: options.tty,
-                // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
-                // we can work with half-close connections (not defined in http). this way, the client
-                // can properly signal that stdin is EOF by closing it's side of the socket. In http,
-                // the whole connection will be dropped when stdin get EOF.
-                // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
-                hijack: true,
-                stream: true,
-                stdin: true,
-                stdout: true,
-                stderr: true
-            };
-            exec.start(startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
-                if (error) return callback(error);
-
-                if (options.rows && options.columns) {
-                    // there is a race where resizing too early results in a 404 "no such exec"
-                    // https://git.cloudron.io/cloudron/box/issues/549
-                    setTimeout(function () {
-                        exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
-                    }, 2000);
-                }
-
-                return callback(null, stream);
-            });
+            callback(null, stream);
        });
    });
 }

 function canAutoupdateApp(app, newManifest) {
-    if (!app.enableAutomaticUpdate) return new Error('Automatic update disabled');
-    if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(newManifest.version))) return new Error('Major version change'); // major changes are blocking
+    if (!app.enableAutomaticUpdate) return false;
+    if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(newManifest.version))) return false; // major changes are blocking

    const newTcpPorts = newManifest.tcpPorts || { };
    const newUdpPorts = newManifest.udpPorts || { };
    const portBindings = app.portBindings; // this is never null

    for (let portName in portBindings) {
-        if (!(portName in newTcpPorts) && !(portName in newUdpPorts)) return new Error(`${portName} was in use but new update removes it`);
+        if (!(portName in newTcpPorts) && !(portName in newUdpPorts)) return false; // portName was in use but new update removes it
    }

    // it's fine if one or more (unused) keys got removed
-    return null;
+    return true;
 }

 function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is { appId -> { manifest } }
@@ -1769,9 +1775,8 @@ function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is {
                return iteratorDone();
            }

-            error = canAutoupdateApp(app, updateInfo[appId].manifest);
-            if (error) {
-                debug('app %s requires manual update. %s', appId, error.message);
+            if (!canAutoupdateApp(app, updateInfo[appId].manifest)) {
+                debug(`app ${app.fqdn} requires manual update`);
                return iteratorDone();
            }

@@ -1841,17 +1846,19 @@ function restoreInstalledApps(callback) {

        async.eachSeries(apps, function (app, iteratorDone) {
            backups.getByAppIdPaged(1, 1, app.id, function (error, results) {
-                let installationState, restoreConfig;
+                let installationState, restoreConfig, oldManifest;
                if (!error && results.length) {
                    installationState = exports.ISTATE_PENDING_RESTORE;
-                    restoreConfig = { backupId: results[0].id, backupFormat: results[0].format, oldManifest: app.manifest };
+                    restoreConfig = { backupId: results[0].id, backupFormat: results[0].format };
+                    oldManifest = app.manifest;
                } else {
                    installationState = exports.ISTATE_PENDING_INSTALL;
                    restoreConfig = null;
+                    oldManifest = null;
                }

                const task = {
-                    args: { restoreConfig, overwriteDns: true },
+                    args: { restoreConfig, overwriteDns: true, oldManifest },
                    values: {},
                    scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                    requireNullTaskId: false // ignore existing stale taskId
@@ -1882,9 +1889,8 @@ function configureInstalledApps(callback) {
        async.eachSeries(apps, function (app, iteratorDone) {
            debug(`configureInstalledApps: marking ${app.fqdn} for reconfigure`);

-            const oldConfig = _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'portBindings');
            const task = {
-                args: { oldConfig, overwriteDns: true },
+                args: {},
                values: {},
                scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                requireNullTaskId: false // ignore existing stale taskId
@@ -1994,7 +2000,8 @@ function uploadFile(appId, sourceFilePath, destFilePath, callback) {

    const done = once(function (error) {
        safe.fs.unlinkSync(sourceFilePath); // remove file in /tmp
-        callback(error);
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, error.message)); // blame it on filesystem for now
+        callback(null);
    });

    // the built-in bash printf understands "%q" but not /usr/bin/printf.
@@ -5,6 +5,9 @@ exports = module.exports = {
    getApp: getApp,
    getAppVersion: getAppVersion,

+    trackBeginSetup: trackBeginSetup,
+    trackFinishedSetup: trackFinishedSetup,
+
    registerWithLoginCredentials: registerWithLoginCredentials,
    registerWithLicense: registerWithLicense,

@@ -375,6 +378,35 @@ function registerCloudron(data, callback) {
    });
 }

+// This works without a Cloudron token as this Cloudron was not yet registered
+let gBeginSetupAlreadyTracked = false;
+function trackBeginSetup(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
+    // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
+    if (gBeginSetupAlreadyTracked) return;
+    gBeginSetupAlreadyTracked = true;
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
+
+    superagent.post(url).send({ provider }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
+// This works without a Cloudron token as this Cloudron was not yet registered
+function trackFinishedSetup(domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;
+
+    superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
 function registerWithLicense(license, domain, callback) {
    assert.strictEqual(typeof license, 'string');
    assert.strictEqual(typeof domain, 'string');
@@ -383,7 +415,10 @@ function registerWithLicense(license, domain, callback) {
    getCloudronToken(function (error, token) {
        if (token) return callback(new BoxError(BoxError.CONFLICT));

-        registerCloudron({ license, domain }, callback);
+        const provider = settings.provider();
+        const version = constants.VERSION;
+
+        registerCloudron({ license, domain, provider, version }, callback);
    });
 }

@@ -406,7 +441,7 @@ function registerWithLoginCredentials(options, callback) {
            login(options.email, options.password, options.totpToken || '', function (error, result) {
                if (error) return callback(error);

-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken }, callback);
+                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, provider: settings.provider(), version: constants.VERSION }, callback);
            });
        });
    });
@@ -64,11 +64,13 @@ function debugApp(app) {
 }

 function makeTaskError(error, app) {
-    let boxError = error instanceof BoxError ? error : new BoxError(BoxError.UNKNOWN_ERROR, error.message); // until we port everything to BoxError
+    assert.strictEqual(typeof error, 'object');
+    assert.strictEqual(typeof app, 'object');
+
    // track a few variables which helps 'repair' restart the task (see also scheduleTask in apps.js)
-    boxError.details.taskId = app.taskId;
-    boxError.details.installationState = app.installationState;
-    return boxError.toPlainObject();
+    error.details.taskId = app.taskId;
+    error.details.installationState = app.installationState;
+    return error.toPlainObject();
 }

 // updates the app object and the database
@@ -459,16 +461,18 @@ function waitForDnsPropagation(app, callback) {
    });
 }

-function moveDataDir(app, sourceDir, callback) {
+function moveDataDir(app, targetDir, callback) {
    assert.strictEqual(typeof app, 'object');
-    assert(sourceDir === null || typeof sourceDir === 'string');
+    assert(targetDir === null || typeof targetDir === 'string');
    assert.strictEqual(typeof callback, 'function');

-    let resolvedSourceDir = apps.getDataDir(app, sourceDir);
-    let resolvedTargetDir = apps.getDataDir(app, app.dataDir);
+    let resolvedSourceDir = apps.getDataDir(app, app.dataDir);
+    let resolvedTargetDir = apps.getDataDir(app, targetDir);

    debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);

+    if (resolvedSourceDir === resolvedTargetDir) return callback();
+
    shell.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}, function (error) {
        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`));

@@ -503,17 +507,6 @@ function startApp(app, callback){
    docker.startContainer(app.id, callback);
 }

-// Ordering is based on the following rationale:
-//   - configure nginx, icon, oauth
-//   - register subdomain.
-//          at this point, the user can visit the site and the above nginx config can show some install screen.
-//          the icon can be displayed in this nginx page and oauth proxy means the page can be protected
-//   - download image
-//   - setup volumes
-//   - setup addons (requires the above volume)
-//   - setup the container (requires image, volumes, addons)
-//   - setup collectd (requires container id)
-// restore is also handled here since restore is just an install with some oldConfig to clean up
 function install(app, args, progressCallback, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
@@ -522,6 +515,7 @@ function install(app, args, progressCallback, callback) {

    const restoreConfig = args.restoreConfig; // has to be set when restoring
    const overwriteDns = args.overwriteDns;
+    const oldManifest = args.oldManifest;

    async.series([
        // this protects against the theoretical possibility of an app being marked for install/restore from
@@ -535,22 +529,25 @@ function install(app, args, progressCallback, callback) {
        function teardownAddons(next) {
            // when restoring, app does not require these addons anymore. remove carefully to preserve the db passwords
            let addonsToRemove;
-            if (restoreConfig && restoreConfig.oldManifest) { // oldManifest is null for clone
-                addonsToRemove = _.omit(restoreConfig.oldManifest.addons, Object.keys(app.manifest.addons));
+            if (oldManifest) {
+                addonsToRemove = _.omit(oldManifest.addons, Object.keys(app.manifest.addons));
            } else {
                addonsToRemove = app.manifest.addons;
            }

            addons.teardownAddons(app, addonsToRemove, next);
        },
-        deleteAppDir.bind(null, app, { removeDirectory: false }), // do not remove any symlinked appdata dir
+
+        function deleteAppDirIfNeeded(done) {
+            if (restoreConfig && !restoreConfig.backupId) return done(); // in-place import should not delete data dir
+
+            deleteAppDir(app, { removeDirectory: false }, done); // do not remove any symlinked appdata dir
+        },

        function deleteImageIfChanged(done) {
-            if (!restoreConfig || !restoreConfig.oldManifest) return done();
+            if (!oldManifest || oldManifest.dockerImage === app.manifest.dockerImage) return done();

-            if (restoreConfig.oldManifest.dockerImage === app.manifest.dockerImage) return done();
-
-            docker.deleteImage(restoreConfig.oldManifest, done);
+            docker.deleteImage(oldManifest, done);
        },

        reserveHttpPort.bind(null, app),
@@ -573,14 +570,22 @@ function install(app, args, progressCallback, callback) {
                    progressCallback.bind(null, { percent: 60, message: 'Setting up addons' }),
                    addons.setupAddons.bind(null, app, app.manifest.addons),
                ], next);
+            } else if (!restoreConfig.backupId) { // in-place import
+                async.series([
+                    progressCallback.bind(null, { percent: 60, message: 'Importing addons in-place' }),
+                    addons.setupAddons.bind(null, app, app.manifest.addons),
+                    addons.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
+                    addons.restoreAddons.bind(null, app, app.manifest.addons),
+                ], next);
            } else {
                async.series([
                    progressCallback.bind(null, { percent: 65, message: 'Download backup and restoring addons' }),
                    addons.setupAddons.bind(null, app, app.manifest.addons),
                    addons.clearAddons.bind(null, app, app.manifest.addons),
-                    backups.restoreApp.bind(null, app, app.manifest.addons, restoreConfig, (progress) => {
-                        progressCallback({ percent: 65, message: `Restore - ${progress.message}` });
-                    })
+                    backups.downloadApp.bind(null, app, restoreConfig, (progress) => {
+                        progressCallback({ percent: 65, message: progress.message });
+                    }),
+                    addons.restoreAddons.bind(null, app, app.manifest.addons)
                ], next);
            }
        },
@@ -624,8 +629,8 @@ function backup(app, args, progressCallback, callback) {
    ], function seriesDone(error) {
        if (error) {
            debugApp(app, 'error backing up app: %s', error);
-            // return to installed state intentionally
-            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: error.toPlainObject ? error.toPlainObject() : error.message }, callback.bind(null, error));
+            // return to installed state intentionally. the error is stashed only in the task and not the app (the UI shows error state otherwise)
+            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }, callback.bind(null, makeTaskError(error, app)));
        }
        callback(null);
    });
@@ -709,7 +714,7 @@ function changeLocation(app, args, progressCallback, callback) {
        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
    ], function seriesDone(error) {
        if (error) {
-            debugApp(app, 'error reconfiguring : %s', error);
+            debugApp(app, 'error changing location : %s', error);
            return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
        }
        callback(null);
@@ -722,8 +727,8 @@ function migrateDataDir(app, args, progressCallback, callback) {
    assert.strictEqual(typeof progressCallback, 'function');
    assert.strictEqual(typeof callback, 'function');

-    let oldDataDir = args.oldDataDir;
-    assert(oldDataDir === null || typeof oldDataDir === 'string');
+    let newDataDir = args.newDataDir;
+    assert(newDataDir === null || typeof newDataDir === 'string');

    async.series([
        progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
@@ -734,69 +739,43 @@ function migrateDataDir(app, args, progressCallback, callback) {

        // re-setup addons since this creates the localStorage volume
        progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        addons.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),

-        // migrate dataDir
-        function (next) {
-            const dataDirChanged = oldDataDir !== app.dataDir;
+        progressCallback.bind(null, { percent: 60, message: 'Moving data dir' }),
+        moveDataDir.bind(null, app, newDataDir),

-            if (!dataDirChanged) return next();
-
-            moveDataDir(app, oldDataDir, next);
-        },
-
-        progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
+        progressCallback.bind(null, { percent: 90, message: 'Creating container' }),
        createContainer.bind(null, app),

        startApp.bind(null, app),

        progressCallback.bind(null, { percent: 100, message: 'Done' }),
-        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
+        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir })
    ], function seriesDone(error) {
        if (error) {
-            debugApp(app, 'error reconfiguring : %s', error);
+            debugApp(app, 'error migrating data dir : %s', error);
            return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
        }
        callback(null);
    });
 }

-// configure is called for an infra update and repair
+// configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local"
 function configure(app, args, progressCallback, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
    assert.strictEqual(typeof progressCallback, 'function');
    assert.strictEqual(typeof callback, 'function');

-    const oldConfig = args.oldConfig || null;
-    const overwriteDns = args.overwriteDns;
-
    async.series([
        progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
        unconfigureReverseProxy.bind(null, app),
        deleteContainers.bind(null, app, { managedOnly: true }),
-        function (next) {
-            if (!oldConfig) return next();
-
-            let obsoleteDomains = oldConfig.alternateDomains.filter(function (o) {
-                return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
-            });
-
-            if (oldConfig.fqdn !== app.fqdn) obsoleteDomains.push({ subdomain: oldConfig.location, domain: oldConfig.domain });
-
-            if (obsoleteDomains.length === 0) return next();
-
-            unregisterSubdomains(app, obsoleteDomains, next);
-        },
-
        reserveHttpPort.bind(null, app),

        progressCallback.bind(null, { percent: 20, message: 'Downloading icon' }),
        downloadIcon.bind(null, app),

-        progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
-        registerSubdomains.bind(null, app, overwriteDns),
-
        progressCallback.bind(null, { percent: 40, message: 'Downloading image' }),
        downloadImage.bind(null, app.manifest),

@@ -1052,7 +1031,7 @@ function run(appId, args, progressCallback, callback) {
            return stop(app, args, progressCallback, callback);
        default:
            debugApp(app, 'apptask launched with invalid command');
-            return callback(new Error('Unknown install command in apptask:' + app.installationState));
+            return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState));
        }
    });
 }
@@ -5,6 +5,7 @@ exports = module.exports = {
 };

 let assert = require('assert'),
+    BoxError = require('./boxerror.js'),
    debug = require('debug')('box:apptaskmanager'),
    fs = require('fs'),
    locker = require('./locker.js'),
@@ -42,7 +43,7 @@ function scheduleTask(appId, taskId, callback) {
    if (!gInitialized) initializeSync();

    if (appId in gActiveTasks) {
-        return callback(new Error(`Task for %s is already active: ${appId}`));
+        return callback(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
    }

    if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
@@ -2,6 +2,7 @@

 exports = module.exports = {
    testConfig: testConfig,
+    testProviderConfig: testProviderConfig,

    getByStatePaged: getByStatePaged,
    getByAppIdPaged: getByAppIdPaged,
@@ -14,7 +15,7 @@ exports = module.exports = {
    restore: restore,

    backupApp: backupApp,
-    restoreApp: restoreApp,
+    downloadApp: downloadApp,

    backupBoxAndApps: backupBoxAndApps,

@@ -118,6 +119,17 @@ function testConfig(backupConfig, callback) {
    api(backupConfig.provider).testConfig(backupConfig, callback);
 }

+
+function testProviderConfig(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var func = api(backupConfig.provider);
+    if (!func) return callback(new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' }));
+
+    api(backupConfig.provider).testConfig(backupConfig, callback);
+}
+
 function getByStatePaged(state, page, perPage, callback) {
    assert.strictEqual(typeof state, 'string');
    assert(typeof page === 'number' && page > 0);
@@ -217,14 +229,14 @@ function createReadStream(sourceFile, key) {

    stream.on('error', function (error) {
        debug('createReadStream: read stream error.', error);
-        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, error.message));
    });

    if (key !== null) {
        var encrypt = crypto.createCipher('aes-256-cbc', key);
        encrypt.on('error', function (error) {
            debug('createReadStream: encrypt stream error.', error);
-            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
        });
        return stream.pipe(encrypt).pipe(ps);
    } else {
@@ -237,17 +249,25 @@ function createWriteStream(destFile, key) {
    assert(key === null || typeof key === 'string');

    var stream = fs.createWriteStream(destFile);
+    var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+
+    stream.on('error', function (error) {
+        debug('createWriteStream: write stream error.', error);
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, error.message));
+    });

    if (key !== null) {
        var decrypt = crypto.createDecipher('aes-256-cbc', key);
        decrypt.on('error', function (error) {
            debug('createWriteStream: decrypt stream error.', error);
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
        });
-        decrypt.pipe(stream);
-        return decrypt;
+        ps.pipe(decrypt).pipe(stream);
    } else {
-        return stream;
+        ps.pipe(stream);
    }
+
+    return ps;
 }

 function tarPack(dataLayout, key, callback) {
@@ -338,7 +358,7 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
                    debug(`read stream error for ${task.path}: ${error.message}`);
                    retryCallback();
                }); // ignore error if file disappears
-                stream.on('progress', function(progress) {
+                stream.on('progress', function (progress) {
                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
                    if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0Mbps looks wrong
                    progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}Mbps` }); // 0M@0Mbps looks wrong
@@ -437,7 +457,7 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback)
                    tarPack(dataLayout, backupConfig.key || null, function (error, tarStream) {
                        if (error) return retryCallback(error);

-                        tarStream.on('progress', function(progress) {
+                        tarStream.on('progress', function (progress) {
                            const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
                            if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0Mbps looks wrong
                            progressCallback({ message: `Uploading backup ${transferred}M@${speed}Mbps` });
@@ -545,24 +565,30 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,

    debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);

-    function downloadFile(entry, callback) {
+    function downloadFile(entry, done) {
        let relativePath = path.relative(backupFilePath, entry.fullPath);
        if (backupConfig.key) {
            relativePath = decryptFilePath(relativePath, backupConfig.key);
-            if (!relativePath) return callback(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file'));
+            if (!relativePath) return done(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file'));
        }
        const destFilePath = dataLayout.toLocalPath('./' + relativePath);

        mkdirp(path.dirname(destFilePath), function (error) {
-            if (error) return callback(new BoxError(BoxError.FS_ERROR, error.message));
+            if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));

            async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
                let destStream = createWriteStream(destFilePath, backupConfig.key || null);

+                destStream.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0Mbps looks wrong
+                    progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}Mbps` });
+                });
+
                // protect against multiple errors. must destroy the write stream so that a previous retry does not write
                let closeAndRetry = once((error) => {
-                    if (error) progressCallback({ message: `Download ${entry.fullPath} errored: ${error.message}` });
-                    else progressCallback({ message: `Download ${entry.fullPath} finished` });
+                    if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                    else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` });
                    destStream.destroy();
                    retryCallback(error);
                });
@@ -571,21 +597,21 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
                    if (error) return closeAndRetry(error);

                    sourceStream.on('error', closeAndRetry);
-                    destStream.on('error', closeAndRetry);
+                    destStream.on('error', closeAndRetry); // already emits BoxError

                    progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });

                    sourceStream.pipe(destStream, { end: true }).on('finish', closeAndRetry);
                });
-            }, callback);
+            }, done);
        });
    }

-    api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, done) {
+    api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
        // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
        const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);

-        async.eachLimit(entries, concurrency, downloadFile, done);
+        async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
    }, callback);
 }

@@ -597,7 +623,7 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback,
    assert.strictEqual(typeof progressCallback, 'function');
    assert.strictEqual(typeof callback, 'function');

-    debug(`download - Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`);
+    debug(`download: Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`);

    const backupFilePath = getBackupFilePath(backupConfig, backupId, format);

@@ -651,9 +677,8 @@ function restore(backupConfig, backupId, progressCallback, callback) {
    });
 }

-function restoreApp(app, addonsToRestore, restoreConfig, progressCallback, callback) {
+function downloadApp(app, restoreConfig, progressCallback, callback) {
    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof addonsToRestore, 'object');
    assert.strictEqual(typeof restoreConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');
    assert.strictEqual(typeof callback, 'function');
@@ -662,30 +687,32 @@ function restoreApp(app, addonsToRestore, restoreConfig, progressCallback, callb
    if (!appDataDir) return callback(safe.error);
    const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);

-    var startTime = new Date();
+    const startTime = new Date();
+    const getBackupConfigFunc = restoreConfig.backupConfig ? (next) => next(null, restoreConfig.backupConfig) : settings.getBackupConfig;

-    settings.getBackupConfig(function (error, backupConfig) {
+    getBackupConfigFunc(function (error, backupConfig) {
        if (error) return callback(error);

-        async.series([
-            download.bind(null, backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback),
-            addons.restoreAddons.bind(null, app, addonsToRestore)
-        ], function (error) {
-            debug('restoreApp: time: %s', (new Date() - startTime)/1000);
+        download(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback, function (error) {
+            debug('downloadApp: time: %s', (new Date() - startTime)/1000);

            callback(error);
        });
    });
 }

-function runBackupUpload(backupId, format, dataLayout, progressCallback, callback) {
-    assert.strictEqual(typeof backupId, 'string');
-    assert.strictEqual(typeof format, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+function runBackupUpload(uploadConfig, progressCallback, callback) {
+    assert.strictEqual(typeof uploadConfig, 'object');
    assert.strictEqual(typeof progressCallback, 'function');
    assert.strictEqual(typeof callback, 'function');

-    let result = '';
+    const { backupId, format, dataLayout, progressTag } = uploadConfig;
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof format, 'string');
+    assert.strictEqual(typeof progressTag, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    let result = ''; // the script communicates error result as a string

    shell.sudo(`backup-${backupId}`, [ BACKUP_UPLOAD_CMD, backupId, format, dataLayout.toString() ], { preserveEnv: true, ipc: true }, function (error) {
        if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
@@ -695,10 +722,10 @@ function runBackupUpload(backupId, format, dataLayout, progressCallback, callbac
        }

        callback();
-    }).on('message', function (message) {
-        if (!message.result) return progressCallback(message);
-        debug(`runBackupUpload: result - ${JSON.stringify(message)}`);
-        result = message.result;
+    }).on('message', function (progress) { // script sends either 'message' or 'result' property
+        if (!progress.result) return progressCallback({ message: `${progress.message} (${progressTag})` });
+        debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
+        result = progress.result;
    });
 }

@@ -752,8 +779,14 @@ function uploadBoxSnapshot(backupConfig, progressCallback, callback) {
        const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
        if (!boxDataDir) return callback(safe.error);

-        const dataLayout = new DataLayout(boxDataDir, []);
-        runBackupUpload('snapshot/box', backupConfig.format, dataLayout, progressCallback, function (error) {
+        const uploadConfig = {
+            backupId: 'snapshot/box',
+            format: backupConfig.format,
+            dataLayout: new DataLayout(boxDataDir, []),
+            progressTag: 'box'
+        };
+
+        runBackupUpload(uploadConfig, progressCallback, function (error) {
            if (error) return callback(error);

            debug('uploadBoxSnapshot: time: %s secs', (new Date() - startTime)/1000);
@@ -782,7 +815,7 @@ function rotateBoxBackup(backupConfig, tag, appBackupIds, progressCallback, call
        if (error) return callback(error);

        var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message }));
+        copy.on('progress', (message) => progressCallback({ message: `box: ${message}` }));
        copy.on('done', function (copyBackupError) {
            const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;

@@ -863,7 +896,7 @@ function rotateAppBackup(backupConfig, app, tag, options, progressCallback, call
        if (error) return callback(error);

        var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message }));
+        copy.on('progress', (message) => progressCallback({ message: `${message} (${app.fqdn})` }));
        copy.on('done', function (copyBackupError) {
            const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;

@@ -898,7 +931,14 @@ function uploadAppSnapshot(backupConfig, app, progressCallback, callback) {

        const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);

-        runBackupUpload(backupId, backupConfig.format, dataLayout, progressCallback, function (error) {
+        const uploadConfig = {
+            backupId,
+            format: backupConfig.format,
+            dataLayout,
+            progressTag: app.fqdn
+        };
+
+        runBackupUpload(uploadConfig, progressCallback, function (error) {
            if (error) return callback(error);

            debugApp(app, 'uploadAppSnapshot: %s done time: %s secs', backupId, (new Date() - startTime)/1000);
@@ -33,6 +33,7 @@ function BoxError(reason, errorOrMessage, details) {
 }
 util.inherits(BoxError, Error);
 BoxError.ACCESS_DENIED = 'Access Denied';
+BoxError.ADDONS_ERROR = 'Addons Error';
 BoxError.ALREADY_EXISTS = 'Already Exists';
 BoxError.BAD_FIELD = 'Bad Field';
 BoxError.BAD_STATE = 'Bad State';
@@ -58,9 +59,10 @@ BoxError.NOT_IMPLEMENTED = 'Not implemented';
 BoxError.NOT_SIGNED = 'Not Signed';
 BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.PLAN_LIMIT = 'Plan Limit';
+BoxError.SPAWN_ERROR = 'Spawn Error';
 BoxError.TASK_ERROR = 'Task Error';
+BoxError.TIMEOUT = 'Timeout';
 BoxError.TRY_AGAIN = 'Try Again';
-BoxError.UNKNOWN_ERROR = 'Unknown Error'; // only used for porting

 BoxError.prototype.toPlainObject = function () {
    return _.extend({}, { message: this.message, reason: this.reason }, this.details);
@@ -86,6 +88,7 @@ BoxError.toHttpError = function (error) {
    case BoxError.FS_ERROR:
    case BoxError.MAIL_ERROR:
    case BoxError.DOCKER_ERROR:
+    case BoxError.ADDONS_ERROR:
        return new HttpError(424, error);
    case BoxError.DATABASE_ERROR:
    case BoxError.INTERNAL_ERROR:
@@ -9,8 +9,8 @@ var assert = require('assert'),
    fs = require('fs'),
    path = require('path'),
    paths = require('../paths.js'),
+    request = require('request'),
    safe = require('safetydance'),
-    superagent = require('superagent'),
    util = require('util'),
    _ = require('underscore');

@@ -41,15 +41,6 @@ function Acme2(options) {
    this.wildcard = !!options.wildcard;
 }

-Acme2.prototype.getNonce = function (callback) {
-    superagent.get(this.directory.newNonce).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(error);
-        if (response.statusCode !== 204) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
-
-        return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
-    });
-};
-
 // urlsafe base64 encoding (jose)
 function urlBase64Encode(string) {
    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
@@ -96,8 +87,12 @@ Acme2.prototype.sendSignedRequest = function (url, payload, callback) {

    var payload64 = b64(payload);

-    this.getNonce(function (error, nonce) {
-        if (error) return callback(error);
+    request.get(this.directory.newNonce, { json: true, timeout: 30000 }, function (error, response) {
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`));
+        if (response.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching nonce : ' + response.statusCode));
+
+        const nonce = response.headers['Replay-Nonce'.toLowerCase()];
+        if (!nonce) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response'));

        debug('sendSignedRequest: using nonce %s for url %s', nonce, url);

@@ -113,14 +108,23 @@ Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
            signature: signature64
        };

-        superagent.post(url).set('Content-Type', 'application/jose+json').set('User-Agent', 'acme-cloudron').send(JSON.stringify(data)).timeout(30 * 1000).end(function (error, res) {
-            if (error && !error.response) return callback(error); // network errors
+        request.post(url, { headers: { 'Content-Type': 'application/jose+json', 'User-Agent': 'acme-cloudron' }, body: JSON.stringify(data), timeout: 30000 }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`)); // network error

-            callback(null, res);
+            // we don't set json: true in request because it ends up mangling the content-type
+            // we don't set json: true in request because it ends up mangling the content-type
+            if (response.headers['content-type'] === 'application/json') response.body = safe.JSON.parse(response.body);
+
+            callback(null, response);
        });
    });
 };

+// https://tools.ietf.org/html/rfc8555#section-6.3
+Acme2.prototype.postAsGet = function (url, callback) {
+    this.sendSignedRequest(url, '', callback);
+};
+
 Acme2.prototype.updateContact = function (registrationUri, callback) {
    assert.strictEqual(typeof registrationUri, 'string');
    assert.strictEqual(typeof callback, 'function');
@@ -134,7 +138,7 @@ Acme2.prototype.updateContact = function (registrationUri, callback) {

    const that = this;
    this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when updating contact: ${error.message}`));
+        if (error) return callback(error);
        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 200, got %s %s', result.statusCode, result.text)));

        debug(`updateContact: contact of user updated to ${that.email}`);
@@ -154,7 +158,7 @@ Acme2.prototype.registerUser = function (callback) {

    var that = this;
    this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when registering user: ${error.message}`));
+        if (error) return callback(error);
        // 200 if already exists. 201 for new accounts
        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register new account. Expecting 200 or 201, got %s %s', result.statusCode, result.text)));

@@ -180,7 +184,7 @@ Acme2.prototype.newOrder = function (domain, callback) {
    debug('newOrder: %s', domain);

    this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when creating new order: ${error.message}`));
+        if (error) return callback(error);
        if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending signed request: ${result.body.detail}`));
        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));

@@ -201,14 +205,15 @@ Acme2.prototype.waitForOrder = function (orderUrl, callback) {
    assert.strictEqual(typeof callback, 'function');

    debug(`waitForOrder: ${orderUrl}`);
+    const that = this;

    async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
        debug('waitForOrder: getting status');

-        superagent.get(orderUrl).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) {
+        that.postAsGet(orderUrl, function (error, result) {
+            if (error) {
                debug('waitForOrder: network error getting uri %s', orderUrl);
-                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for order: ${error.message}`)); // network error
+                return retryCallback(error);
            }
            if (result.statusCode !== 200) {
                debug('waitForOrder: invalid response code getting uri %s', result.statusCode);
@@ -253,7 +258,7 @@ Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
    };

    this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when notifying challenge: ${error.message}`));
+        if (error) return callback(error);
        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 200, got %s %s', result.statusCode, result.text)));

        callback();
@@ -265,14 +270,15 @@ Acme2.prototype.waitForChallenge = function (challenge, callback) {
    assert.strictEqual(typeof callback, 'function');

    debug('waitingForChallenge: %j', challenge);
+    const that = this;

    async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
        debug('waitingForChallenge: getting status');

-        superagent.get(challenge.url).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) {
+        that.postAsGet(challenge.url, function (error, result) {
+            if (error) {
                debug('waitForChallenge: network error getting uri %s', challenge.url);
-                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for challenge: ${error.message}`));
+                return retryCallback(error);
            }
            if (result.statusCode !== 200) {
                debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
@@ -305,7 +311,7 @@ Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, cal
    debug('signCertificate: sending sign request');

    this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when signing certificate: ${error.message}`));
+        if (error) return callback(error);
        // 429 means we reached the cert limit for this domain
        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 200, got %s %s', result.statusCode, result.text)));

@@ -348,20 +354,17 @@ Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
    assert.strictEqual(typeof callback, 'function');

    var outdir = paths.APP_CERTS_DIR;
+    const that = this;

    async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
        debug('downloadCertificate: downloading certificate');

-        superagent.get(certUrl).buffer().parse(function (res, done) {
-            var data = [ ];
-            res.on('data', function(chunk) { data.push(chunk); });
-            res.on('end', function () { res.text = Buffer.concat(data); done(); });
-        }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
-            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry'));
+        that.postAsGet(certUrl, function (error, result) {
+            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
+            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));

-            const fullChainPem = result.text;
+            const fullChainPem = result.body; // buffer

            const certName = hostname.replace('*.', '_.');
            var certificateFile = path.join(outdir, `${certName}.cert`);
@@ -488,8 +491,8 @@ Acme2.prototype.prepareChallenge = function (hostname, domain, authorizationUrl,
    debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);

    const that = this;
-    superagent.get(authorizationUrl).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when preparing challenge: ${error.message}`));
+    this.postAsGet(authorizationUrl, function (error, response) {
+        if (error) return callback(error);
        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code getting authorization : ' + response.statusCode));

        const authorization = response.body;
@@ -569,13 +572,13 @@ Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
 Acme2.prototype.getDirectory = function (callback) {
    const that = this;

-    superagent.get(this.caDirectory).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
+    request.get(this.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));

        if (typeof response.body.newNonce !== 'string' ||
            typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') return callback(new Error(`Invalid response body : ${response.body}`));
+            typeof response.body.newAccount !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));

        that.directory = response.body;

@@ -10,7 +10,8 @@ exports = module.exports = {
    getCertificate: getCertificate
 };

-var assert = require('assert');
+var assert = require('assert'),
+    BoxError = require('../boxerror.js');

 function getCertificate(hostname, domain, options, callback) {
    assert.strictEqual(typeof hostname, 'string');
@@ -18,6 +19,6 @@ function getCertificate(hostname, domain, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    return callback(new Error('Not implemented'));
+    return callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'getCertificate is not implemented'));
 }

@@ -167,7 +167,7 @@ function runSystemChecks() {
 function checkBackupConfiguration(callback) {
    assert.strictEqual(typeof callback, 'function');

-    debug('Checking backup configuration');
+    debug('checking backup configuration');

    backups.checkConfiguration(function (error, message) {
        if (error) return callback(error);
@@ -18,12 +18,12 @@ var appHealthMonitor = require('./apphealthmonitor.js'),
    constants = require('./constants.js'),
    CronJob = require('cron').CronJob,
    debug = require('debug')('box:cron'),
-    disks = require('./disks.js'),
    dyndns = require('./dyndns.js'),
    eventlog = require('./eventlog.js'),
    janitor = require('./janitor.js'),
    scheduler = require('./scheduler.js'),
    settings = require('./settings.js'),
+    system = require('./system.js'),
    updater = require('./updater.js'),
    updateChecker = require('./updatechecker.js');

@@ -116,7 +116,7 @@ function recreateJobs(tz) {
    if (gJobs.diskSpaceChecker) gJobs.diskSpaceChecker.stop();
    gJobs.diskSpaceChecker = new CronJob({
        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: () => disks.checkDiskSpace(NOOP_CALLBACK),
+        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
        start: true,
        runOnInit: true, // run system check immediately
        timeZone: tz
@@ -14,6 +14,7 @@ exports = module.exports = {

 var assert = require('assert'),
    async = require('async'),
+    BoxError = require('./boxerror.js'),
    child_process = require('child_process'),
    constants = require('./constants.js'),
    mysql = require('mysql'),
@@ -112,7 +113,7 @@ function clear(callback) {
 function beginTransaction(callback) {
    assert.strictEqual(typeof callback, 'function');

-    if (gConnectionPool === null) return callback(new Error('No database connection pool.'));
+    if (gConnectionPool === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No database connection pool.'));

    gConnectionPool.getConnection(function (error, connection) {
        if (error) {
@@ -156,7 +157,7 @@ function query() {
    var callback = args[args.length - 1];
    assert.strictEqual(typeof callback, 'function');

-    if (gDefaultConnection === null) return callback(new Error('No connection to database'));
+    if (gDefaultConnection === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No connection to database'));

    args[args.length -1 ] = function (error, result) {
        if (error && error.fatal) {
@@ -126,7 +126,7 @@ function del(domainObject, location, type, values, callback) {

    debug(`get: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);

-    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, new Error('Record deletion is not supported by GoDaddy API')));
+    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Record deletion is not supported by GoDaddy API'));

    // check if the record exists at all so that we don't insert the "Dead" record for no reason
    get(domainObject, location, type, function (error, values) {
@@ -17,6 +17,7 @@ exports = module.exports = {
 };

 var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
    util = require('util');

 function removePrivateFields(domainObject) {
@@ -24,6 +25,7 @@ function removePrivateFields(domainObject) {
    return domainObject;
 }

+// eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
    // in-place injection of tokens and api keys which came in with domains.SECRET_PLACEHOLDER
 }
@@ -37,7 +39,7 @@ function upsert(domainObject, location, type, values, callback) {

    // Result: none

-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upsert is not implemented'));
 }

 function get(domainObject, location, type, callback) {
@@ -48,7 +50,7 @@ function get(domainObject, location, type, callback) {

    // Result: Array of matching DNS records in string format

-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'get is not implemented'));
 }

 function del(domainObject, location, type, values, callback) {
@@ -60,7 +62,7 @@ function del(domainObject, location, type, values, callback) {

    // Result: none

-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'del is not implemented'));
 }

 function wait(domainObject, location, type, value, options, callback) {
@@ -80,5 +82,5 @@ function verifyDnsConfig(domainObject, callback) {

    // Result: dnsConfig object

-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'verifyDnsConfig is not implemented'));
 }
@@ -1,8 +1,6 @@
 'use strict';

 exports = module.exports = {
-    connection: connectionInstance(),
-
    testRegistryConfig: testRegistryConfig,
    setRegistryConfig: setRegistryConfig,
    injectPrivateFields: injectPrivateFields,
@@ -27,6 +25,7 @@ exports = module.exports = {
    getContainerIdByIp: getContainerIdByIp,
    inspect: inspect,
    inspectByName: inspect,
+    execContainer: execContainer,
    getEvents: getEvents,
    memoryUsage: memoryUsage,
    createVolume: createVolume,
@@ -34,20 +33,14 @@ exports = module.exports = {
    clearVolume: clearVolume
 };

-// timeout is optional
-function connectionInstance(timeout) {
-    var Docker = require('dockerode');
-    var docker = new Docker({ socketPath: '/var/run/docker.sock', timeout: timeout });
-    return docker;
-}
-
 var addons = require('./addons.js'),
    async = require('async'),
    assert = require('assert'),
    BoxError = require('./boxerror.js'),
    child_process = require('child_process'),
    constants = require('./constants.js'),
-    debug = require('debug')('box:docker.js'),
+    debug = require('debug')('box:docker'),
+    Docker = require('dockerode'),
    path = require('path'),
    settings = require('./settings.js'),
    shell = require('./shell.js'),
@@ -58,6 +51,9 @@ var addons = require('./addons.js'),
 const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
    MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');

+const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
+const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
+
 function debugApp(app) {
    assert(typeof app === 'object');

@@ -68,8 +64,7 @@ function testRegistryConfig(auth, callback) {
    assert.strictEqual(typeof auth, 'object');
    assert.strictEqual(typeof callback, 'function');

-    let docker = exports.connection;
-    docker.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
+    gConnection.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
        if (error) return callback(new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' }));

        callback();
@@ -108,9 +103,9 @@ function ping(callback) {
    assert.strictEqual(typeof callback, 'function');

    // do not let the request linger
-    var docker = connectionInstance(1000);
+    const connection = new Docker({ socketPath: DOCKER_SOCKET_PATH, timeout: 1000 });

-    docker.ping(function (error, result) {
+    connection.ping(function (error, result) {
        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
        if (result !== 'OK') return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon'));

@@ -140,14 +135,12 @@ function getRegistryConfig(image, callback) {
 }

 function pullImage(manifest, callback) {
-    var docker = exports.connection;
-
    getRegistryConfig(manifest.dockerImage, function (error, authConfig) {
        if (error) return callback(error);

        debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${authConfig ? 'yes' : 'no'}`);

-        docker.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
+        gConnection.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to pull image. Please check the network or if the image needs authentication. statusCode: ' + error.statusCode));

            // https://github.com/dotcloud/docker/issues/1074 says each status message
@@ -203,8 +196,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var docker = exports.connection,
-        isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers
+    let isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers

    var manifest = app.manifest;
    var exposedPorts = {}, dockerPortBindings = { };
@@ -330,7 +322,7 @@ function createSubcontainer(app, name, cmd, options, callback) {

        debugApp(app, 'Creating container for %s', app.manifest.dockerImage);

-        docker.createContainer(containerOptions, function (error, container) {
+        gConnection.createContainer(containerOptions, function (error, container) {
            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

            callback(null, container);
@@ -346,9 +338,7 @@ function startContainer(containerId, callback) {
    assert.strictEqual(typeof containerId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    var docker = exports.connection;
-
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
    debug('Starting container %s', containerId);

    container.start(function (error) {
@@ -369,8 +359,7 @@ function stopContainer(containerId, callback) {
        return callback();
    }

-    var docker = exports.connection;
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
    debug('Stopping container %s', containerId);

    var options = {
@@ -400,8 +389,7 @@ function deleteContainer(containerId, callback) {

    if (containerId === null) return callback(null);

-    var docker = exports.connection;
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);

    var removeOptions = {
        force: true, // kill container if it's running
@@ -425,14 +413,12 @@ function deleteContainers(appId, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var docker = exports.connection;
-
    debug('deleting containers of %s', appId);

    let labels = [ 'appId=' + appId ];
    if (options.managedOnly) labels.push('isCloudronManaged=true');

-    docker.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) {
+    gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) {
        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

        async.eachSeries(containers, function (container, iteratorDone) {
@@ -445,11 +431,9 @@ function stopContainers(appId, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    var docker = exports.connection;
-
    debug('stopping containers of %s', appId);

-    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+    gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

        async.eachSeries(containers, function (container, iteratorDone) {
@@ -465,8 +449,6 @@ function deleteImage(manifest, callback) {
    var dockerImage = manifest ? manifest.dockerImage : null;
    if (!dockerImage) return callback(null);

-    var docker = exports.connection;
-
    var removeOptions = {
        force: false, // might be shared with another instance of this app
        noprune: false // delete untagged parents
@@ -475,7 +457,7 @@ function deleteImage(manifest, callback) {
    // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
    // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
    // not created anymore after https://github.com/docker/docker/pull/10571
-    docker.getImage(dockerImage).remove(removeOptions, function (error) {
+    gConnection.getImage(dockerImage).remove(removeOptions, function (error) {
        if (error && error.statusCode === 400) return callback(null); // invalid image format. this can happen if user installed with a bad --docker-image
        if (error && error.statusCode === 404) return callback(null); // not found
        if (error && error.statusCode === 409) return callback(null); // another container using the image
@@ -493,10 +475,8 @@ function getContainerIdByIp(ip, callback) {
    assert.strictEqual(typeof ip, 'string');
    assert.strictEqual(typeof callback, 'function');

-    var docker = exports.connection;
-
-    docker.getNetwork('cloudron').inspect(function (error, bridge) {
-        if (error && error.statusCode === 404) return callback(new Error('Unable to find the cloudron network'));
+    gConnection.getNetwork('cloudron').inspect(function (error, bridge) {
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to find the cloudron network'));
        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

        var containerId;
@@ -506,7 +486,7 @@ function getContainerIdByIp(ip, callback) {
                break;
            }
        }
-        if (!containerId) return callback(new Error('No container with that ip'));
+        if (!containerId) return callback(new BoxError(BoxError.DOCKER_ERROR, 'No container with that ip'));

        callback(null, containerId);
    });
@@ -516,7 +496,7 @@ function inspect(containerId, callback) {
    assert.strictEqual(typeof containerId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    var container = exports.connection.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);

    container.inspect(function (error, result) {
        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -526,13 +506,38 @@ function inspect(containerId, callback) {
    });
 }

+function execContainer(containerId, options, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = gConnection.getContainer(containerId);
+
+    container.exec(options.execOptions, function (error, exec) {
+        if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running
+        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+
+        exec.start(options.startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+
+            if (options.rows && options.columns) {
+                // there is a race where resizing too early results in a 404 "no such exec"
+                // https://git.cloudron.io/cloudron/box/issues/549
+                setTimeout(function () {
+                    exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
+                }, 2000);
+            }
+
+            callback(null, stream);
+        });
+    });
+}
+
 function getEvents(options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    let docker = exports.connection;
-
-    docker.getEvents(options, function (error, stream) {
+    gConnection.getEvents(options, function (error, stream) {
        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

        callback(null, stream);
@@ -543,7 +548,7 @@ function memoryUsage(containerId, callback) {
    assert.strictEqual(typeof containerId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    var container = exports.connection.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);

    container.stats({ stream: false }, function (error, result) {
        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -559,8 +564,6 @@ function createVolume(app, name, volumeDataDir, callback) {
    assert.strictEqual(typeof volumeDataDir, 'string');
    assert.strictEqual(typeof callback, 'function');

-    let docker = exports.connection;
-
    const volumeOptions = {
        Name: name,
        Driver: 'local',
@@ -577,9 +580,9 @@ function createVolume(app, name, volumeDataDir, callback) {

    // requires sudo because the path can be outside appsdata
    shell.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}, function (error) {
-        if (error) return callback(new Error(`Error creating app data dir: ${error.message}`));
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`));

-        docker.createVolume(volumeOptions, function (error) {
+        gConnection.createVolume(volumeOptions, function (error) {
            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

            callback();
@@ -593,8 +596,7 @@ function clearVolume(app, name, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    let docker = exports.connection;
-    let volume = docker.getVolume(name);
+    let volume = gConnection.getVolume(name);
    volume.inspect(function (error, v) {
        if (error && error.statusCode === 404) return callback();
        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
@@ -614,9 +616,7 @@ function removeVolume(app, name, callback) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof callback, 'function');

-    let docker = exports.connection;
-
-    let volume = docker.getVolume(name);
+    let volume = gConnection.getVolume(name);
    volume.remove(function (error) {
        if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume of ${app.id} ${error.message}`));

@@ -627,9 +627,7 @@ function removeVolume(app, name, callback) {
 function info(callback) {
    assert.strictEqual(typeof callback, 'function');

-    let docker = exports.connection;
-
-    docker.info(function (error, result) {
+    gConnection.info(function (error, result) {
        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker'));

        callback(null, result);
@@ -3,8 +3,9 @@
 var assert = require('assert'),
    async = require('async'),
    authcodedb = require('./authcodedb.js'),
+    BoxError = require('./boxerror.js'),
    debug = require('debug')('box:janitor'),
-    docker = require('./docker.js').connection,
+    Docker = require('dockerode'),
    tokendb = require('./tokendb.js');

 exports = module.exports = {
@@ -12,7 +13,9 @@ exports = module.exports = {
    cleanupDockerVolumes: cleanupDockerVolumes
 };

-var NOOP_CALLBACK = function () { };
+const NOOP_CALLBACK = function () { };
+
+const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });

 function ignoreError(func) {
    return function (callback) {
@@ -67,16 +70,16 @@ function cleanupTmpVolume(containerInfo, callback) {

    debug('cleanupTmpVolume %j', containerInfo.Names);

-    docker.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
-        if (error) return callback(new Error('Failed to exec container : ' + error.message));
+    gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
+        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`));

        execContainer.start({ hijack: true }, function (error, stream) {
-            if (error) return callback(new Error('Failed to start exec container : ' + error.message));
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to start exec container: ${error.message}`));

            stream.on('error', callback);
            stream.on('end', callback);

-            docker.modem.demuxStream(stream, process.stdout, process.stderr);
+            gConnection.modem.demuxStream(stream, process.stdout, process.stderr);
        });
    });
 }
@@ -88,7 +91,7 @@ function cleanupDockerVolumes(callback) {

    debug('Cleaning up docker volumes');

-    docker.listContainers({ all: 0 }, function (error, containers) {
+    gConnection.listContainers({ all: 0 }, function (error, containers) {
        if (error) return callback(error);

        async.eachSeries(containers, function (container, iteratorDone) {
@@ -1,6 +1,7 @@
 'use strict';

 var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
    debug = require('debug')('box:locker'),
    EventEmitter = require('events').EventEmitter,
    util = require('util');
@@ -23,7 +24,7 @@ Locker.prototype.lock = function (operation) {
    assert.strictEqual(typeof operation, 'string');

    if (this._operation !== null) {
-        let error = new Error(`Locked for ${this._operation}`);
+        let error = new BoxError(BoxError.CONFLICT, `Locked for ${this._operation}`);
        error.operation = this._operation;
        return error;
    }
@@ -54,7 +55,7 @@ Locker.prototype.recursiveLock = function (operation) {
 Locker.prototype.unlock = function (operation) {
    assert.strictEqual(typeof operation, 'string');

-    if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
+    if (this._operation !== operation) throw BoxError(BoxError.BAD_STATE, 'Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error

    if (--this._lockDepth === 0) {
        debug('Released : %s', this._operation);
@@ -123,13 +123,13 @@ function checkOutboundPort25(callback) {
        relay.status = false;
        relay.value = `Connect to ${smtpServer} timed out. Check if port 25 (outbound) is blocked`;
        client.destroy();
-        callback(new Error('Timeout'), relay);
+        callback(new BoxError(BoxError.TIMEOUT, `Connect to ${smtpServer} timed out.`), relay);
    });
    client.on('error', function (error) {
        relay.status = false;
        relay.value = `Connect to ${smtpServer} failed: ${error.message}. Check if port 25 (outbound) is blocked`;
        client.destroy();
-        callback(error, relay);
+        callback(new BoxError(BoxError.NETWORK_ERROR, `Connect to ${smtpServer} failed.`), relay);
    });
 }

@@ -199,7 +199,7 @@ function checkDkim(mailDomain, callback) {
    };

    var dkimKey = readDkimPublicKeySync(domain);
-    if (!dkimKey) return callback(new Error('Failed to read dkim public key'), dkim);
+    if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, `Failed to read dkim public key of ${domain}`));

    dkim.expected = 'v=DKIM1; t=s; p=' + dkimKey;

@@ -567,12 +567,12 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
        // mail_domain is used for SRS
        if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
            `mail_in_domains=${mailInDomains}\nmail_out_domains=${mailOutDomains}\nmail_server_name=${mailFqdn}\nmail_domain=${mailDomain}\n\n`, 'utf8')) {
-            return callback(new Error('Could not create mail var file:' + safe.error.message));
+            return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
        }

        // enable_outbound makes plugin forward email for relayed mail. non-relayed mail always hits LMTP plugin first
        if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/smtp_forward.ini'), 'enable_outbound=false\ndomain_selector=mail_from\n', 'utf8')) {
-            return callback(new Error('Could not create smtp forward file:' + safe.error.message));
+            return callback(new BoxError(BoxError.FS_ERROR, 'Could not create smtp forward file:' + safe.error.message));
        }

        // create sections for per-domain configuration
@@ -582,7 +582,7 @@ function createMailConfig(mailFqdn, mailDomain, callback) {

            if (!safe.fs.appendFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
                `[${domain.domain}]\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\n\n`, 'utf8')) {
-                return callback(new Error('Could not create mail var file:' + safe.error.message));
+                return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
            }

            const relay = domain.relay;
@@ -598,7 +598,7 @@ function createMailConfig(mailFqdn, mailDomain, callback) {

            if (!safe.fs.appendFileSync(paths.ADDON_CONFIG_DIR + '/mail/smtp_forward.ini',
                `[${domain.domain}]\nenable_outbound=true\nhost=${host}\nport=${port}\nenable_tls=true\nauth_type=${authType}\nauth_user=${username}\nauth_pass=${password}\n\n`, 'utf8')) {
-                return callback(new Error('Could not create mail var file:' + safe.error.message));
+                return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
            }
        });

@@ -627,8 +627,8 @@ function configureMail(mailFqdn, mailDomain, callback) {
        const mailCertFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_cert.pem');
        const mailKeyFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_key.pem');

-        if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new Error('Could not create cert file:' + safe.error.message));
-        if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new Error('Could not create key file:' + safe.error.message));
+        if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create cert file:' + safe.error.message));
+        if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create key file:' + safe.error.message));

        shell.exec('startMail', 'docker rm -f mail || true', function (error) {
            if (error) return callback(error);
@@ -846,7 +846,7 @@ function upsertDnsRecords(domain, mailFqdn, callback) {
        if (process.env.BOX_ENV === 'test') return callback();

        var dkimKey = readDkimPublicKeySync(domain);
-        if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, new Error('Failed to read dkim public key')));
+        if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, 'Failed to read dkim public key'));

        // t=s limits the domainkey to this domain and not it's subdomains
        var dkimRecord = { subdomain: `${mailDomain.dkimSelector}._domainkey`, domain: domain, type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
@@ -1264,7 +1264,7 @@ function removeList(name, domain, auditSource, callback) {
    mailboxdb.del(name, domain, function (error) {
        if (error) return callback(error);

-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });

        callback();
    });
@@ -44,6 +44,7 @@ exports = module.exports = {

    // this is not part of appdata because an icon may be set before install
    APP_ICONS_DIR: path.join(baseDir(), 'boxdata/appicons'),
+    PROFILE_ICONS_DIR: path.join(baseDir(), 'boxdata/profileicons'),
    MAIL_DATA_DIR: path.join(baseDir(), 'boxdata/mail'),
    ACME_ACCOUNT_KEY_FILE: path.join(baseDir(), 'boxdata/acme/acme.key'),
    APP_CERTS_DIR: path.join(baseDir(), 'boxdata/certs'),
@@ -171,7 +171,7 @@ function reload(callback) {
    if (process.env.BOX_ENV === 'test') return callback();

    shell.sudo('reload', [ RELOAD_NGINX_CMD ], {}, function (error) {
-        if (error) return callback(new BoxError(BoxError.NGINX_ERROR, error));
+        if (error) return callback(new BoxError(BoxError.NGINX_ERROR, `Error reloading nginx: ${error.message}`));

        callback();
    });
@@ -346,7 +346,7 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
                debug('ensureCertificate: getting certificate for %s with options %j', vhost, apiOptions);

                api.getCertificate(vhost, domain, apiOptions, function (error, certFilePath, keyFilePath) {
-                    debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath}`);
+                    debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath || 'null'}`);

                    eventlog.add(currentBundle ? eventlog.ACTION_CERTIFICATE_RENEWAL : eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: vhost, errorMessage: error ? error.message : '' });

@@ -39,7 +39,7 @@ function initialize(callback) {
    });


-    // used when username/password is sent in request body. used in CLI tool login route
+    // used when username/password is sent in request body. used in CLI login & oauth2 login route
    passport.use(new LocalStrategy(function (username, password, callback) {

        // TODO we should only do this for dashboard logins
@@ -349,19 +349,10 @@ function repairApp(req, res, next) {

    const data = req.body;

-    if (data.backupId && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
-    if (data.backupFormat && typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string or null'));
-
-    if (data.location && typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (data.domain && typeof data.domain !== 'string') return next(new HttpError(400, 'domain is required'));
-
-    if ('alternateDomains' in data) {
-        if (!Array.isArray(data.alternateDomains)) return next(new HttpError(400, 'alternateDomains must be an array'));
-        if (data.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
+    if ('manifest' in data) {
+        if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'backupId must be an object'));
    }

-    if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
-
    apps.repair(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
        if (error) return next(BoxError.toHttpError(error));

@@ -377,10 +368,9 @@ function restoreApp(req, res, next) {

    debug('Restore app id:%s', req.params.id);

-    if (!('backupId' in req.body)) return next(new HttpError(400, 'backupId is required'));
-    if (data.backupId !== null && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
+    if (!data.backupId || typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be non-empty string'));

-    apps.restore(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
+    apps.restore(req.params.id, data.backupId, auditSource.fromRequest(req), function (error, result) {
        if (error) return next(BoxError.toHttpError(error));

        next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -395,8 +385,23 @@ function importApp(req, res, next) {

    debug('Importing app id:%s', req.params.id);

-    if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string'));
-    if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
+    if ('backupId' in data) { // if not provided, we import in-place
+        if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string'));
+        if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
+
+        if ('backupConfig' in data && typeof data.backupConfig !== 'object') return next(new HttpError(400, 'backupConfig must be an object'));
+
+        const backupConfig = req.body.backupConfig;
+
+        if (req.body.backupConfig) {
+            if (typeof backupConfig.provider !== 'string') return next(new HttpError(400, 'provider is required'));
+            if ('key' in backupConfig && typeof backupConfig.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+            if ('acceptSelfSignedCerts' in backupConfig && typeof backupConfig.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
+
+            // testing backup config can take sometime
+            req.clearTimeout();
+        }
+    }

    apps.importApp(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
        if (error) return next(BoxError.toHttpError(error));
@@ -5,6 +5,7 @@ exports = module.exports = {
    isRebootRequired: isRebootRequired,
    getConfig: getConfig,
    getDisks: getDisks,
+    getMemory: getMemory,
    getUpdateInfo: getUpdateInfo,
    update: update,
    checkForUpdates: checkForUpdates,
@@ -23,11 +24,11 @@ let assert = require('assert'),
    BoxError = require('../boxerror.js'),
    cloudron = require('../cloudron.js'),
    custom = require('../custom.js'),
-    disks = require('../disks.js'),
    externalLdap = require('../externalldap.js'),
    HttpError = require('connect-lastmile').HttpError,
    HttpSuccess = require('connect-lastmile').HttpSuccess,
    sysinfo = require('../sysinfo.js'),
+    system = require('../system.js'),
    updater = require('../updater.js'),
    updateChecker = require('../updatechecker.js');

@@ -55,7 +56,15 @@ function getConfig(req, res, next) {
 }

 function getDisks(req, res, next) {
-    disks.getDisks(function (error, result) {
+    system.getDisks(function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function getMemory(req, res, next) {
+    system.getMemory(function (error, result) {
        if (error) return next(BoxError.toHttpError(error));

        next(new HttpSuccess(200, result));
@@ -69,7 +69,7 @@ function add(req, res, next) {
    domains.add(req.body.domain, data, auditSource.fromRequest(req), function (error) {
        if (error) return next(BoxError.toHttpError(error));

-        next(new HttpSuccess(201, { domain: req.body.domain, config: req.body.config }));
+        next(new HttpSuccess(201, {}));
    });
 }

@@ -3,6 +3,9 @@
 exports = module.exports = {
    get: get,
    update: update,
+    getAvatar: getAvatar,
+    setAvatar: setAvatar,
+    clearAvatar: clearAvatar,
    changePassword: changePassword,
    setTwoFactorAuthenticationSecret: setTwoFactorAuthenticationSecret,
    enableTwoFactorAuthentication: enableTwoFactorAuthentication,
@@ -12,14 +15,21 @@ exports = module.exports = {
 var assert = require('assert'),
    auditSource = require('../auditsource.js'),
    BoxError = require('../boxerror.js'),
+    fs = require('fs'),
    HttpError = require('connect-lastmile').HttpError,
    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    path = require('path'),
+    paths = require('../paths.js'),
+    safe = require('safetydance'),
    users = require('../users.js'),
+    settings = require('../settings.js'),
    _ = require('underscore');

 function get(req, res, next) {
    assert.strictEqual(typeof req.user, 'object');

+    const emailHash = require('crypto').createHash('md5').update(req.user.email).digest('hex');
+
    next(new HttpSuccess(200, {
        id: req.user.id,
        username: req.user.username,
@@ -28,7 +38,8 @@ function get(req, res, next) {
        displayName: req.user.displayName,
        twoFactorAuthenticationEnabled: req.user.twoFactorAuthenticationEnabled,
        admin: req.user.admin,
-        source: req.user.source
+        source: req.user.source,
+        avatarUrl: fs.existsSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id)) ? `${settings.adminOrigin()}/api/v1/profile/avatar/${req.user.id}` : `https://www.gravatar.com/avatar/${emailHash}.jpg?s=128&d=mm`
    }));
 }

@@ -49,6 +60,28 @@ function update(req, res, next) {
    });
 }

+function setAvatar(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    if (!req.files.avatar) return next(new HttpError(400, 'avatar is missing'));
+
+    if (!safe.fs.renameSync(req.files.avatar.path, path.join(paths.PROFILE_ICONS_DIR, req.user.id))) return next(new HttpError(500, safe.error));
+
+    next(new HttpSuccess(202, {}));
+}
+
+function clearAvatar(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    safe.fs.unlinkSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id));
+
+    next(new HttpSuccess(202, {}));
+}
+
+function getAvatar(req, res) {
+    res.sendFile(path.join(paths.PROFILE_ICONS_DIR, req.params.identifier));
+}
+
 function changePassword(req, res, next) {
    assert.strictEqual(typeof req.body, 'object');
    assert.strictEqual(typeof req.user, 'object');
@@ -9,14 +9,15 @@ exports = module.exports = {
 };

 var assert = require('assert'),
+    appstore = require('../appstore.js'),
    auditSource = require('../auditsource.js'),
    BoxError = require('../boxerror.js'),
    debug = require('debug')('box:routes/setup'),
    HttpError = require('connect-lastmile').HttpError,
    HttpSuccess = require('connect-lastmile').HttpSuccess,
    provision = require('../provision.js'),
-    settings = require('../settings.js'),
-    superagent = require('superagent');
+    request = require('request'),
+    settings = require('../settings.js');

 function providerTokenAuth(req, res, next) {
    assert.strictEqual(typeof req.body, 'object');
@@ -24,11 +25,11 @@ function providerTokenAuth(req, res, next) {
    if (settings.provider() === 'ami') {
        if (typeof req.body.providerToken !== 'string' || !req.body.providerToken) return next(new HttpError(400, 'providerToken must be a non empty string'));

-        superagent.get('http://169.254.169.254/latest/meta-data/instance-id').timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return next(new HttpError(500, error));
-            if (result.statusCode !== 200) return next(new HttpError(500, 'Unable to get meta data'));
+        request.get('http://169.254.169.254/latest/meta-data/instance-id', { timeout: 30 * 1000 }, function (error, result) {
+            if (error) return next(new HttpError(422, `Network error getting EC2 metadata: ${error.message}`));
+            if (result.statusCode !== 200) return next(new HttpError(422, `Unable to get EC2 meta data. statusCode: ${result.statusCode}`));

-            if (result.text !== req.body.providerToken) return next(new HttpError(401, 'Invalid providerToken'));
+            if (result.body !== req.body.providerToken) return next(new HttpError(422, 'Instance ID does not match'));

            next();
        });
@@ -62,6 +63,8 @@ function setup(req, res, next) {
        if (error) return next(BoxError.toHttpError(error));

        next(new HttpSuccess(200, {}));
+
+        appstore.trackFinishedSetup(dnsConfig.domain);
    });
 }

@@ -116,5 +119,10 @@ function getStatus(req, res, next) {
        if (error) return next(BoxError.toHttpError(error));

        next(new HttpSuccess(200, status));
+
+        // check if Cloudron is not in setup state nor activated and let appstore know of the attempt
+        if (!status.activated && !status.setup.active && !status.restore.active) {
+            appstore.trackBeginSetup(status.provider);
+        }
    });
 }
@@ -11,7 +11,7 @@ let apps = require('../../apps.js'),
    constants = require('../../constants.js'),
    crypto = require('crypto'),
    database = require('../../database.js'),
-    docker = require('../../docker.js').connection,
+    Docker = require('dockerode'),
    expect = require('expect.js'),
    fs = require('fs'),
    hat = require('../../hat.js'),
@@ -33,6 +33,8 @@ let apps = require('../../apps.js'),

 var SERVER_URL = 'http://localhost:' + constants.PORT;

+const docker = new Docker({ socketPath: '/var/run/docker.sock' });
+
 // Test image information
 var TEST_IMAGE_REPO = 'cloudron/test';
 var TEST_IMAGE_TAG = '25.19.0';
@@ -49,7 +49,16 @@ function initializeExpressSync() {
    app.enable('trust proxy');

    if (process.env.BOX_ENV !== 'test') {
-        app.use(middleware.morgan('Box :method :url :status :response-time ms - :res[content-length]', {
+        app.use(middleware.morgan(function (tokens, req, res) {
+            return [
+                'Box',
+                tokens.method(req, res),
+                tokens.url(req, res).replace(/(access_token=)[^\&]+/, '$1' + '<redacted>'),
+                tokens.status(req, res),
+                tokens['response-time'](req, res), 'ms', '-',
+                tokens.res(req, res, 'content-length')
+            ].join(' ');
+        }, {
            immediate: false,
            // only log failed requests by default
            skip: function (req, res) { return res.statusCode < 400; }
@@ -95,7 +104,7 @@ function initializeExpressSync() {
        .use(router)
        .use(middleware.lastMile());

-    // NOTE: these limits have to be in sync with nginx limits
+    // NOTE: routes that use multi-part have to be whitelisted in the reverse proxy
    var FILE_SIZE_LIMIT = '256mb', // max file size that can be uploaded (see also client_max_body_size in nginx)
        FILE_TIMEOUT = 60 * 1000; // increased timeout for file uploads (1 min)

@@ -144,6 +153,7 @@ function initializeExpressSync() {
    router.post('/api/v1/cloudron/reboot', cloudronScope, routes.cloudron.reboot);
    router.get ('/api/v1/cloudron/graphs', cloudronScope, routes.graphs.getGraphs);
    router.get ('/api/v1/cloudron/disks', cloudronScope, routes.cloudron.getDisks);
+    router.get ('/api/v1/cloudron/memory', cloudronScope, routes.cloudron.getMemory);
    router.get ('/api/v1/cloudron/logs/:unit', cloudronScope, routes.cloudron.getLogs);
    router.get ('/api/v1/cloudron/logstream/:unit', cloudronScope, routes.cloudron.getLogStream);
    router.get ('/api/v1/cloudron/eventlog', cloudronScope, routes.eventlog.list);
@@ -174,6 +184,9 @@ function initializeExpressSync() {
    // working off the user behind the provided token
    router.get ('/api/v1/profile', profileScope, routes.profile.get);
    router.post('/api/v1/profile', profileScope, routes.profile.update);
+    router.get ('/api/v1/profile/avatar/:identifier', routes.profile.getAvatar);
+    router.post('/api/v1/profile/avatar', profileScope, multipart, routes.profile.setAvatar);
+    router.del ('/api/v1/profile/avatar', profileScope, routes.profile.clearAvatar);
    router.post('/api/v1/profile/password', profileScope, routes.users.verifyPassword, routes.profile.changePassword);
    router.post('/api/v1/profile/twofactorauthentication', profileScope, routes.profile.setTwoFactorAuthenticationSecret);
    router.post('/api/v1/profile/twofactorauthentication/enable', profileScope, routes.profile.enableTwoFactorAuthentication);
@@ -255,8 +268,8 @@ function initializeExpressSync() {
    router.post('/api/v1/apps/:id/configure/env', appsManageScope, routes.apps.setEnvironment);
    router.post('/api/v1/apps/:id/configure/data_dir', appsManageScope, routes.apps.setDataDir);
    router.post('/api/v1/apps/:id/configure/location', appsManageScope, routes.apps.setLocation);
-    router.post('/api/v1/apps/:id/configure/repair', appsManageScope, routes.apps.repairApp);

+    router.post('/api/v1/apps/:id/repair', appsManageScope, routes.apps.repairApp);
    router.post('/api/v1/apps/:id/update',    appsManageScope, routes.apps.updateApp);
    router.post('/api/v1/apps/:id/restore',   appsManageScope, routes.apps.restoreApp);
    router.post('/api/v1/apps/:id/import',    appsManageScope, routes.apps.importApp);
@@ -498,6 +498,8 @@ function setSysinfoConfig(sysinfoConfig, callback) {
    assert.strictEqual(typeof sysinfoConfig, 'object');
    assert.strictEqual(typeof callback, 'function');

+    if (isDemo()) return callback(new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode'));
+
    sysinfo.testConfig(sysinfoConfig, function (error) {
        if (error) return callback(error);

@@ -7,6 +7,7 @@ exports = module.exports = {
 };

 var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
    child_process = require('child_process'),
    debug = require('debug')('box:shell'),
    once = require('once'),
@@ -60,7 +61,7 @@ function spawn(tag, file, args, options, callback) {
        if (code || signal) debug(tag + ' code: %s, signal: %s', code, signal);
        if (code === 0) return callback(null);

-        var e = new Error(util.format(tag + ' exited with error %s signal %s', code, signal));
+        let e = new BoxError(BoxError.SPAWN_ERROR, `${tag} exited with code ${code} signal ${signal}`);
        e.code = code;
        e.signal = signal;
        callback(e);
@@ -68,7 +69,8 @@ function spawn(tag, file, args, options, callback) {

    cp.on('error', function (error) {
        debug(tag + ' code: %s, signal: %s', error.code, error.signal);
-        callback(error);
+        let e = new BoxError(BoxError.SPAWN_ERROR, `${tag} errored with code ${error.code} message ${error.message}`);
+        callback(e);
    });

    return cp;
@@ -115,27 +115,25 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac
        query.maxResults = batchSize;
    }

-    async.forever(function listAndDownload(foreverCallback) {
-        bucket.getFiles(query, function (error, files, nextQuery) {
-            if (error) return foreverCallback(error);
+    let done = false;

-            if (files.length === 0) return foreverCallback(new Error('Done'));
+    async.whilst(() => !done, function listAndDownload(whilstCallback) {
+        bucket.getFiles(query, function (error, files, nextQuery) {
+            if (error) return whilstCallback(error);
+
+            if (files.length === 0) { done = true; return whilstCallback(); }

            const entries = files.map(function (f) { return { fullPath: f.name }; });
            iteratorCallback(entries, function (error) {
-                if (error) return foreverCallback(error);
-                if (!nextQuery) return foreverCallback(new Error('Done'));
+                if (error) return whilstCallback(error);
+                if (!nextQuery) { done = true; return whilstCallback(); }

                query = nextQuery;

-                foreverCallback();
+                whilstCallback();
            });
        });
-    }, function (error) {
-        if (error.message === 'Done') return callback(null);
-
-        callback(error);
-    });
+    }, callback);
 }

 function copy(apiConfig, oldFilePath, newFilePath) {
@@ -28,6 +28,7 @@ exports = module.exports = {
 };

 var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
    EventEmitter = require('events');

 function removePrivateFields(apiConfig) {
@@ -35,6 +36,7 @@ function removePrivateFields(apiConfig) {
    return apiConfig;
 }

+// eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
    // in-place injection of tokens and api keys which came in with domains.SECRET_PLACEHOLDER
 }
@@ -48,7 +50,7 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
    // Result: none
    // sourceStream errors are handled upstream

-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upload is not implemented'));
 }

 function download(apiConfig, backupFilePath, callback) {
@@ -57,7 +59,7 @@ function download(apiConfig, backupFilePath, callback) {
    assert.strictEqual(typeof callback, 'function');

    // Result: download stream
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'download is not implemented'));
 }

 function downloadDir(apiConfig, backupFilePath, destDir) {
@@ -87,7 +89,7 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
    assert.strictEqual(typeof iteratorCallback, 'function');
    assert.strictEqual(typeof callback, 'function');

-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'listDir is not implemented'));
 }

 function remove(apiConfig, filename, callback) {
@@ -97,7 +99,7 @@ function remove(apiConfig, filename, callback) {

    // Result: none

-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'remove is not implemented'));
 }

 function removeDir(apiConfig, pathPrefix) {
@@ -106,7 +108,7 @@ function removeDir(apiConfig, pathPrefix) {

    // Result: none
    var events = new EventEmitter();
-    process.nextTick(function () { events.emit('done', new Error('not implemented')); });
+    process.nextTick(function () { events.emit('done', new BoxError(BoxError.NOT_IMPLEMENTED, 'removeDir is not implemented')); });
    return events;
 }

@@ -116,6 +118,6 @@ function testConfig(apiConfig, callback) {

    // Result: none - first callback argument error if config does not pass the test

-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'testConfig is not implemented'));
 }

@@ -17,6 +17,7 @@ exports = module.exports = {
 };

 var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
    debug = require('debug')('box:storage/noop'),
    EventEmitter = require('events');

@@ -38,7 +39,7 @@ function download(apiConfig, backupFilePath, callback) {

    debug('download: %s', backupFilePath);

-    callback(new Error('Cannot download from noop backend'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'Cannot download from noop backend'));
 }

 function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
@@ -60,7 +61,7 @@ function downloadDir(apiConfig, backupFilePath, destDir) {
    process.nextTick(function () {
        debug('downloadDir: %s -> %s', backupFilePath, destDir);

-        events.emit('done', new Error('Cannot download from noop backend'));
+        events.emit('done', new BoxError(BoxError.NOT_IMPLEMENTED, 'Cannot download from noop backend'));
    });
    return events;
 }
@@ -109,5 +110,6 @@ function removePrivateFields(apiConfig) {
    return apiConfig;
 }

+// eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
 }
@@ -161,29 +161,27 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
            MaxKeys: batchSize
        };

-        async.forever(function listAndDownload(foreverCallback) {
-            s3.listObjects(listParams, function (error, listData) {
-                if (error) return foreverCallback(error);
+        let done = false;

-                if (listData.Contents.length === 0) return foreverCallback(new Error('Done'));
+        async.whilst(() => !done, function listAndDownload(whilstCallback) {
+            s3.listObjects(listParams, function (error, listData) {
+                if (error) return whilstCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message || error.code));
+
+                if (listData.Contents.length === 0) { done = true; return whilstCallback(); }

                const entries = listData.Contents.map(function (c) { return { fullPath: c.Key, size: c.Size }; });

                iteratorCallback(entries, function (error) {
-                    if (error) return foreverCallback(error);
+                    if (error) return whilstCallback(error);

-                    if (!listData.IsTruncated) return foreverCallback(new Error('Done'));
+                    if (!listData.IsTruncated) { done = true; return whilstCallback(); }

                    listParams.Marker = listData.Contents[listData.Contents.length - 1].Key; // NextMarker is returned only with delimiter

-                    foreverCallback();
+                    whilstCallback();
                });
            });
-        }, function (error) {
-            if (error.message === 'Done') return callback(null);
-
-            callback(error);
-        });
+        }, callback);
    });
 }

@@ -351,9 +349,9 @@ function remove(apiConfig, filename, callback) {

        // deleteObjects does not return error if key is not found
        s3.deleteObjects(deleteParams, function (error) {
-            if (error) debug(`remove: Unable to remove ${deleteParams.Key}. error: ${error.message}`);
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message || error.code}`)); // DO sets 'code'

-            callback(error);
+            callback(null);
        });
    });
 }
@@ -388,9 +386,12 @@ function removeDir(apiConfig, pathPrefix) {

                // deleteObjects does not return error if key is not found
                s3.deleteObjects(deleteParams, function (error /*, deleteData */) {
-                    if (error) events.emit('progress', `Unable to remove ${deleteParams.Key} ${error.message}`);
+                    if (error) {
+                        events.emit('progress', `Unable to remove ${deleteParams.Key} ${error.message || error.code}`);
+                        return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message || error.code}`)); // DO sets 'code'
+                    }

-                    iteratorCallback(error);
+                    iteratorCallback(null);
                });
            }, done);
        }, function (error) {
@@ -2,6 +2,7 @@

 var assert = require('assert'),
    async = require('async'),
+    BoxError = require('./boxerror.js'),
    DataLayout = require('./datalayout.js'),
    debug = require('debug')('box:syncer'),
    fs = require('fs'),
@@ -85,7 +86,7 @@ function sync(dataLayout, taskProcessor, concurrency, callback) {
    }

    var newCacheFd = safe.fs.openSync(newCacheFile, 'w'); // truncates any existing file
-    if (newCacheFd === -1) return callback(new Error('Error opening new cache file: ' + safe.error.message));
+    if (newCacheFd === -1) return callback(new BoxError(BoxError.FS_ERROR, 'Error opening new cache file: ' + safe.error.message));

    function advanceCache(entryPath) {
        var lastRemovedDir = null;
@@ -11,13 +11,14 @@ exports = module.exports = {
    testConfig
 };

-var assert = require('assert');
+var assert = require('assert'),
+    BoxError = require('../boxerror.js');

 function getServerIp(config, callback) {
-    assert.strictEqual(typeo config, 'object');
+    assert.strictEqual(typeof config, 'object');
    assert.strictEqual(typeof callback, 'function');

-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'testConfig is not implemented'));
 }

 function testConfig(config, callback) {
@@ -2,7 +2,8 @@

 exports = module.exports = {
    getDisks: getDisks,
-    checkDiskSpace: checkDiskSpace
+    checkDiskSpace: checkDiskSpace,
+    getMemory: getMemory
 };

 const apps = require('./apps.js'),
@@ -13,7 +14,9 @@ const apps = require('./apps.js'),
    df = require('@sindresorhus/df'),
    docker = require('./docker.js'),
    notifications = require('./notifications.js'),
-    paths = require('./paths.js');
+    os = require('os'),
+    paths = require('./paths.js'),
+    safe = require('safetydance');

 function getDisks(callback) {
    assert.strictEqual(typeof callback, 'function');
@@ -94,3 +97,15 @@ function checkDiskSpace(callback) {
        notifications.alert(notifications.ALERT_DISK_SPACE, 'Server is running out of disk space', oos ? JSON.stringify(disks.disks, null, 4) : '', callback);
    });
 }
+
+function getMemory(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    const stdout = safe.child_process.execSync('swapon --noheadings --raw --bytes --show=SIZE', { encoding: 'utf8' });
+    const swap = !stdout ? 0 : stdout.trim().split('\n').map(x => parseInt(x, 10) || 0).reduce((acc, cur) => acc + cur);
+
+    callback(null, {
+        memory: os.totalmem(),
+        swap: swap
+    });
+}
@@ -125,7 +125,7 @@ function add(type, args, callback) {
    assert(Array.isArray(args));
    assert.strictEqual(typeof callback, 'function');

-    taskdb.add({ type: type, percent: 0, message: 'Starting ...', args: args }, function (error, taskId) {
+    taskdb.add({ type: type, percent: 0, message: 'Queued', args: args }, function (error, taskId) {
        if (error) return callback(error);

        callback(null, taskId);
@@ -166,7 +166,7 @@ function startTask(taskId, options, callback) {
            } else if (!error && task.error) {
                taskError = task.error;
            } else if (!task) { // db got cleared in tests
-                taskError = new Error(`No such task ${taskId}`);
+                taskError = new BoxError(BoxError.NOT_FOUND, `No such task ${taskId}`);
            }

            delete gTasks[taskId];
@@ -8,8 +8,8 @@

 var async = require('async'),
    database = require('../database.js'),
-    disks = require('../disks.js'),
-    expect = require('expect.js');
+    expect = require('expect.js'),
+    system = require('../system.js');

 function setup(done) {
    async.series([
@@ -25,12 +25,12 @@ function cleanup(done) {
    ], done);
 }

-describe('Disks', function () {
+describe('System', function () {
    before(setup);
    after(cleanup);

    it('can get disks', function (done) {
-        disks.getDisks(function (error, disks) {
+        system.getDisks(function (error, disks) {
            expect(!error).to.be.ok();
            expect(disks).to.be.ok();
            done();
@@ -38,10 +38,19 @@ describe('Disks', function () {
    });

    it('can check for disk space', function (done) {
-        disks.checkDiskSpace(function (error) {
+        system.checkDiskSpace(function (error) {
            expect(!error).to.be.ok();
            done();
        });
    });
+
+    it('can get memory', function (done) {
+        system.getMemory(function (error, memory) {
+            expect(!error).to.be.ok();
+            expect(memory.memory).to.be.ok();
+            expect(memory.swap).to.be.ok();
+            done();
+        });
+    });
 });

@@ -105,8 +105,8 @@ function checkAppUpdates(callback) {
                        return iteratorDone();
                    }

-                    const updateIsBlocked = apps.canAutoupdateApp(app, updateInfo.manifest);
-                    if (autoupdatesEnabled && !updateIsBlocked) return iteratorDone();
+                    const canAutoupdateApp = apps.canAutoupdateApp(app, updateInfo.manifest);
+                    if (autoupdatesEnabled && canAutoupdateApp) return iteratorDone();

                    debug('Notifying of app update for %s from %s to %s', app.id, app.manifest.version, updateInfo.manifest.version);
                    notificationPending.push({
Author	SHA1	Message	Date
Girish Ramakrishnan	9dd3e4537a	return 422 on instance id mismatch the ui redirects otherwise	2019-12-11 15:13:38 -08:00
Girish Ramakrishnan	a5f31e8724	Revert "rename ami to aws-mp" This reverts commit `72ac00b69a`. Existing code relies on this, so don't change it	2019-12-11 12:56:30 -08:00
Girish Ramakrishnan	72ac00b69a	rename ami to aws-mp	2019-12-11 12:27:55 -08:00
Girish Ramakrishnan	ae5722a7d4	eventlog: typo when mail list is removed	2019-12-11 10:05:45 -08:00
Johannes Zellner	4e3192d450	Avoid double dns setup tracking	2019-12-11 14:02:40 +01:00
Johannes Zellner	ccca3aca04	Send setup state to get the actually correct ip	2019-12-10 18:01:07 +01:00
Girish Ramakrishnan	e4dd5d6434	Fix crash when uploading file	2019-12-09 15:02:51 -08:00
Girish Ramakrishnan	9a77fb6306	acme2: implement post-as-get https://tools.ietf.org/html/rfc8555#section-6.3 https://community.letsencrypt.org/t/post-as-get-and-empty-payload-instead-of/86720/3 https://community.letsencrypt.org/t/problem-with-renew-certificates-the-request-message-was-malformed-method-not-allowed/107889/17	2019-12-08 19:17:52 -08:00
Girish Ramakrishnan	3ec5c713bf	debug: certFilePath is undefined	2019-12-08 18:23:12 -08:00
Girish Ramakrishnan	837fc27e94	canAutoupdateApp now returns bool	2019-12-08 16:55:56 -08:00
Girish Ramakrishnan	9ad6025310	search and replace gone wrong	2019-12-06 13:52:43 -08:00
Girish Ramakrishnan	d765e4c619	add a note	2019-12-06 12:39:46 -08:00
Girish Ramakrishnan	f5217236d6	Change the version number	2019-12-06 12:28:08 -08:00
Girish Ramakrishnan	8f8d099faf	Add to changes	2019-12-06 12:23:49 -08:00
Girish Ramakrishnan	16660e083f	Also set overwriteDns when manifest is not provided	2019-12-06 12:21:28 -08:00
Girish Ramakrishnan	4e35020a1c	Set overwriteDns for install task	2019-12-06 12:11:34 -08:00
Girish Ramakrishnan	111e0bcb5f	Fix repair route path	2019-12-06 11:44:41 -08:00
Girish Ramakrishnan	d7f9a547fc	Disable requiredState check for now there is a race but this is mitigated by the checkAppState non-db logic for now	2019-12-06 11:29:35 -08:00
Girish Ramakrishnan	6a64f24e98	Fix repair If a task fails, we can either: * allow other task ops to be called - we cannot do this because the ops are fine-grained. for example, a restore failure removes many things and calling set-memory or set-location in that state won't make sense. * provide a generic repair route - this allows one to override args and call the failed task again. this is what we have now but has the issue that this repair function has to know about all the other op functions. for example, for argument validation. we can do some complicated refactoring to make it work if we want. * just a generic total re-configure - this does not work because clone/restore/backup/datadir/uninstall/update failure leaves the app in a state which re-configure cannot do anything about. * allow the failed op to be called again - this seems the easiest. we just allow the route to be called again in the error state. * if we hit a state where even providing extra args, cannot get you out of this "error" state, we have to provide some repair route. for example, maybe the container disappeared by some docke error. user clicks 'repair' to recreate the container. this route does not have to take any args. The final solution is: * a failed task can be called again via the route. so we can resubmit any args and we get validation * repair route just re-configures and can be called in any state to just rebuild container. re-configure is also doing only local changes (docker, nginx) * install/clone failures are fixed using repair route. updated manifest can be passed in. * UI shows backup selector for restore failures * UI shows domain selector for change location failulre	2019-12-06 09:56:09 -08:00
Girish Ramakrishnan	37d7be93b5	Move oldManifest out of restoreConfig	2019-12-06 09:56:03 -08:00
Girish Ramakrishnan	9c809aa6e1	remove dead comment	2019-12-06 09:35:08 -08:00
Girish Ramakrishnan	7ab9f3fa2f	re-configure does not require oldConfig this is only needed when changing location now. the configure() is now entirely local i.e rebuild local container and the reverse proxy config	2019-12-06 09:23:58 -08:00
Girish Ramakrishnan	ffeb484a10	No need to return args as part of task.get This reverts commit `831e22b4ff`. This reverts commit `6774514bd2`.	2019-12-06 08:42:49 -08:00
Girish Ramakrishnan	2ffb32ae60	Skip moving data if source and target are same	2019-12-06 08:09:43 -08:00
Girish Ramakrishnan	905bb92bad	s3: ensure BoxError return	2019-12-05 21:50:44 -08:00
Girish Ramakrishnan	3926efd153	restore: only take non-empty backupId	2019-12-05 21:16:35 -08:00
Girish Ramakrishnan	c5e5bb90e3	better error message	2019-12-05 21:16:35 -08:00
Girish Ramakrishnan	cea543cba5	On backup error, only set the task error at some point, the backup ui can show this error	2019-12-05 16:34:40 -08:00
Girish Ramakrishnan	a8b489624d	fix error messages	2019-12-05 16:27:00 -08:00
Girish Ramakrishnan	49d3bddb62	Show download progress when restoring rsync backups	2019-12-05 15:44:52 -08:00
Girish Ramakrishnan	c0ff3cbd22	move progressTag to the end	2019-12-05 15:44:52 -08:00
Girish Ramakrishnan	1de97d6967	do not clear localstorage during in-place import	2019-12-05 12:42:08 -08:00
Girish Ramakrishnan	a44a82083e	Add backups.testProviderConfig fields like format/retention won't be validated here since it's only testing the access credentials	2019-12-05 11:55:53 -08:00
Girish Ramakrishnan	d57681ff21	put fqdn in the end	2019-12-05 11:15:21 -08:00
Girish Ramakrishnan	e3de2f81d3	setup and clear addons before import	2019-12-05 11:12:40 -08:00
Girish Ramakrishnan	e8c5f8164c	do not delete data dir for in-place import	2019-12-05 11:01:27 -08:00
Girish Ramakrishnan	c07e215148	Use BoxError in on error cases	2019-12-05 09:54:29 -08:00
Girish Ramakrishnan	4bb676fb5c	add asserts	2019-12-05 09:32:45 -08:00
Johannes Zellner	dbdf86edfc	No need to return the same data which the route got passed in	2019-12-05 18:02:57 +01:00
Johannes Zellner	2c8e6330ce	Do not allow to change the sysinfo in demo mode	2019-12-05 16:06:21 +01:00
Girish Ramakrishnan	1b563854a7	implement in-place import and custom backup config	2019-12-04 19:27:05 -08:00
Girish Ramakrishnan	80b890101b	Add changes	2019-12-04 17:53:02 -08:00
Girish Ramakrishnan	c3696469ff	Add app fqdn to backup progress message	2019-12-04 17:49:31 -08:00
Girish Ramakrishnan	3e08e7c653	Typo in docker socket path	2019-12-04 14:37:00 -08:00
Girish Ramakrishnan	53e39f571c	Make addons code remove a BoxError	2019-12-04 14:28:42 -08:00
Girish Ramakrishnan	c992853cca	lint	2019-12-04 11:18:39 -08:00
Girish Ramakrishnan	85e17b570b	Use whilst instead of forever this gets rid of the Error object	2019-12-04 11:17:44 -08:00
Girish Ramakrishnan	30eccfb54b	Use BoxError instead of Error in all places This moves everything other than the addon code and some 'done' logic	2019-12-04 11:02:54 -08:00
Girish Ramakrishnan	3623831390	Typo	2019-12-04 10:23:16 -08:00
Girish Ramakrishnan	d0a3d00492	Use NOT_IMPLEMENTED error code	2019-12-04 10:22:22 -08:00
Girish Ramakrishnan	0b6fbfd910	Better addon error messages	2019-12-04 10:09:57 -08:00
Girish Ramakrishnan	8cfb27fdcd	Add changes	2019-12-03 15:39:29 -08:00
Girish Ramakrishnan	841ab54565	better logs	2019-12-03 15:11:27 -08:00
Girish Ramakrishnan	a2e9254343	lint	2019-12-03 15:10:06 -08:00
Johannes Zellner	43cb03a292	Send provider and version during registration	2019-12-02 18:19:51 +01:00
Johannes Zellner	f2fca33309	Add support to upload custom profile avatar	2019-12-02 18:03:54 +01:00
Johannes Zellner	14d26fe064	Do not crash on migration A bit late but still	2019-12-02 18:03:54 +01:00
Girish Ramakrishnan	9cc968e790	Pass the new data dir as a task argument	2019-11-25 14:22:27 -08:00
Girish Ramakrishnan	831e22b4ff	Fix failing test	2019-11-23 18:35:15 -08:00
Girish Ramakrishnan	6774514bd2	Return args as part of task.get the ui needs this to repair any failed app task	2019-11-23 18:06:33 -08:00
Girish Ramakrishnan	f543b98764	Remove BoxError.UNKNOWN_ERROR	2019-11-22 14:27:41 -08:00
Johannes Zellner	2e94600afe	Don't set 'Starting ...' as initial task progress message This is confusing for tasks like "stop" as it will say "Starting ..."	2019-11-22 13:54:43 +01:00
Johannes Zellner	9295ce783a	Other logs are lowercase	2019-11-22 12:31:41 +01:00
Johannes Zellner	134f8a28bf	Hide access tokens from logs	2019-11-22 12:29:13 +01:00
Girish Ramakrishnan	ab5e4e998c	Fix reduce usage	2019-11-21 13:48:31 -08:00
Girish Ramakrishnan	a98551f99c	rename disks to system	2019-11-21 13:01:08 -08:00
Girish Ramakrishnan	42fe84152a	return swap information	2019-11-21 12:55:17 -08:00
Girish Ramakrishnan	8a3d212bd4	Fix note	2019-11-20 16:17:47 -08:00