More 4.4.4 changes

https://forum.cloudron.io/topic/2050/switch-to-debian-ubuntu-spying

CHANGES

@@ -1735,3 +1735,34 @@
 * Add 'New Apps' section to Appstore view
 * Fix issue where graphs of some apps were not appearing
 
+[4.4.0]
+* Show swap in graphs
+* Make avatars customizable
+* Hide access tokens from logs
+* Add missing '@' sign for email address in app mailbox
+* Add app fqdn to backup progress message
+* import: add option to import app in-place
+* import: add option to import app from arbitrary backup config
+* Show download progress for rsync backups
+* Fix various repair workflows
+* acme2: Implement post-as-get
+
+[4.4.1]
+* ami: fix AWS provider validation
+
+[4.4.2]
+* Fix crash when reporting that DKIM is not setup correctly
+* Stopped apps cannot be updated or auto-updated
+* eventlog: track support ticket creation and remote support status
+
+[4.4.3]
+* Add restart button in recovery section
+* Fix issue where memory usage was not computed correctly
+* cloudflare: support API tokens
+
+[4.4.4]
+* Fix bug where restart button in terminal was not working
+* Add search field in apps view
+* Make app view tags and domain filter persistent
+* Add timezone UI
+

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2019 Cloudron UG
+Copyright (c) 2020 Cloudron UG
 
 With regard to the Cloudron Software:
 

baseimage/initializeBaseUbuntuImage.sh

@@ -123,6 +123,13 @@ timedatectl set-ntp 1
 # mysql follows the system timezone
 timedatectl set-timezone UTC
 
+echo "==> Adding sshd configuration warning"
+sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://cloudron.io/documentation/security/#securing-ssh-access' -i /etc/ssh/sshd_config
+
+# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
+echo "==> Disabling motd news"
+sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
+
 # Disable bind for good measure (on online.net, kimsufi servers these are pre-installed and conflicts with unbound)
 systemctl stop bind9 || true
 systemctl disable bind9 || true

migrations/20190911193842-mailboxes-make-members-fully-qualified.js

@@ -9,7 +9,7 @@ exports.up = function(db, callback) {
             if (!mailbox.membersJson) return iteratorDone();
 
             let members = JSON.parse(mailbox.membersJson);
-            members = members.map((m) => m.indexOf('@') === -1 ? `${m}@${mailbox.domain}` : m); // only because we don't do things in a xction
+            members = members.map((m) => m && m.indexOf('@') === -1 ? `${m}@${mailbox.domain}` : m); // only because we don't do things in a xction
 
             db.runSql('UPDATE mailboxes SET membersJson=? WHERE name=? AND domain=?', [ JSON.stringify(members), mailbox.name, mailbox.domain ], iteratorDone);
         }, callback);

migrations/20200101235723-domains-set-cloudflare-tokenType.js

@@ -0,0 +1,22 @@
+'use strict';
+
+let async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('SELECT * FROM domains', function (error, domains) {
+        if (error) return callback(error);
+
+        async.eachSeries(domains, function (domain, iteratorCallback) {
+            if (domain.provider !== 'cloudflare') return iteratorCallback();
+
+            let config = JSON.parse(domain.configJson);
+            config.tokenType = 'GlobalApiKey';
+
+            db.runSql('UPDATE domains SET configJson = ? WHERE domain = ?', [ JSON.stringify(config), domain.domain ], iteratorCallback);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

runTests

@@ -22,7 +22,7 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
 mkdir -p platformdata/addons/mail platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
 
 # put cert

scripts/cloudron-setup

@@ -99,6 +99,7 @@ if [[ -z "${provider}" ]]; then
 elif [[  \
             "${provider}" != "ami" && \
             "${provider}" != "azure" && \
+            "${provider}" != "azure-image" && \
             "${provider}" != "caas" && \
             "${provider}" != "cloudscale" && \
             "${provider}" != "contabo" && \

setup/start.sh

@@ -51,6 +51,7 @@ mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
 mkdir -p "${PLATFORM_DATA_DIR}/update"
 
 mkdir -p "${BOX_DATA_DIR}/appicons"
+mkdir -p "${BOX_DATA_DIR}/profileicons"
 mkdir -p "${BOX_DATA_DIR}/certs"
 mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
 mkdir -p "${BOX_DATA_DIR}/mail/dkim"

setup/start/custom.yml

@@ -37,4 +37,4 @@
 #   notifyCloudronAdmins: false
 #
 # footer:
-#   body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
+#   body: '&copy; 2020 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'

src/addons.js

@@ -40,7 +40,6 @@ var accesscontrol = require('./accesscontrol.js'),
     crypto = require('crypto'),
     debug = require('debug')('box:addons'),
     docker = require('./docker.js'),
-    dockerConnection = docker.connection,
     fs = require('fs'),
     graphs = require('./graphs.js'),
     hat = require('./hat.js'),
@@ -232,33 +231,10 @@ function dumpPath(addon, appId) {
     }
 }
 
-function restartContainer(serviceName, callback) {
-    assert.strictEqual(typeof serviceName, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    assert(KNOWN_SERVICES[serviceName], `Unknown service ${serviceName}`);
-
-    docker.stopContainer(serviceName, function (error) {
-        if (error) return callback(error);
-
-        docker.startContainer(serviceName, function (error) {
-            if (error && error.reason === BoxError.NOT_FOUND) {
-                callback(null); // callback early since rebuilding takes long
-                return rebuildService(serviceName, function (error) { if (error) console.error(`Unable to rebuild service ${serviceName}`, error); });
-            }
-            if (error) return callback(error);
-
-            callback(null);
-        });
-    });
-}
-
 function rebuildService(serviceName, callback) {
     assert.strictEqual(typeof serviceName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    assert(KNOWN_SERVICES[serviceName], `Unknown service ${serviceName}`);
-
     // this attempts to recreate the service docker container if they don't exist but platform infra version is unchanged
     // passing an infra version of 'none' will not attempt to purge existing data, not sure if this is good or bad
     if (serviceName === 'mongodb') return startMongodb({ version: 'none' }, callback);
@@ -271,6 +247,24 @@ function rebuildService(serviceName, callback) {
     callback();
 }
 
+function restartContainer(serviceName, callback) {
+    assert.strictEqual(typeof serviceName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.stopContainer(serviceName, function (error) {
+        if (error) return callback(error);
+
+        docker.startContainer(serviceName, function (error) {
+            if (error && error.reason === BoxError.NOT_FOUND) {
+                callback(null); // callback early since rebuilding takes long
+                return rebuildService(serviceName, function (error) { if (error) console.error(`Unable to rebuild service ${serviceName}`, error); });
+            }
+
+            callback(error);
+        });
+    });
+}
+
 function getServiceDetails(containerName, tokenEnvName, callback) {
     assert.strictEqual(typeof containerName, 'string');
     assert.strictEqual(typeof tokenEnvName, 'string');
@@ -280,15 +274,15 @@ function getServiceDetails(containerName, tokenEnvName, callback) {
         if (error) return callback(error);
 
         const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
-        if (!ip) return callback(new BoxError(BoxError.INACTIVE, `Error getting ${containerName} container ip`));
+        if (!ip) return callback(new BoxError(BoxError.INACTIVE, `Error getting IP of ${containerName} service`));
 
         // extract the cloudron token for auth
         const env = safe.query(result, 'Config.Env', null);
-        if (!env) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} env`));
+        if (!env) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error inspecting environment of ${containerName} service`));
         const tmp = env.find(function (e) { return e.indexOf(tokenEnvName) === 0; });
-        if (!tmp) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} cloudron token env var`));
+        if (!tmp) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`));
         const token = tmp.slice(tokenEnvName.length + 1); // +1 for the = sign
-        if (!token)  return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} cloudron token`));
+        if (!token)  return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`));
 
         callback(null, { ip: ip, token: token, state: result.State });
     });
@@ -339,6 +333,9 @@ function getService(serviceName, callback) {
     var tmp = {
         name: serviceName,
         status: null,
+        memoryUsed: 0,
+        memoryPercent: 0,
+        error: null,
         config: {
             // If a property is not set then we cannot change it through the api, see below
             // memory: 0,
@@ -483,8 +480,8 @@ function waitForService(containerName, tokenEnvName, callback) {
 
         async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
             request.get(`https://${result.ip}:3000/healthcheck?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
-                if (error) return retryCallback(new Error(`Error waiting for ${containerName}: ${error.message}`));
-                if (response.statusCode !== 200 || !response.body.status) return retryCallback(new Error(`Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Network error waiting for ${containerName}: ${error.message}`));
+                if (response.statusCode !== 200 || !response.body.status) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 retryCallback(null);
             });
@@ -502,7 +499,7 @@ function setupAddons(app, addons, callback) {
     debugApp(app, 'setupAddons: Setting up %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         debugApp(app, 'Setting up addon %s with options %j', addon, addons[addon]);
 
@@ -520,7 +517,7 @@ function teardownAddons(app, addons, callback) {
     debugApp(app, 'teardownAddons: Tearing down %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         debugApp(app, 'Tearing down addon %s with options %j', addon, addons[addon]);
 
@@ -540,7 +537,7 @@ function backupAddons(app, addons, callback) {
     debugApp(app, 'backupAddons: Backing up %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         KNOWN_ADDONS[addon].backup(app, addons[addon], iteratorCallback);
     }, callback);
@@ -558,7 +555,7 @@ function clearAddons(app, addons, callback) {
     debugApp(app, 'clearAddons: clearing %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         KNOWN_ADDONS[addon].clear(app, addons[addon], iteratorCallback);
     }, callback);
@@ -576,7 +573,7 @@ function restoreAddons(app, addons, callback) {
     debugApp(app, 'restoreAddons: restoring %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         KNOWN_ADDONS[addon].restore(app, addons[addon], iteratorCallback);
     }, callback);
@@ -587,7 +584,7 @@ function importAppDatabase(app, addon, callback) {
     assert.strictEqual(typeof addon, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!(addon in KNOWN_ADDONS)) return callback(new Error(`No such addon: ${addon}`));
+    if (!(addon in KNOWN_ADDONS)) return callback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
     async.series([
         KNOWN_ADDONS[addon].setup.bind(null, app, app.manifest.addons[addon]),
@@ -1052,8 +1049,8 @@ function setupMySql(app, options, callback) {
             if (error) return callback(error);
 
             request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up mysql: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mysql: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
@@ -1090,9 +1087,10 @@ function clearMySql(app, options, callback) {
     getServiceDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mysql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+
             callback();
         });
     });
@@ -1109,9 +1107,9 @@ function teardownMySql(app, options, callback) {
     getServiceDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.delete(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mysql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             appdb.unsetAddonConfig(app.id, 'mysql', callback);
         });
@@ -1130,14 +1128,15 @@ function pipeRequestToFile(url, filename, callback) {
         callback(error);
     });
 
-    writeStream.on('error', done);
+    writeStream.on('error', (error) => done(new BoxError(BoxError.FS_ERROR, `Error writing to ${filename}: ${error.message}`)));
+
     writeStream.on('open', function () {
         // note: do not attach to post callback handler because this will buffer the entire reponse!
         // see https://github.com/request/request/issues/2270
         const req = request.post(url, { rejectUnauthorized: false });
-        req.on('error', done); // network error, dns error, request errored in middle etc
+        req.on('error', (error) => done(new BoxError(BoxError.NETWORK_ERROR, `Request error writing to ${filename}: ${error.message}`))); // network error, dns error, request errored in middle etc
         req.on('response', function (response) {
-            if (response.statusCode !== 200) return done(new Error(`Unexpected response code: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`));
+            if (response.statusCode !== 200) return done(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code when piping ${url}: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`));
 
             response.pipe(writeStream).on('finish', done); // this is hit after data written to disk
         });
@@ -1176,11 +1175,11 @@ function restoreMySql(app, options, callback) {
         if (error) return callback(error);
 
         var input = fs.createReadStream(dumpPath('mysql', app.id));
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mysql: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from mysql addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1265,8 +1264,8 @@ function setupPostgreSql(app, options, callback) {
             if (error) return callback(error);
 
             request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up postgresql: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up postgresql: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
@@ -1298,9 +1297,9 @@ function clearPostgreSql(app, options, callback) {
     getServiceDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing postgresql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1317,9 +1316,9 @@ function teardownPostgreSql(app, options, callback) {
     getServiceDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error tearing down postgresql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error tearing down postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error tearing down postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             appdb.unsetAddonConfig(app.id, 'postgresql', callback);
         });
@@ -1358,11 +1357,11 @@ function restorePostgreSql(app, options, callback) {
         if (error) return callback(error);
 
         var input = fs.createReadStream(dumpPath('postgresql', app.id));
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring postgresql: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from postgresql addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1441,8 +1440,8 @@ function setupMongoDb(app, options, callback) {
             if (error) return callback(error);
 
             request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up mongodb: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up mongodb. Status code: ${response.statusCode}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mongodb: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
@@ -1476,9 +1475,9 @@ function clearMongodb(app, options, callback) {
     getServiceDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/databases/${app.id}/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mongodb: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/databases/${app.id}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback();
         });
@@ -1495,9 +1494,9 @@ function teardownMongoDb(app, options, callback) {
     getServiceDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.delete(`https://${result.ip}:3000/databases/${app.id}?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error tearing down mongodb: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error tearing down mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/databases/${app.id}?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             appdb.unsetAddonConfig(app.id, 'mongodb', callback);
         });
@@ -1532,11 +1531,11 @@ function restoreMongoDb(app, options, callback) {
         if (error) return callback(error);
 
         const readStream = fs.createReadStream(dumpPath('mongodb', app.id));
-        readStream.on('error', callback);
+        readStream.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mongodb: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${app.id}/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from mongodb addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/databases/${app.id}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1654,9 +1653,9 @@ function clearRedis(app, options, callback) {
     getServiceDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing redis: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing redis. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing redis: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing redis. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1668,18 +1667,11 @@ function teardownRedis(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = dockerConnection.getContainer('redis-' + app.id);
-
-    var removeOptions = {
-        force: true, // kill container if it's running
-        v: true // removes volumes associated with the container
-    };
-
-    container.remove(removeOptions, function (error) {
-        if (error && error.statusCode !== 404) return callback(new Error('Error removing container:' + error));
+    docker.deleteContainer(`redis-${app.id}`, function (error) {
+        if (error) return callback(error);
 
         shell.sudo('removeVolume', [ RMADDONDIR_CMD, 'redis', app.id ], {}, function (error) {
-            if (error) return callback(new Error('Error removing redis data:' + error));
+            if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`));
 
             rimraf(path.join(paths.LOG_DIR, `redis-${app.id}`), function (error) {
                 if (error) debugApp(app, 'cannot cleanup logs: %s', error);
@@ -1712,6 +1704,8 @@ function restoreRedis(app, options, callback) {
 
     debugApp(app, 'Restoring redis');
 
+    callback = once(callback); // protect from multiple returns with streams
+
     getServiceDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) {
         if (error) return callback(error);
 
@@ -1722,11 +1716,11 @@ function restoreRedis(app, options, callback) {
         } else { // old location of dumps
             input = fs.createReadStream(path.join(paths.APPS_DATA_DIR, app.id, 'redis/dump.rdb'));
         }
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring redis: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from redis addon: ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1811,7 +1805,7 @@ function statusGraphite(callback) {
         if (error && error.reason === BoxError.NOT_FOUND) return callback(null, { status: exports.SERVICE_STATUS_STOPPED });
         if (error) return callback(error);
 
-        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { timeout: 3000 }, function (error, response) {
+        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { json: true, timeout: 3000 }, function (error, response) {
             if (error) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite: ${error.message}` });
             if (response.statusCode !== 200) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite. Status code: ${response.statusCode} message: ${response.body.message}` });
 

src/apphealthmonitor.js

@@ -186,9 +186,10 @@ function processApp(callback) {
         async.each(result, checkAppHealth, function (error) {
             if (error) console.error(error);
 
-            var alive = result
+            const alive = result
                 .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; })
-                .map(function (a) { return (a.location || 'naked_domain') + '|' + (a.manifest.id || 'customapp'); }).join(', ');
+                .map(a => a.fqdn)
+                .join(', ');
 
             debug('apps alive: [%s]', alive);
 

src/apps.js

@@ -43,6 +43,7 @@ exports = module.exports = {
 
     start: start,
     stop: stop,
+    restart: restart,
 
     exec: exec,
 
@@ -79,6 +80,7 @@ exports = module.exports = {
     ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app. this is state because it blocks other operations
     ISTATE_PENDING_START: 'pending_start',
     ISTATE_PENDING_STOP: 'pending_stop',
+    ISTATE_PENDING_RESTART: 'pending_restart',
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
@@ -622,8 +624,8 @@ function addTask(appId, installationState, task, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     const { args, values } = task;
-    // by default, a task can only run on installed state. if it's null, it can be run on any state
-    const requiredState = 'requiredState' in task ? task.requiredState : exports.ISTATE_INSTALLED;
+    // TODO: match the SQL logic to match checkAppState. this means checking the error.installationState and installationState. Unfortunately, former is JSON right now
+    const requiredState = null; // 'requiredState' in task ? task.requiredState : exports.ISTATE_INSTALLED;
     const scheduleNow = 'scheduleNow' in task ? task.scheduleNow : true;
     const requireNullTaskId = 'requireNullTaskId' in task ? task.requireNullTaskId : true;
 
@@ -645,10 +647,14 @@ function checkAppState(app, state) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof state, 'string');
 
-    if (app.taskId) return new BoxError(BoxError.BAD_STATE, `Not allowed in this app state : ${app.installationState} / ${app.runState}`);
+    if (app.taskId) return new BoxError(BoxError.BAD_STATE, `Locked by task ${app.taskId} : ${app.installationState} / ${app.runState}`);
 
     if (app.installationState === exports.ISTATE_ERROR) {
-        if (state !== exports.ISTATE_PENDING_UNINSTALL) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
+        // allow task to be called again if that was the errored task
+        if (app.error.installationState === state) return null;
+
+        // allow uninstall from any state
+        if (state !== exports.ISTATE_PENDING_UNINSTALL && state !== exports.ISTATE_PENDING_RESTORE) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
     }
 
     return null;
@@ -1196,13 +1202,13 @@ function setDataDir(appId, dataDir, auditSource, callback) {
         if (error) return callback(error);
 
         const task = {
-            args: { oldDataDir: app.dataDir },
-            values: { dataDir: dataDir }
+            args: { newDataDir: dataDir },
+            values: { }
         };
         addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, function (error, result) {
             if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId, app: app, dataDir: dataDir, taskId: result.taskId });
+            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId, app: app, newDataDir: dataDir, taskId: result.taskId });
 
             callback(null, { taskId: result.taskId });
         });
@@ -1225,6 +1231,8 @@ function update(appId, data, auditSource, callback) {
         error = checkAppState(app, exports.ISTATE_PENDING_UPDATE);
         if (error) return callback(error);
 
+        if (app.runState === exports.RSTATE_STOPPED) return callback(new BoxError(BoxError.BAD_STATE, 'Stopped apps cannot be updated'));
+
         downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
             if (error) return callback(error);
 
@@ -1344,9 +1352,10 @@ function getLogs(appId, options, callback) {
     });
 }
 
+// does a re-configure when called from most states. for install/clone errors, it re-installs with an optional manifest
 function repair(appId, data, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof data, 'object'); // { manifest }
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -1355,42 +1364,44 @@ function repair(appId, data, auditSource, callback) {
     get(appId, function (error, app) {
         if (error) return callback(error);
 
-        const appError = app.error || {}; // repair can always be called
-        const newState = appError.installationState ? appError.installationState : exports.ISTATE_PENDING_CONFIGURE;
+        let errorState = (app.error && app.error.installationState) || exports.ISTATE_PENDING_CONFIGURE;
 
-        debug(`Repairing app with error: ${JSON.stringify(error)} and state: ${newState}`);
+        const task = {
+            args: {},
+            values: {},
+            requiredState: null
+        };
 
-        let values = _.pick(data, 'location', 'domain', 'alternateDomains');
+        // maybe split this into a separate route like reinstall?
+        if (errorState === exports.ISTATE_PENDING_INSTALL || errorState === exports.ISTATE_PENDING_CLONE) {
+            task.args = { overwriteDns: true };
+            if (data.manifest) {
+                error = manifestFormat.parse(data.manifest);
+                if (error) return callback(new BoxError(BoxError.BAD_FIELD, `manifest error: ${error.message}`));
 
-        const locations = (values.location ? [ { subdomain: values.location, domain: values.domain } ] : []).concat(values.alternateDomains || []);
-        validateLocations(locations, function (error, domainObjectMap) {
+                error = checkManifestConstraints(data.manifest);
+                if (error) return callback(error);
+
+                task.values.manifest = data.manifest;
+                task.args.oldManifest = app.manifest;
+            }
+        } else {
+            errorState = exports.ISTATE_PENDING_CONFIGURE;
+        }
+
+        addTask(appId, errorState, task, function (error, result) {
             if (error) return callback(error);
 
-            tasks.get(appError.taskId || '', function (error, task) {
-                let args = !error ? task.args[1] : {}; // pick args for the failed task. the first argument is the app id
+            eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { taskId: result.taskId, app });
 
-                if ('backupId' in data) {
-                    args.restoreConfig = data.backupId ? { backupId: data.backupId, backupFormat: data.backupFormat, oldManifest: app.manifest } : null; // when null, apptask simply reinstalls
-                }
-                args.overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false;
-
-                // create a new task instead of updating the old one, since it helps tracking
-                addTask(appId, newState, { args, values, requiredState: null }, function (error, result) {
-                    if (error && error.reason === BoxError.ALREADY_EXISTS) error = getDuplicateErrorDetails(error.message, locations, domainObjectMap, { /* portBindings */});
-                    if (error) return callback(error);
-
-                    eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { taskId: result.taskId, app, newState });
-
-                    callback(null, { taskId: result.taskId });
-                });
-            });
+            callback(null, { taskId: result.taskId });
         });
     });
 }
 
-function restore(appId, data, auditSource, callback) {
+function restore(appId, backupId, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -1403,7 +1414,7 @@ function restore(appId, data, auditSource, callback) {
         if (error) return callback(error);
 
         // for empty or null backupId, use existing manifest to mimic a reinstall
-        var func = data.backupId ? backups.get.bind(null, data.backupId) : function (next) { return next(null, { manifest: app.manifest }); };
+        var func = backupId ? backups.get.bind(null, backupId) : function (next) { return next(null, { manifest: app.manifest }); };
 
         func(function (error, backupInfo) {
             if (error) return callback(error);
@@ -1414,11 +1425,12 @@ function restore(appId, data, auditSource, callback) {
             error = checkManifestConstraints(backupInfo.manifest);
             if (error) return callback(error);
 
-            const restoreConfig = { backupId: data.backupId, backupFormat: backupInfo.format, oldManifest: app.manifest };
+            const restoreConfig = { backupId, backupFormat: backupInfo.format };
 
             const task = {
                 args: {
                     restoreConfig,
+                    oldManifest: app.manifest,
                     overwriteDns: true
                 },
                 values: {
@@ -1447,28 +1459,41 @@ function importApp(appId, data, auditSource, callback) {
     get(appId, function (error, app) {
         if (error) return callback(error);
 
-        error = validateBackupFormat(data.backupFormat);
+        // all fields are optional
+        data.backupId = data.backupId || null;
+        data.backupFormat = data.backupFormat || null;
+        data.backupConfig = data.backupConfig || null;
+        const { backupId, backupFormat, backupConfig } = data;
+
+        error = backupFormat ? validateBackupFormat(backupFormat) : null;
         if (error) return callback(error);
 
         error = checkAppState(app, exports.ISTATE_PENDING_RESTORE);
         if (error) return callback(error);
 
-        // TODO: check if the file exists in the storage backend
-        const restoreConfig = { backupId: data.backupId, backupFormat: data.backupFormat, oldManifest: app.manifest };
+        // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
+        const testBackupConfig = backupConfig ? backups.testProviderConfig.bind(null, backupConfig) : (next) => next();
 
-        const task = {
-            args: {
-                restoreConfig,
-                overwriteDns: true
-            },
-            values: {}
-        };
-        addTask(appId, exports.ISTATE_PENDING_RESTORE, task, function (error, result) {
+        testBackupConfig(function (error) {
             if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId: data.backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId: result.taskId });
+            const restoreConfig = { backupId, backupFormat, backupConfig };
 
-            callback(null, { taskId: result.taskId });
+            const task = {
+                args: {
+                    restoreConfig,
+                    oldManifest: app.manifest,
+                    overwriteDns: true
+                },
+                values: {}
+            };
+            addTask(appId, exports.ISTATE_PENDING_RESTORE, task, function (error, result) {
+                if (error) return callback(error);
+
+                eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId: result.taskId });
+
+                callback(null, { taskId: result.taskId });
+            });
         });
     });
 }
@@ -1526,7 +1551,7 @@ function clone(appId, data, user, auditSource, callback) {
             error = validatePortBindings(portBindings, manifest);
             if (error) return callback(error);
 
-            const mailboxName = mailboxNameForLocation(location, manifest);
+            const mailboxName = app.mailboxName.endsWith('.app') ? mailboxNameForLocation(location, manifest) : app.mailboxName;
             const locations = [{subdomain: location, domain}];
             validateLocations(locations, function (error, domainObjectMap) {
                 if (error) return callback(error);
@@ -1554,9 +1579,9 @@ function clone(appId, data, user, auditSource, callback) {
                     purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' }, function (error) {
                         if (error) return callback(error);
 
-                        const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format, oldManifest: null };
+                        const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format };
                         const task = {
-                            args: { restoreConfig, overwriteDns },
+                            args: { restoreConfig, overwriteDns, oldManifest: null },
                             values: {},
                             requiredState: exports.ISTATE_PENDING_CLONE
                         };
@@ -1657,6 +1682,30 @@ function stop(appId, callback) {
     });
 }
 
+function restart(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will restart app with id:%s', appId);
+
+    get(appId, function (error, app) {
+        if (error) return callback(error);
+
+        error = checkAppState(app, exports.ISTATE_PENDING_RESTART);
+        if (error) return callback(error);
+
+        const task = {
+            args: {},
+            values: { runState: exports.RSTATE_RUNNING }
+        };
+        addTask(appId, exports.ISTATE_PENDING_RESTART, task, function (error, result) {
+            if (error) return callback(error);
+
+            callback(null, { taskId: result.taskId });
+        });
+    });
+}
+
 function checkManifestConstraints(manifest) {
     assert(manifest && typeof manifest === 'object');
 
@@ -1690,8 +1739,6 @@ function exec(appId, options, callback) {
             return callback(new BoxError(BoxError.BAD_STATE, 'App not installed or running'));
         }
 
-        var container = docker.connection.getContainer(app.containerId);
-
         var execOptions = {
             AttachStdin: true,
             AttachStdout: true,
@@ -1704,55 +1751,46 @@ function exec(appId, options, callback) {
             Cmd: cmd
         };
 
-        container.exec(execOptions, function (error, exec) {
+        var startOptions = {
+            Detach: false,
+            Tty: options.tty,
+            // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
+            // we can work with half-close connections (not defined in http). this way, the client
+            // can properly signal that stdin is EOF by closing it's side of the socket. In http,
+            // the whole connection will be dropped when stdin get EOF.
+            // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
+            hijack: true,
+            stream: true,
+            stdin: true,
+            stdout: true,
+            stderr: true
+        };
+
+        docker.execContainer(app.containerId, { execOptions, startOptions, rows: options.rows, columns: options.columns }, function (error, stream) {
             if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running
             if (error) return callback(error);
 
-            var startOptions = {
-                Detach: false,
-                Tty: options.tty,
-                // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
-                // we can work with half-close connections (not defined in http). this way, the client
-                // can properly signal that stdin is EOF by closing it's side of the socket. In http,
-                // the whole connection will be dropped when stdin get EOF.
-                // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
-                hijack: true,
-                stream: true,
-                stdin: true,
-                stdout: true,
-                stderr: true
-            };
-            exec.start(startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
-                if (error) return callback(error);
-
-                if (options.rows && options.columns) {
-                    // there is a race where resizing too early results in a 404 "no such exec"
-                    // https://git.cloudron.io/cloudron/box/issues/549
-                    setTimeout(function () {
-                        exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
-                    }, 2000);
-                }
-
-                return callback(null, stream);
-            });
+            callback(null, stream);
         });
     });
 }
 
 function canAutoupdateApp(app, newManifest) {
-    if (!app.enableAutomaticUpdate) return new Error('Automatic update disabled');
-    if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(newManifest.version))) return new Error('Major version change'); // major changes are blocking
+    if (!app.enableAutomaticUpdate) return false;
+    if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(newManifest.version))) return false; // major changes are blocking
+
+    if (app.runState === exports.RSTATE_STOPPED) return false; // stopped apps won't run migration scripts and shouldn't be updated
 
     const newTcpPorts = newManifest.tcpPorts || { };
     const newUdpPorts = newManifest.udpPorts || { };
     const portBindings = app.portBindings; // this is never null
 
     for (let portName in portBindings) {
-        if (!(portName in newTcpPorts) && !(portName in newUdpPorts)) return new Error(`${portName} was in use but new update removes it`);
+        if (!(portName in newTcpPorts) && !(portName in newUdpPorts)) return false; // portName was in use but new update removes it
     }
 
     // it's fine if one or more (unused) keys got removed
-    return null;
+    return true;
 }
 
 function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is { appId -> { manifest } }
@@ -1769,9 +1807,8 @@ function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is {
                 return iteratorDone();
             }
 
-            error = canAutoupdateApp(app, updateInfo[appId].manifest);
-            if (error) {
-                debug('app %s requires manual update. %s', appId, error.message);
+            if (!canAutoupdateApp(app, updateInfo[appId].manifest)) {
+                debug(`app ${app.fqdn} requires manual update`);
                 return iteratorDone();
             }
 
@@ -1841,17 +1878,19 @@ function restoreInstalledApps(callback) {
 
         async.eachSeries(apps, function (app, iteratorDone) {
             backups.getByAppIdPaged(1, 1, app.id, function (error, results) {
-                let installationState, restoreConfig;
+                let installationState, restoreConfig, oldManifest;
                 if (!error && results.length) {
                     installationState = exports.ISTATE_PENDING_RESTORE;
-                    restoreConfig = { backupId: results[0].id, backupFormat: results[0].format, oldManifest: app.manifest };
+                    restoreConfig = { backupId: results[0].id, backupFormat: results[0].format };
+                    oldManifest = app.manifest;
                 } else {
                     installationState = exports.ISTATE_PENDING_INSTALL;
                     restoreConfig = null;
+                    oldManifest = null;
                 }
 
                 const task = {
-                    args: { restoreConfig, overwriteDns: true },
+                    args: { restoreConfig, overwriteDns: true, oldManifest },
                     values: {},
                     scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                     requireNullTaskId: false // ignore existing stale taskId
@@ -1882,9 +1921,8 @@ function configureInstalledApps(callback) {
         async.eachSeries(apps, function (app, iteratorDone) {
             debug(`configureInstalledApps: marking ${app.fqdn} for reconfigure`);
 
-            const oldConfig = _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'portBindings');
             const task = {
-                args: { oldConfig, overwriteDns: true },
+                args: {},
                 values: {},
                 scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                 requireNullTaskId: false // ignore existing stale taskId
@@ -1994,7 +2032,8 @@ function uploadFile(appId, sourceFilePath, destFilePath, callback) {
 
     const done = once(function (error) {
         safe.fs.unlinkSync(sourceFilePath); // remove file in /tmp
-        callback(error);
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, error.message)); // blame it on filesystem for now
+        callback(null);
     });
 
     // the built-in bash printf understands "%q" but not /usr/bin/printf.

src/appstore.js

@@ -5,6 +5,9 @@ exports = module.exports = {
     getApp: getApp,
     getAppVersion: getAppVersion,
 
+    trackBeginSetup: trackBeginSetup,
+    trackFinishedSetup: trackFinishedSetup,
+
     registerWithLoginCredentials: registerWithLoginCredentials,
     registerWithLicense: registerWithLicense,
 
@@ -375,6 +378,35 @@ function registerCloudron(data, callback) {
     });
 }
 
+// This works without a Cloudron token as this Cloudron was not yet registered
+let gBeginSetupAlreadyTracked = false;
+function trackBeginSetup(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
+    // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
+    if (gBeginSetupAlreadyTracked) return;
+    gBeginSetupAlreadyTracked = true;
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
+
+    superagent.post(url).send({ provider }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
+// This works without a Cloudron token as this Cloudron was not yet registered
+function trackFinishedSetup(domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;
+
+    superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
 function registerWithLicense(license, domain, callback) {
     assert.strictEqual(typeof license, 'string');
     assert.strictEqual(typeof domain, 'string');
@@ -383,7 +415,10 @@ function registerWithLicense(license, domain, callback) {
     getCloudronToken(function (error, token) {
         if (token) return callback(new BoxError(BoxError.CONFLICT));
 
-        registerCloudron({ license, domain }, callback);
+        const provider = settings.provider();
+        const version = constants.VERSION;
+
+        registerCloudron({ license, domain, provider, version }, callback);
     });
 }
 
@@ -406,19 +441,20 @@ function registerWithLoginCredentials(options, callback) {
             login(options.email, options.password, options.totpToken || '', function (error, result) {
                 if (error) return callback(error);
 
-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken }, callback);
+                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, provider: settings.provider(), version: constants.VERSION }, callback);
             });
         });
     });
 }
 
-function createTicket(info, callback) {
+function createTicket(info, auditSource, callback) {
     assert.strictEqual(typeof info, 'object');
     assert.strictEqual(typeof info.email, 'string');
     assert.strictEqual(typeof info.displayName, 'string');
     assert.strictEqual(typeof info.type, 'string');
     assert.strictEqual(typeof info.subject, 'string');
     assert.strictEqual(typeof info.description, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     function collectAppInfoIfNeeded(callback) {
@@ -443,6 +479,8 @@ function createTicket(info, callback) {
                 if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
                 if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
+                eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+
                 callback(null);
             });
         });

src/apptask.js

@@ -64,11 +64,13 @@ function debugApp(app) {
 }
 
 function makeTaskError(error, app) {
-    let boxError = error instanceof BoxError ? error : new BoxError(BoxError.UNKNOWN_ERROR, error.message); // until we port everything to BoxError
+    assert.strictEqual(typeof error, 'object');
+    assert.strictEqual(typeof app, 'object');
+
     // track a few variables which helps 'repair' restart the task (see also scheduleTask in apps.js)
-    boxError.details.taskId = app.taskId;
-    boxError.details.installationState = app.installationState;
-    return boxError.toPlainObject();
+    error.details.taskId = app.taskId;
+    error.details.installationState = app.installationState;
+    return error.toPlainObject();
 }
 
 // updates the app object and the database
@@ -459,16 +461,18 @@ function waitForDnsPropagation(app, callback) {
     });
 }
 
-function moveDataDir(app, sourceDir, callback) {
+function moveDataDir(app, targetDir, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert(sourceDir === null || typeof sourceDir === 'string');
+    assert(targetDir === null || typeof targetDir === 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let resolvedSourceDir = apps.getDataDir(app, sourceDir);
-    let resolvedTargetDir = apps.getDataDir(app, app.dataDir);
+    let resolvedSourceDir = apps.getDataDir(app, app.dataDir);
+    let resolvedTargetDir = apps.getDataDir(app, targetDir);
 
     debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
 
+    if (resolvedSourceDir === resolvedTargetDir) return callback();
+
     shell.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}, function (error) {
         if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`));
 
@@ -503,17 +507,6 @@ function startApp(app, callback){
     docker.startContainer(app.id, callback);
 }
 
-// Ordering is based on the following rationale:
-//   - configure nginx, icon, oauth
-//   - register subdomain.
-//          at this point, the user can visit the site and the above nginx config can show some install screen.
-//          the icon can be displayed in this nginx page and oauth proxy means the page can be protected
-//   - download image
-//   - setup volumes
-//   - setup addons (requires the above volume)
-//   - setup the container (requires image, volumes, addons)
-//   - setup collectd (requires container id)
-// restore is also handled here since restore is just an install with some oldConfig to clean up
 function install(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
@@ -522,6 +515,7 @@ function install(app, args, progressCallback, callback) {
 
     const restoreConfig = args.restoreConfig; // has to be set when restoring
     const overwriteDns = args.overwriteDns;
+    const oldManifest = args.oldManifest;
 
     async.series([
         // this protects against the theoretical possibility of an app being marked for install/restore from
@@ -535,22 +529,25 @@ function install(app, args, progressCallback, callback) {
         function teardownAddons(next) {
             // when restoring, app does not require these addons anymore. remove carefully to preserve the db passwords
             let addonsToRemove;
-            if (restoreConfig && restoreConfig.oldManifest) { // oldManifest is null for clone
-                addonsToRemove = _.omit(restoreConfig.oldManifest.addons, Object.keys(app.manifest.addons));
+            if (oldManifest) {
+                addonsToRemove = _.omit(oldManifest.addons, Object.keys(app.manifest.addons));
             } else {
                 addonsToRemove = app.manifest.addons;
             }
 
             addons.teardownAddons(app, addonsToRemove, next);
         },
-        deleteAppDir.bind(null, app, { removeDirectory: false }), // do not remove any symlinked appdata dir
+
+        function deleteAppDirIfNeeded(done) {
+            if (restoreConfig && !restoreConfig.backupId) return done(); // in-place import should not delete data dir
+
+            deleteAppDir(app, { removeDirectory: false }, done); // do not remove any symlinked appdata dir
+        },
 
         function deleteImageIfChanged(done) {
-            if (!restoreConfig || !restoreConfig.oldManifest) return done();
+            if (!oldManifest || oldManifest.dockerImage === app.manifest.dockerImage) return done();
 
-            if (restoreConfig.oldManifest.dockerImage === app.manifest.dockerImage) return done();
-
-            docker.deleteImage(restoreConfig.oldManifest, done);
+            docker.deleteImage(oldManifest, done);
         },
 
         reserveHttpPort.bind(null, app),
@@ -573,14 +570,22 @@ function install(app, args, progressCallback, callback) {
                     progressCallback.bind(null, { percent: 60, message: 'Setting up addons' }),
                     addons.setupAddons.bind(null, app, app.manifest.addons),
                 ], next);
+            } else if (!restoreConfig.backupId) { // in-place import
+                async.series([
+                    progressCallback.bind(null, { percent: 60, message: 'Importing addons in-place' }),
+                    addons.setupAddons.bind(null, app, app.manifest.addons),
+                    addons.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
+                    addons.restoreAddons.bind(null, app, app.manifest.addons),
+                ], next);
             } else {
                 async.series([
                     progressCallback.bind(null, { percent: 65, message: 'Download backup and restoring addons' }),
                     addons.setupAddons.bind(null, app, app.manifest.addons),
                     addons.clearAddons.bind(null, app, app.manifest.addons),
-                    backups.restoreApp.bind(null, app, app.manifest.addons, restoreConfig, (progress) => {
-                        progressCallback({ percent: 65, message: `Restore - ${progress.message}` });
-                    })
+                    backups.downloadApp.bind(null, app, restoreConfig, (progress) => {
+                        progressCallback({ percent: 65, message: progress.message });
+                    }),
+                    addons.restoreAddons.bind(null, app, app.manifest.addons)
                 ], next);
             }
         },
@@ -624,8 +629,8 @@ function backup(app, args, progressCallback, callback) {
     ], function seriesDone(error) {
         if (error) {
             debugApp(app, 'error backing up app: %s', error);
-            // return to installed state intentionally
-            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: error.toPlainObject ? error.toPlainObject() : error.message }, callback.bind(null, error));
+            // return to installed state intentionally. the error is stashed only in the task and not the app (the UI shows error state otherwise)
+            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }, callback.bind(null, makeTaskError(error, app)));
         }
         callback(null);
     });
@@ -709,7 +714,7 @@ function changeLocation(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error reconfiguring : %s', error);
+            debugApp(app, 'error changing location : %s', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -722,8 +727,8 @@ function migrateDataDir(app, args, progressCallback, callback) {
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    let oldDataDir = args.oldDataDir;
-    assert(oldDataDir === null || typeof oldDataDir === 'string');
+    let newDataDir = args.newDataDir;
+    assert(newDataDir === null || typeof newDataDir === 'string');
 
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
@@ -734,69 +739,43 @@ function migrateDataDir(app, args, progressCallback, callback) {
 
         // re-setup addons since this creates the localStorage volume
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        addons.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),
 
-        // migrate dataDir
-        function (next) {
-            const dataDirChanged = oldDataDir !== app.dataDir;
+        progressCallback.bind(null, { percent: 60, message: 'Moving data dir' }),
+        moveDataDir.bind(null, app, newDataDir),
 
-            if (!dataDirChanged) return next();
-
-            moveDataDir(app, oldDataDir, next);
-        },
-
-        progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
+        progressCallback.bind(null, { percent: 90, message: 'Creating container' }),
         createContainer.bind(null, app),
 
         startApp.bind(null, app),
 
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
-        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
+        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error reconfiguring : %s', error);
+            debugApp(app, 'error migrating data dir : %s', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
     });
 }
 
-// configure is called for an infra update and repair
+// configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local"
 function configure(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    const oldConfig = args.oldConfig || null;
-    const overwriteDns = args.overwriteDns;
-
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
         unconfigureReverseProxy.bind(null, app),
         deleteContainers.bind(null, app, { managedOnly: true }),
-        function (next) {
-            if (!oldConfig) return next();
-
-            let obsoleteDomains = oldConfig.alternateDomains.filter(function (o) {
-                return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
-            });
-
-            if (oldConfig.fqdn !== app.fqdn) obsoleteDomains.push({ subdomain: oldConfig.location, domain: oldConfig.domain });
-
-            if (obsoleteDomains.length === 0) return next();
-
-            unregisterSubdomains(app, obsoleteDomains, next);
-        },
-
         reserveHttpPort.bind(null, app),
 
         progressCallback.bind(null, { percent: 20, message: 'Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
-        registerSubdomains.bind(null, app, overwriteDns),
-
         progressCallback.bind(null, { percent: 40, message: 'Downloading image' }),
         downloadImage.bind(null, app.manifest),
 
@@ -846,7 +825,7 @@ function update(app, args, progressCallback, callback) {
     async.series([
         // this protects against the theoretical possibility of an app being marked for update from
         // a previous version of box code
-        progressCallback.bind(null, { percent: 0, message: 'Verify manifest' }),
+        progressCallback.bind(null, { percent: 5, message: 'Verify manifest' }),
         verifyManifest.bind(null, updateConfig.manifest),
 
         function (next) {
@@ -973,6 +952,27 @@ function stop(app, args, progressCallback, callback) {
     });
 }
 
+function restart(app, args, progressCallback, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        progressCallback.bind(null, { percent: 20, message: 'Restarting container' }),
+        docker.restartContainer.bind(null, app.id),
+
+        progressCallback.bind(null, { percent: 100, message: 'Done' }),
+        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error starting app: %s', error);
+            return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
+        }
+        callback(null);
+    });
+}
+
 function uninstall(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
@@ -1050,9 +1050,11 @@ function run(appId, args, progressCallback, callback) {
             return start(app, args, progressCallback, callback);
         case apps.ISTATE_PENDING_STOP:
             return stop(app, args, progressCallback, callback);
+        case apps.ISTATE_PENDING_RESTART:
+            return restart(app, args, progressCallback, callback);
         default:
             debugApp(app, 'apptask launched with invalid command');
-            return callback(new Error('Unknown install command in apptask:' + app.installationState));
+            return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState));
         }
     });
 }

src/apptaskmanager.js

@@ -5,6 +5,7 @@ exports = module.exports = {
 };
 
 let assert = require('assert'),
+    BoxError = require('./boxerror.js'),
     debug = require('debug')('box:apptaskmanager'),
     fs = require('fs'),
     locker = require('./locker.js'),
@@ -42,12 +43,12 @@ function scheduleTask(appId, taskId, callback) {
     if (!gInitialized) initializeSync();
 
     if (appId in gActiveTasks) {
-        return callback(new Error(`Task for %s is already active: ${appId}`));
+        return callback(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
     }
 
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 0, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
+        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
         gPendingTasks.push({ appId, taskId, callback });
         return;
     }
@@ -56,7 +57,7 @@ function scheduleTask(appId, taskId, callback) {
 
     if (lockError) {
         debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 0, message: waitText(lockError.operation) }, NOOP_CALLBACK);
+        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
         gPendingTasks.push({ appId, taskId, callback });
         return;
     }

src/backups.js

@@ -2,6 +2,7 @@
 
 exports = module.exports = {
     testConfig: testConfig,
+    testProviderConfig: testProviderConfig,
 
     getByStatePaged: getByStatePaged,
     getByAppIdPaged: getByAppIdPaged,
@@ -14,7 +15,7 @@ exports = module.exports = {
     restore: restore,
 
     backupApp: backupApp,
-    restoreApp: restoreApp,
+    downloadApp: downloadApp,
 
     backupBoxAndApps: backupBoxAndApps,
 
@@ -118,6 +119,17 @@ function testConfig(backupConfig, callback) {
     api(backupConfig.provider).testConfig(backupConfig, callback);
 }
 
+
+function testProviderConfig(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var func = api(backupConfig.provider);
+    if (!func) return callback(new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' }));
+
+    api(backupConfig.provider).testConfig(backupConfig, callback);
+}
+
 function getByStatePaged(state, page, perPage, callback) {
     assert.strictEqual(typeof state, 'string');
     assert(typeof page === 'number' && page > 0);
@@ -217,14 +229,14 @@ function createReadStream(sourceFile, key) {
 
     stream.on('error', function (error) {
         debug('createReadStream: read stream error.', error);
-        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, error.message));
     });
 
     if (key !== null) {
         var encrypt = crypto.createCipher('aes-256-cbc', key);
         encrypt.on('error', function (error) {
             debug('createReadStream: encrypt stream error.', error);
-            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
         });
         return stream.pipe(encrypt).pipe(ps);
     } else {
@@ -237,17 +249,25 @@ function createWriteStream(destFile, key) {
     assert(key === null || typeof key === 'string');
 
     var stream = fs.createWriteStream(destFile);
+    var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+
+    stream.on('error', function (error) {
+        debug('createWriteStream: write stream error.', error);
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, error.message));
+    });
 
     if (key !== null) {
         var decrypt = crypto.createDecipher('aes-256-cbc', key);
         decrypt.on('error', function (error) {
             debug('createWriteStream: decrypt stream error.', error);
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
         });
-        decrypt.pipe(stream);
-        return decrypt;
+        ps.pipe(decrypt).pipe(stream);
     } else {
-        return stream;
+        ps.pipe(stream);
     }
+
+    return ps;
 }
 
 function tarPack(dataLayout, key, callback) {
@@ -338,7 +358,7 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
                     debug(`read stream error for ${task.path}: ${error.message}`);
                     retryCallback();
                 }); // ignore error if file disappears
-                stream.on('progress', function(progress) {
+                stream.on('progress', function (progress) {
                     const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
                     if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0Mbps looks wrong
                     progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}Mbps` }); // 0M@0Mbps looks wrong
@@ -437,7 +457,7 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback)
                     tarPack(dataLayout, backupConfig.key || null, function (error, tarStream) {
                         if (error) return retryCallback(error);
 
-                        tarStream.on('progress', function(progress) {
+                        tarStream.on('progress', function (progress) {
                             const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
                             if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0Mbps looks wrong
                             progressCallback({ message: `Uploading backup ${transferred}M@${speed}Mbps` });
@@ -545,24 +565,30 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
 
     debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
 
-    function downloadFile(entry, callback) {
+    function downloadFile(entry, done) {
         let relativePath = path.relative(backupFilePath, entry.fullPath);
         if (backupConfig.key) {
             relativePath = decryptFilePath(relativePath, backupConfig.key);
-            if (!relativePath) return callback(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file'));
+            if (!relativePath) return done(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file'));
         }
         const destFilePath = dataLayout.toLocalPath('./' + relativePath);
 
         mkdirp(path.dirname(destFilePath), function (error) {
-            if (error) return callback(new BoxError(BoxError.FS_ERROR, error.message));
+            if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));
 
             async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
                 let destStream = createWriteStream(destFilePath, backupConfig.key || null);
 
+                destStream.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0Mbps looks wrong
+                    progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}Mbps` });
+                });
+
                 // protect against multiple errors. must destroy the write stream so that a previous retry does not write
                 let closeAndRetry = once((error) => {
-                    if (error) progressCallback({ message: `Download ${entry.fullPath} errored: ${error.message}` });
-                    else progressCallback({ message: `Download ${entry.fullPath} finished` });
+                    if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                    else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` });
                     destStream.destroy();
                     retryCallback(error);
                 });
@@ -571,21 +597,21 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
                     if (error) return closeAndRetry(error);
 
                     sourceStream.on('error', closeAndRetry);
-                    destStream.on('error', closeAndRetry);
+                    destStream.on('error', closeAndRetry); // already emits BoxError
 
                     progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
 
                     sourceStream.pipe(destStream, { end: true }).on('finish', closeAndRetry);
                 });
-            }, callback);
+            }, done);
         });
     }
 
-    api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, done) {
+    api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
         // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
         const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
 
-        async.eachLimit(entries, concurrency, downloadFile, done);
+        async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
     }, callback);
 }
 
@@ -597,7 +623,7 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback,
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`download - Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`);
+    debug(`download: Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`);
 
     const backupFilePath = getBackupFilePath(backupConfig, backupId, format);
 
@@ -651,9 +677,8 @@ function restore(backupConfig, backupId, progressCallback, callback) {
     });
 }
 
-function restoreApp(app, addonsToRestore, restoreConfig, progressCallback, callback) {
+function downloadApp(app, restoreConfig, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof addonsToRestore, 'object');
     assert.strictEqual(typeof restoreConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
@@ -662,30 +687,32 @@ function restoreApp(app, addonsToRestore, restoreConfig, progressCallback, callb
     if (!appDataDir) return callback(safe.error);
     const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
 
-    var startTime = new Date();
+    const startTime = new Date();
+    const getBackupConfigFunc = restoreConfig.backupConfig ? (next) => next(null, restoreConfig.backupConfig) : settings.getBackupConfig;
 
-    settings.getBackupConfig(function (error, backupConfig) {
+    getBackupConfigFunc(function (error, backupConfig) {
         if (error) return callback(error);
 
-        async.series([
-            download.bind(null, backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback),
-            addons.restoreAddons.bind(null, app, addonsToRestore)
-        ], function (error) {
-            debug('restoreApp: time: %s', (new Date() - startTime)/1000);
+        download(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback, function (error) {
+            debug('downloadApp: time: %s', (new Date() - startTime)/1000);
 
             callback(error);
         });
     });
 }
 
-function runBackupUpload(backupId, format, dataLayout, progressCallback, callback) {
-    assert.strictEqual(typeof backupId, 'string');
-    assert.strictEqual(typeof format, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+function runBackupUpload(uploadConfig, progressCallback, callback) {
+    assert.strictEqual(typeof uploadConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    let result = '';
+    const { backupId, format, dataLayout, progressTag } = uploadConfig;
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof format, 'string');
+    assert.strictEqual(typeof progressTag, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    let result = ''; // the script communicates error result as a string
 
     shell.sudo(`backup-${backupId}`, [ BACKUP_UPLOAD_CMD, backupId, format, dataLayout.toString() ], { preserveEnv: true, ipc: true }, function (error) {
         if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
@@ -695,10 +722,10 @@ function runBackupUpload(backupId, format, dataLayout, progressCallback, callbac
         }
 
         callback();
-    }).on('message', function (message) {
-        if (!message.result) return progressCallback(message);
-        debug(`runBackupUpload: result - ${JSON.stringify(message)}`);
-        result = message.result;
+    }).on('message', function (progress) { // script sends either 'message' or 'result' property
+        if (!progress.result) return progressCallback({ message: `${progress.message} (${progressTag})` });
+        debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
+        result = progress.result;
     });
 }
 
@@ -752,8 +779,14 @@ function uploadBoxSnapshot(backupConfig, progressCallback, callback) {
         const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
         if (!boxDataDir) return callback(safe.error);
 
-        const dataLayout = new DataLayout(boxDataDir, []);
-        runBackupUpload('snapshot/box', backupConfig.format, dataLayout, progressCallback, function (error) {
+        const uploadConfig = {
+            backupId: 'snapshot/box',
+            format: backupConfig.format,
+            dataLayout: new DataLayout(boxDataDir, []),
+            progressTag: 'box'
+        };
+
+        runBackupUpload(uploadConfig, progressCallback, function (error) {
             if (error) return callback(error);
 
             debug('uploadBoxSnapshot: time: %s secs', (new Date() - startTime)/1000);
@@ -782,7 +815,7 @@ function rotateBoxBackup(backupConfig, tag, appBackupIds, progressCallback, call
         if (error) return callback(error);
 
         var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message }));
+        copy.on('progress', (message) => progressCallback({ message: `box: ${message}` }));
         copy.on('done', function (copyBackupError) {
             const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
 
@@ -863,7 +896,7 @@ function rotateAppBackup(backupConfig, app, tag, options, progressCallback, call
         if (error) return callback(error);
 
         var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message }));
+        copy.on('progress', (message) => progressCallback({ message: `${message} (${app.fqdn})` }));
         copy.on('done', function (copyBackupError) {
             const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
 
@@ -898,7 +931,14 @@ function uploadAppSnapshot(backupConfig, app, progressCallback, callback) {
 
         const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
 
-        runBackupUpload(backupId, backupConfig.format, dataLayout, progressCallback, function (error) {
+        const uploadConfig = {
+            backupId,
+            format: backupConfig.format,
+            dataLayout,
+            progressTag: app.fqdn
+        };
+
+        runBackupUpload(uploadConfig, progressCallback, function (error) {
             if (error) return callback(error);
 
             debugApp(app, 'uploadAppSnapshot: %s done time: %s secs', backupId, (new Date() - startTime)/1000);

src/boxerror.js

@@ -33,6 +33,7 @@ function BoxError(reason, errorOrMessage, details) {
 }
 util.inherits(BoxError, Error);
 BoxError.ACCESS_DENIED = 'Access Denied';
+BoxError.ADDONS_ERROR = 'Addons Error';
 BoxError.ALREADY_EXISTS = 'Already Exists';
 BoxError.BAD_FIELD = 'Bad Field';
 BoxError.BAD_STATE = 'Bad State';
@@ -58,9 +59,10 @@ BoxError.NOT_IMPLEMENTED = 'Not implemented';
 BoxError.NOT_SIGNED = 'Not Signed';
 BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.PLAN_LIMIT = 'Plan Limit';
+BoxError.SPAWN_ERROR = 'Spawn Error';
 BoxError.TASK_ERROR = 'Task Error';
+BoxError.TIMEOUT = 'Timeout';
 BoxError.TRY_AGAIN = 'Try Again';
-BoxError.UNKNOWN_ERROR = 'Unknown Error'; // only used for porting
 
 BoxError.prototype.toPlainObject = function () {
     return _.extend({}, { message: this.message, reason: this.reason }, this.details);
@@ -86,6 +88,7 @@ BoxError.toHttpError = function (error) {
     case BoxError.FS_ERROR:
     case BoxError.MAIL_ERROR:
     case BoxError.DOCKER_ERROR:
+    case BoxError.ADDONS_ERROR:
         return new HttpError(424, error);
     case BoxError.DATABASE_ERROR:
     case BoxError.INTERNAL_ERROR:

src/cert/acme2.js

@@ -9,8 +9,8 @@ var assert = require('assert'),
     fs = require('fs'),
     path = require('path'),
     paths = require('../paths.js'),
+    request = require('request'),
     safe = require('safetydance'),
-    superagent = require('superagent'),
     util = require('util'),
     _ = require('underscore');
 
@@ -41,15 +41,6 @@ function Acme2(options) {
     this.wildcard = !!options.wildcard;
 }
 
-Acme2.prototype.getNonce = function (callback) {
-    superagent.get(this.directory.newNonce).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(error);
-        if (response.statusCode !== 204) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
-
-        return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
-    });
-};
-
 // urlsafe base64 encoding (jose)
 function urlBase64Encode(string) {
     return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
@@ -96,8 +87,12 @@ Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
 
     var payload64 = b64(payload);
 
-    this.getNonce(function (error, nonce) {
-        if (error) return callback(error);
+    request.get(this.directory.newNonce, { json: true, timeout: 30000 }, function (error, response) {
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`));
+        if (response.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching nonce : ' + response.statusCode));
+
+        const nonce = response.headers['Replay-Nonce'.toLowerCase()];
+        if (!nonce) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response'));
 
         debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
 
@@ -113,14 +108,23 @@ Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
             signature: signature64
         };
 
-        superagent.post(url).set('Content-Type', 'application/jose+json').set('User-Agent', 'acme-cloudron').send(JSON.stringify(data)).timeout(30 * 1000).end(function (error, res) {
-            if (error && !error.response) return callback(error); // network errors
+        request.post(url, { headers: { 'Content-Type': 'application/jose+json', 'User-Agent': 'acme-cloudron' }, body: JSON.stringify(data), timeout: 30000 }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`)); // network error
 
-            callback(null, res);
+            // we don't set json: true in request because it ends up mangling the content-type
+            // we don't set json: true in request because it ends up mangling the content-type
+            if (response.headers['content-type'] === 'application/json') response.body = safe.JSON.parse(response.body);
+
+            callback(null, response);
         });
     });
 };
 
+// https://tools.ietf.org/html/rfc8555#section-6.3
+Acme2.prototype.postAsGet = function (url, callback) {
+    this.sendSignedRequest(url, '', callback);
+};
+
 Acme2.prototype.updateContact = function (registrationUri, callback) {
     assert.strictEqual(typeof registrationUri, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -134,7 +138,7 @@ Acme2.prototype.updateContact = function (registrationUri, callback) {
 
     const that = this;
     this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when updating contact: ${error.message}`));
+        if (error) return callback(error);
         if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 200, got %s %s', result.statusCode, result.text)));
 
         debug(`updateContact: contact of user updated to ${that.email}`);
@@ -154,7 +158,7 @@ Acme2.prototype.registerUser = function (callback) {
 
     var that = this;
     this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when registering user: ${error.message}`));
+        if (error) return callback(error);
         // 200 if already exists. 201 for new accounts
         if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register new account. Expecting 200 or 201, got %s %s', result.statusCode, result.text)));
 
@@ -180,7 +184,7 @@ Acme2.prototype.newOrder = function (domain, callback) {
     debug('newOrder: %s', domain);
 
     this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when creating new order: ${error.message}`));
+        if (error) return callback(error);
         if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending signed request: ${result.body.detail}`));
         if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
 
@@ -201,14 +205,15 @@ Acme2.prototype.waitForOrder = function (orderUrl, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     debug(`waitForOrder: ${orderUrl}`);
+    const that = this;
 
     async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
         debug('waitForOrder: getting status');
 
-        superagent.get(orderUrl).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) {
+        that.postAsGet(orderUrl, function (error, result) {
+            if (error) {
                 debug('waitForOrder: network error getting uri %s', orderUrl);
-                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for order: ${error.message}`)); // network error
+                return retryCallback(error);
             }
             if (result.statusCode !== 200) {
                 debug('waitForOrder: invalid response code getting uri %s', result.statusCode);
@@ -253,7 +258,7 @@ Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
     };
 
     this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when notifying challenge: ${error.message}`));
+        if (error) return callback(error);
         if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 200, got %s %s', result.statusCode, result.text)));
 
         callback();
@@ -265,14 +270,15 @@ Acme2.prototype.waitForChallenge = function (challenge, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     debug('waitingForChallenge: %j', challenge);
+    const that = this;
 
     async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
         debug('waitingForChallenge: getting status');
 
-        superagent.get(challenge.url).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) {
+        that.postAsGet(challenge.url, function (error, result) {
+            if (error) {
                 debug('waitForChallenge: network error getting uri %s', challenge.url);
-                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for challenge: ${error.message}`));
+                return retryCallback(error);
             }
             if (result.statusCode !== 200) {
                 debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
@@ -305,7 +311,7 @@ Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, cal
     debug('signCertificate: sending sign request');
 
     this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when signing certificate: ${error.message}`));
+        if (error) return callback(error);
         // 429 means we reached the cert limit for this domain
         if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 200, got %s %s', result.statusCode, result.text)));
 
@@ -348,20 +354,17 @@ Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var outdir = paths.APP_CERTS_DIR;
+    const that = this;
 
     async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
         debug('downloadCertificate: downloading certificate');
 
-        superagent.get(certUrl).buffer().parse(function (res, done) {
-            var data = [ ];
-            res.on('data', function(chunk) { data.push(chunk); });
-            res.on('end', function () { res.text = Buffer.concat(data); done(); });
-        }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
-            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry'));
+        that.postAsGet(certUrl, function (error, result) {
+            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
+            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
             if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
 
-            const fullChainPem = result.text;
+            const fullChainPem = result.body; // buffer
 
             const certName = hostname.replace('*.', '_.');
             var certificateFile = path.join(outdir, `${certName}.cert`);
@@ -488,8 +491,8 @@ Acme2.prototype.prepareChallenge = function (hostname, domain, authorizationUrl,
     debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
 
     const that = this;
-    superagent.get(authorizationUrl).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when preparing challenge: ${error.message}`));
+    this.postAsGet(authorizationUrl, function (error, response) {
+        if (error) return callback(error);
         if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code getting authorization : ' + response.statusCode));
 
         const authorization = response.body;
@@ -569,13 +572,13 @@ Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
 Acme2.prototype.getDirectory = function (callback) {
     const that = this;
 
-    superagent.get(this.caDirectory).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
+    request.get(this.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
         if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
 
         if (typeof response.body.newNonce !== 'string' ||
             typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') return callback(new Error(`Invalid response body : ${response.body}`));
+            typeof response.body.newAccount !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));
 
         that.directory = response.body;
 

src/cert/interface.js

@@ -10,7 +10,8 @@ exports = module.exports = {
     getCertificate: getCertificate
 };
 
-var assert = require('assert');
+var assert = require('assert'),
+    BoxError = require('../boxerror.js');
 
 function getCertificate(hostname, domain, options, callback) {
     assert.strictEqual(typeof hostname, 'string');
@@ -18,6 +19,6 @@ function getCertificate(hostname, domain, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    return callback(new Error('Not implemented'));
+    return callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'getCertificate is not implemented'));
 }
 

src/cloudron.js

@@ -134,7 +134,6 @@ function getConfig(callback) {
             mailFqdn: settings.mailFqdn(),
             version: constants.VERSION,
             isDemo: settings.isDemo(),
-            memory: os.totalmem(),
             provider: settings.provider(),
             cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
             uiSpec: custom.uiSpec()
@@ -154,20 +153,20 @@ function isRebootRequired(callback) {
 }
 
 // called from cron.js
-function runSystemChecks() {
+function runSystemChecks(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     async.parallel([
         checkBackupConfiguration,
         checkMailStatus,
         checkRebootRequired
-    ], function (error) {
-        debug('runSystemChecks: done', error);
-    });
+    ], callback);
 }
 
 function checkBackupConfiguration(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    debug('Checking backup configuration');
+    debug('checking backup configuration');
 
     backups.checkConfiguration(function (error, message) {
         if (error) return callback(error);

src/cron.js

@@ -18,12 +18,12 @@ var appHealthMonitor = require('./apphealthmonitor.js'),
     constants = require('./constants.js'),
     CronJob = require('cron').CronJob,
     debug = require('debug')('box:cron'),
-    disks = require('./disks.js'),
     dyndns = require('./dyndns.js'),
     eventlog = require('./eventlog.js'),
     janitor = require('./janitor.js'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
+    system = require('./system.js'),
     updater = require('./updater.js'),
     updateChecker = require('./updatechecker.js');
 
@@ -107,18 +107,16 @@ function recreateJobs(tz) {
     if (gJobs.systemChecks) gJobs.systemChecks.stop();
     gJobs.systemChecks = new CronJob({
         cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: cloudron.runSystemChecks,
+        onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
         start: true,
-        runOnInit: true, // run system check immediately
         timeZone: tz
     });
 
     if (gJobs.diskSpaceChecker) gJobs.diskSpaceChecker.stop();
     gJobs.diskSpaceChecker = new CronJob({
         cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: () => disks.checkDiskSpace(NOOP_CALLBACK),
+        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
         start: true,
-        runOnInit: true, // run system check immediately
         timeZone: tz
     });
 

src/custom.js

@@ -41,7 +41,7 @@ const DEFAULT_SPEC = {
         notifyCloudronAdmins: true
     },
     footer: {
-        body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
+        body: '&copy; 2020 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
     }
 };
 
@@ -63,4 +63,4 @@ function uiSpec() {
 
 function spec() {
     return gSpec;
-}
+}

src/database.js

@@ -14,6 +14,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
+    BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
     constants = require('./constants.js'),
     mysql = require('mysql'),
@@ -112,7 +113,7 @@ function clear(callback) {
 function beginTransaction(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (gConnectionPool === null) return callback(new Error('No database connection pool.'));
+    if (gConnectionPool === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No database connection pool.'));
 
     gConnectionPool.getConnection(function (error, connection) {
         if (error) {
@@ -156,7 +157,7 @@ function query() {
     var callback = args[args.length - 1];
     assert.strictEqual(typeof callback, 'function');
 
-    if (gDefaultConnection === null) return callback(new Error('No connection to database'));
+    if (gDefaultConnection === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No connection to database'));
 
     args[args.length -1 ] = function (error, result) {
         if (error && error.fatal) {

src/dns/cloudflare.js

@@ -48,15 +48,29 @@ function translateRequestError(result, callback) {
     callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
 }
 
+function createRequest(method, url, dnsConfig) {
+    assert.strictEqual(typeof method, 'string');
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof dnsConfig, 'object');
+
+    let request = superagent(method, url)
+        .timeout(30 * 1000);
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        request.set('X-Auth-Key', dnsConfig.token).set('X-Auth-Email', dnsConfig.email);
+    } else {
+        request.set('Authorization', 'Bearer ' + dnsConfig.token);
+    }
+
+    return request;
+}
+
 function getZoneByName(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active')
-        .set('X-Auth-Key', dnsConfig.token)
-        .set('X-Auth-Email', dnsConfig.email)
-        .timeout(30 * 1000)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active', dnsConfig)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -74,11 +88,8 @@ function getDnsRecords(dnsConfig, zoneId, fqdn, type, callback) {
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-        .set('X-Auth-Key',dnsConfig.token)
-        .set('X-Auth-Email',dnsConfig.email)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
         .query({ type: type, name: fqdn })
-        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -132,11 +143,8 @@ function upsert(domainObject, location, type, values, callback) {
                 if (i >= dnsRecords.length) { // create a new record
                     debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: false`);
 
-                    superagent.post(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('POST', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             if (error && !error.response) return iteratorCallback(error);
                             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, iteratorCallback);
@@ -148,11 +156,8 @@ function upsert(domainObject, location, type, values, callback) {
 
                     debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
 
-                    superagent.put(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id)
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('PUT', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id, dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             ++i; // increment, as we have consumed the record
 
@@ -217,10 +222,7 @@ function del(domainObject, location, type, values, callback) {
             if (tmp.length === 0) return callback(null);
 
             async.eachSeries(tmp, function (record, callback) {
-                superagent.del(CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id)
-                    .set('X-Auth-Key', dnsConfig.token)
-                    .set('X-Auth-Email', dnsConfig.email)
-                    .timeout(30 * 1000)
+                createRequest('DELETE', CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id, dnsConfig)
                     .end(function (error, result) {
                         if (error && !error.response) return callback(error);
                         if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -277,14 +279,20 @@ function verifyDnsConfig(domainObject, callback) {
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
+    // token can be api token or global api key
     if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
-    if (!dnsConfig.email || typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    if (dnsConfig.tokenType !== 'GlobalApiKey' && dnsConfig.tokenType !== 'ApiToken')  return callback(new BoxError(BoxError.BAD_FIELD, 'tokenType is required', { field: 'tokenType' }));
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        if ('email' in dnsConfig && typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    }
 
     const ip = '127.0.0.1';
 
     var credentials = {
         token: dnsConfig.token,
-        email: dnsConfig.email
+        tokenType: dnsConfig.tokenType,
+        email: dnsConfig.email || null
     };
 
     if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here

src/dns/godaddy.js

@@ -126,7 +126,7 @@ function del(domainObject, location, type, values, callback) {
 
     debug(`get: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, new Error('Record deletion is not supported by GoDaddy API')));
+    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Record deletion is not supported by GoDaddy API'));
 
     // check if the record exists at all so that we don't insert the "Dead" record for no reason
     get(domainObject, location, type, function (error, values) {

src/dns/interface.js

@@ -17,6 +17,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
     util = require('util');
 
 function removePrivateFields(domainObject) {
@@ -24,6 +25,7 @@ function removePrivateFields(domainObject) {
     return domainObject;
 }
 
+// eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
     // in-place injection of tokens and api keys which came in with domains.SECRET_PLACEHOLDER
 }
@@ -37,7 +39,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     // Result: none
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upsert is not implemented'));
 }
 
 function get(domainObject, location, type, callback) {
@@ -48,7 +50,7 @@ function get(domainObject, location, type, callback) {
 
     // Result: Array of matching DNS records in string format
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'get is not implemented'));
 }
 
 function del(domainObject, location, type, values, callback) {
@@ -60,7 +62,7 @@ function del(domainObject, location, type, values, callback) {
 
     // Result: none
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'del is not implemented'));
 }
 
 function wait(domainObject, location, type, value, options, callback) {
@@ -80,5 +82,5 @@ function verifyDnsConfig(domainObject, callback) {
 
     // Result: dnsConfig object
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'verifyDnsConfig is not implemented'));
 }

src/docker.js

@@ -1,8 +1,6 @@
 'use strict';
 
 exports = module.exports = {
-    connection: connectionInstance(),
-
     testRegistryConfig: testRegistryConfig,
     setRegistryConfig: setRegistryConfig,
     injectPrivateFields: injectPrivateFields,
@@ -16,6 +14,7 @@ exports = module.exports = {
     downloadImage: downloadImage,
     createContainer: createContainer,
     startContainer: startContainer,
+    restartContainer: restartContainer,
     stopContainer: stopContainer,
     stopContainerByName: stopContainer,
     stopContainers: stopContainers,
@@ -27,6 +26,7 @@ exports = module.exports = {
     getContainerIdByIp: getContainerIdByIp,
     inspect: inspect,
     inspectByName: inspect,
+    execContainer: execContainer,
     getEvents: getEvents,
     memoryUsage: memoryUsage,
     createVolume: createVolume,
@@ -34,20 +34,14 @@ exports = module.exports = {
     clearVolume: clearVolume
 };
 
-// timeout is optional
-function connectionInstance(timeout) {
-    var Docker = require('dockerode');
-    var docker = new Docker({ socketPath: '/var/run/docker.sock', timeout: timeout });
-    return docker;
-}
-
 var addons = require('./addons.js'),
     async = require('async'),
     assert = require('assert'),
     BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
     constants = require('./constants.js'),
-    debug = require('debug')('box:docker.js'),
+    debug = require('debug')('box:docker'),
+    Docker = require('dockerode'),
     path = require('path'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
@@ -58,6 +52,9 @@ var addons = require('./addons.js'),
 const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
     MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');
 
+const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
+const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
+
 function debugApp(app) {
     assert(typeof app === 'object');
 
@@ -68,8 +65,7 @@ function testRegistryConfig(auth, callback) {
     assert.strictEqual(typeof auth, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-    docker.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
+    gConnection.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
         if (error) return callback(new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' }));
 
         callback();
@@ -108,9 +104,9 @@ function ping(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     // do not let the request linger
-    var docker = connectionInstance(1000);
+    const connection = new Docker({ socketPath: DOCKER_SOCKET_PATH, timeout: 1000 });
 
-    docker.ping(function (error, result) {
+    connection.ping(function (error, result) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
         if (result !== 'OK') return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon'));
 
@@ -140,14 +136,12 @@ function getRegistryConfig(image, callback) {
 }
 
 function pullImage(manifest, callback) {
-    var docker = exports.connection;
-
     getRegistryConfig(manifest.dockerImage, function (error, authConfig) {
         if (error) return callback(error);
 
         debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${authConfig ? 'yes' : 'no'}`);
 
-        docker.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
+        gConnection.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to pull image. Please check the network or if the image needs authentication. statusCode: ' + error.statusCode));
 
             // https://github.com/dotcloud/docker/issues/1074 says each status message
@@ -203,8 +197,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection,
-        isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers
+    let isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers
 
     var manifest = app.manifest;
     var exposedPorts = {}, dockerPortBindings = { };
@@ -330,7 +323,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
 
         debugApp(app, 'Creating container for %s', app.manifest.dockerImage);
 
-        docker.createContainer(containerOptions, function (error, container) {
+        gConnection.createContainer(containerOptions, function (error, container) {
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
             callback(null, container);
@@ -346,15 +339,29 @@ function startContainer(containerId, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
     debug('Starting container %s', containerId);
 
     container.start(function (error) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
         if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
-        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); // 304 means already started
+
+        return callback(null);
+    });
+}
+
+function restartContainer(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = gConnection.getContainer(containerId);
+    debug('Restarting container %s', containerId);
+
+    container.restart(function (error) {
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+        if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
+        if (error && error.statusCode !== 204) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         return callback(null);
     });
@@ -369,8 +376,7 @@ function stopContainer(containerId, callback) {
         return callback();
     }
 
-    var docker = exports.connection;
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
     debug('Stopping container %s', containerId);
 
     var options = {
@@ -400,8 +406,7 @@ function deleteContainer(containerId, callback) {
 
     if (containerId === null) return callback(null);
 
-    var docker = exports.connection;
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
 
     var removeOptions = {
         force: true, // kill container if it's running
@@ -425,14 +430,12 @@ function deleteContainers(appId, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
     debug('deleting containers of %s', appId);
 
     let labels = [ 'appId=' + appId ];
     if (options.managedOnly) labels.push('isCloudronManaged=true');
 
-    docker.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) {
+    gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         async.eachSeries(containers, function (container, iteratorDone) {
@@ -445,11 +448,9 @@ function stopContainers(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
     debug('stopping containers of %s', appId);
 
-    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+    gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         async.eachSeries(containers, function (container, iteratorDone) {
@@ -465,8 +466,6 @@ function deleteImage(manifest, callback) {
     var dockerImage = manifest ? manifest.dockerImage : null;
     if (!dockerImage) return callback(null);
 
-    var docker = exports.connection;
-
     var removeOptions = {
         force: false, // might be shared with another instance of this app
         noprune: false // delete untagged parents
@@ -475,7 +474,7 @@ function deleteImage(manifest, callback) {
     // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
     // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
     // not created anymore after https://github.com/docker/docker/pull/10571
-    docker.getImage(dockerImage).remove(removeOptions, function (error) {
+    gConnection.getImage(dockerImage).remove(removeOptions, function (error) {
         if (error && error.statusCode === 400) return callback(null); // invalid image format. this can happen if user installed with a bad --docker-image
         if (error && error.statusCode === 404) return callback(null); // not found
         if (error && error.statusCode === 409) return callback(null); // another container using the image
@@ -493,10 +492,8 @@ function getContainerIdByIp(ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
-    docker.getNetwork('cloudron').inspect(function (error, bridge) {
-        if (error && error.statusCode === 404) return callback(new Error('Unable to find the cloudron network'));
+    gConnection.getNetwork('cloudron').inspect(function (error, bridge) {
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to find the cloudron network'));
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         var containerId;
@@ -506,7 +503,7 @@ function getContainerIdByIp(ip, callback) {
                 break;
             }
         }
-        if (!containerId) return callback(new Error('No container with that ip'));
+        if (!containerId) return callback(new BoxError(BoxError.DOCKER_ERROR, 'No container with that ip'));
 
         callback(null, containerId);
     });
@@ -516,7 +513,7 @@ function inspect(containerId, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = exports.connection.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
 
     container.inspect(function (error, result) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -526,13 +523,38 @@ function inspect(containerId, callback) {
     });
 }
 
+function execContainer(containerId, options, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = gConnection.getContainer(containerId);
+
+    container.exec(options.execOptions, function (error, exec) {
+        if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running
+        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+
+        exec.start(options.startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+
+            if (options.rows && options.columns) {
+                // there is a race where resizing too early results in a 404 "no such exec"
+                // https://git.cloudron.io/cloudron/box/issues/549
+                setTimeout(function () {
+                    exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
+                }, 2000);
+            }
+
+            callback(null, stream);
+        });
+    });
+}
+
 function getEvents(options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
-    docker.getEvents(options, function (error, stream) {
+    gConnection.getEvents(options, function (error, stream) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         callback(null, stream);
@@ -543,7 +565,7 @@ function memoryUsage(containerId, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = exports.connection.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
 
     container.stats({ stream: false }, function (error, result) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -559,8 +581,6 @@ function createVolume(app, name, volumeDataDir, callback) {
     assert.strictEqual(typeof volumeDataDir, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
     const volumeOptions = {
         Name: name,
         Driver: 'local',
@@ -577,9 +597,9 @@ function createVolume(app, name, volumeDataDir, callback) {
 
     // requires sudo because the path can be outside appsdata
     shell.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}, function (error) {
-        if (error) return callback(new Error(`Error creating app data dir: ${error.message}`));
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`));
 
-        docker.createVolume(volumeOptions, function (error) {
+        gConnection.createVolume(volumeOptions, function (error) {
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
             callback();
@@ -593,8 +613,7 @@ function clearVolume(app, name, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-    let volume = docker.getVolume(name);
+    let volume = gConnection.getVolume(name);
     volume.inspect(function (error, v) {
         if (error && error.statusCode === 404) return callback();
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
@@ -614,9 +633,7 @@ function removeVolume(app, name, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
-    let volume = docker.getVolume(name);
+    let volume = gConnection.getVolume(name);
     volume.remove(function (error) {
         if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume of ${app.id} ${error.message}`));
 
@@ -627,9 +644,7 @@ function removeVolume(app, name, callback) {
 function info(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
-    docker.info(function (error, result) {
+    gConnection.info(function (error, result) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker'));
 
         callback(null, result);

src/domains.js

@@ -86,9 +86,9 @@ function verifyDnsConfig(dnsConfig, domain, zoneName, provider, callback) {
 
     const domainObject = { config: dnsConfig, domain: domain, zoneName: zoneName };
     api(provider).verifyDnsConfig(domainObject, function (error, result) {
-        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, 'Incorrect configuration. Access denied'));
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, 'Zone not found'));
-        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, 'Configuration error: ' + error.message));
+        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`));
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`));
+        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`));
         if (error) return callback(error);
 
         result.hyphenatedSubdomains = !!dnsConfig.hyphenatedSubdomains;

src/eventlog.js

@@ -57,6 +57,9 @@ exports = module.exports = {
 
     ACTION_DYNDNS_UPDATE: 'dyndns.update',
 
+    ACTION_SUPPORT_TICKET: 'support.ticket',
+    ACTION_SUPPORT_SSH: 'support.ssh',
+
     ACTION_PROCESS_CRASH: 'system.crash'
 };
 

src/janitor.js

@@ -3,8 +3,9 @@
 var assert = require('assert'),
     async = require('async'),
     authcodedb = require('./authcodedb.js'),
+    BoxError = require('./boxerror.js'),
     debug = require('debug')('box:janitor'),
-    docker = require('./docker.js').connection,
+    Docker = require('dockerode'),
     tokendb = require('./tokendb.js');
 
 exports = module.exports = {
@@ -12,7 +13,9 @@ exports = module.exports = {
     cleanupDockerVolumes: cleanupDockerVolumes
 };
 
-var NOOP_CALLBACK = function () { };
+const NOOP_CALLBACK = function () { };
+
+const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });
 
 function ignoreError(func) {
     return function (callback) {
@@ -67,16 +70,16 @@ function cleanupTmpVolume(containerInfo, callback) {
 
     debug('cleanupTmpVolume %j', containerInfo.Names);
 
-    docker.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
-        if (error) return callback(new Error('Failed to exec container : ' + error.message));
+    gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
+        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`));
 
         execContainer.start({ hijack: true }, function (error, stream) {
-            if (error) return callback(new Error('Failed to start exec container : ' + error.message));
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to start exec container: ${error.message}`));
 
             stream.on('error', callback);
             stream.on('end', callback);
 
-            docker.modem.demuxStream(stream, process.stdout, process.stderr);
+            gConnection.modem.demuxStream(stream, process.stdout, process.stderr);
         });
     });
 }
@@ -88,7 +91,7 @@ function cleanupDockerVolumes(callback) {
 
     debug('Cleaning up docker volumes');
 
-    docker.listContainers({ all: 0 }, function (error, containers) {
+    gConnection.listContainers({ all: 0 }, function (error, containers) {
         if (error) return callback(error);
 
         async.eachSeries(containers, function (container, iteratorDone) {

src/ldap.js

@@ -143,10 +143,11 @@ function userSearch(req, res, next) {
             var obj = {
                 dn: dn.toString(),
                 attributes: {
-                    objectclass: ['user'],
+                    objectclass: ['user', 'inetorgperson', 'person' ],
                     objectcategory: 'person',
                     cn: entry.id,
                     uid: entry.id,
+                    entryuuid: entry.id, // to support OpenLDAP clients
                     mail: entry.email,
                     mailAlternateAddress: entry.fallbackEmail,
                     displayname: displayName,

src/locker.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
     debug = require('debug')('box:locker'),
     EventEmitter = require('events').EventEmitter,
     util = require('util');
@@ -23,7 +24,7 @@ Locker.prototype.lock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
     if (this._operation !== null) {
-        let error = new Error(`Locked for ${this._operation}`);
+        let error = new BoxError(BoxError.CONFLICT, `Locked for ${this._operation}`);
         error.operation = this._operation;
         return error;
     }
@@ -54,7 +55,7 @@ Locker.prototype.recursiveLock = function (operation) {
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
-    if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
+    if (this._operation !== operation) throw BoxError(BoxError.BAD_STATE, 'Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
     if (--this._lockDepth === 0) {
         debug('Released : %s', this._operation);

src/mail.js

@@ -123,13 +123,13 @@ function checkOutboundPort25(callback) {
         relay.status = false;
         relay.value = `Connect to ${smtpServer} timed out. Check if port 25 (outbound) is blocked`;
         client.destroy();
-        callback(new Error('Timeout'), relay);
+        callback(new BoxError(BoxError.TIMEOUT, `Connect to ${smtpServer} timed out.`), relay);
     });
     client.on('error', function (error) {
         relay.status = false;
         relay.value = `Connect to ${smtpServer} failed: ${error.message}. Check if port 25 (outbound) is blocked`;
         client.destroy();
-        callback(error, relay);
+        callback(new BoxError(BoxError.NETWORK_ERROR, `Connect to ${smtpServer} failed.`), relay);
     });
 }
 
@@ -199,7 +199,7 @@ function checkDkim(mailDomain, callback) {
     };
 
     var dkimKey = readDkimPublicKeySync(domain);
-    if (!dkimKey) return callback(new Error('Failed to read dkim public key'), dkim);
+    if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, `Failed to read dkim public key of ${domain}`), dkim);
 
     dkim.expected = 'v=DKIM1; t=s; p=' + dkimKey;
 
@@ -456,7 +456,7 @@ function getStatus(domain, callback) {
 
     // ensure we always have a valid toplevel properties for the api
     var results = {
-        dns: {}, // { mx/dmar/dkim/spf/ptr: { expected, value, name, domain, type } }
+        dns: {}, // { mx/dmarc/dkim/spf/ptr: { expected, value, name, domain, type } }
         rbl: {}, // { status, ip, servers: [{name,site,dns}]} optional. only for cloudron-smtp
         relay: {} // { status, value } always checked
     };
@@ -466,7 +466,7 @@ function getStatus(domain, callback) {
             func(function (error, result) {
                 if (error) debug('Ignored error - ' + what + ':', error);
 
-                safe.set(results, what, result);
+                safe.set(results, what, result || {});
 
                 callback();
             });
@@ -567,12 +567,12 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
         // mail_domain is used for SRS
         if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
             `mail_in_domains=${mailInDomains}\nmail_out_domains=${mailOutDomains}\nmail_server_name=${mailFqdn}\nmail_domain=${mailDomain}\n\n`, 'utf8')) {
-            return callback(new Error('Could not create mail var file:' + safe.error.message));
+            return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
         }
 
         // enable_outbound makes plugin forward email for relayed mail. non-relayed mail always hits LMTP plugin first
         if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/smtp_forward.ini'), 'enable_outbound=false\ndomain_selector=mail_from\n', 'utf8')) {
-            return callback(new Error('Could not create smtp forward file:' + safe.error.message));
+            return callback(new BoxError(BoxError.FS_ERROR, 'Could not create smtp forward file:' + safe.error.message));
         }
 
         // create sections for per-domain configuration
@@ -582,7 +582,7 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
 
             if (!safe.fs.appendFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
                 `[${domain.domain}]\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\n\n`, 'utf8')) {
-                return callback(new Error('Could not create mail var file:' + safe.error.message));
+                return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
             }
 
             const relay = domain.relay;
@@ -598,7 +598,7 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
 
             if (!safe.fs.appendFileSync(paths.ADDON_CONFIG_DIR + '/mail/smtp_forward.ini',
                 `[${domain.domain}]\nenable_outbound=true\nhost=${host}\nport=${port}\nenable_tls=true\nauth_type=${authType}\nauth_user=${username}\nauth_pass=${password}\n\n`, 'utf8')) {
-                return callback(new Error('Could not create mail var file:' + safe.error.message));
+                return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
             }
         });
 
@@ -627,8 +627,8 @@ function configureMail(mailFqdn, mailDomain, callback) {
         const mailCertFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_cert.pem');
         const mailKeyFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_key.pem');
 
-        if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new Error('Could not create cert file:' + safe.error.message));
-        if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new Error('Could not create key file:' + safe.error.message));
+        if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create cert file:' + safe.error.message));
+        if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create key file:' + safe.error.message));
 
         shell.exec('startMail', 'docker rm -f mail || true', function (error) {
             if (error) return callback(error);
@@ -846,7 +846,7 @@ function upsertDnsRecords(domain, mailFqdn, callback) {
         if (process.env.BOX_ENV === 'test') return callback();
 
         var dkimKey = readDkimPublicKeySync(domain);
-        if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, new Error('Failed to read dkim public key')));
+        if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, 'Failed to read dkim public key'));
 
         // t=s limits the domainkey to this domain and not it's subdomains
         var dkimRecord = { subdomain: `${mailDomain.dkimSelector}._domainkey`, domain: domain, type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
@@ -1264,7 +1264,7 @@ function removeList(name, domain, auditSource, callback) {
     mailboxdb.del(name, domain, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
 
         callback();
     });

src/paths.js

@@ -44,6 +44,7 @@ exports = module.exports = {
 
     // this is not part of appdata because an icon may be set before install
     APP_ICONS_DIR: path.join(baseDir(), 'boxdata/appicons'),
+    PROFILE_ICONS_DIR: path.join(baseDir(), 'boxdata/profileicons'),
     MAIL_DATA_DIR: path.join(baseDir(), 'boxdata/mail'),
     ACME_ACCOUNT_KEY_FILE: path.join(baseDir(), 'boxdata/acme/acme.key'),
     APP_CERTS_DIR: path.join(baseDir(), 'boxdata/certs'),

src/provision.js

@@ -274,6 +274,7 @@ function getStatus(callback) {
             callback(null, _.extend({
                 version: constants.VERSION,
                 apiServerOrigin: settings.apiServerOrigin(), // used by CaaS tool
+                webServerOrigin: settings.webServerOrigin(), // used by CaaS tool
                 provider: settings.provider(),
                 cloudronName: cloudronName,
                 adminFqdn: settings.adminDomain() ? settings.adminFqdn() : null,

src/reverseproxy.js

@@ -171,7 +171,7 @@ function reload(callback) {
     if (process.env.BOX_ENV === 'test') return callback();
 
     shell.sudo('reload', [ RELOAD_NGINX_CMD ], {}, function (error) {
-        if (error) return callback(new BoxError(BoxError.NGINX_ERROR, error));
+        if (error) return callback(new BoxError(BoxError.NGINX_ERROR, `Error reloading nginx: ${error.message}`));
 
         callback();
     });
@@ -346,7 +346,7 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
                 debug('ensureCertificate: getting certificate for %s with options %j', vhost, apiOptions);
 
                 api.getCertificate(vhost, domain, apiOptions, function (error, certFilePath, keyFilePath) {
-                    debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath}`);
+                    debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath || 'null'}`);
 
                     eventlog.add(currentBundle ? eventlog.ACTION_CERTIFICATE_RENEWAL : eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: vhost, errorMessage: error ? error.message : '' });
 

src/routes/accesscontrol.js

@@ -39,7 +39,7 @@ function initialize(callback) {
     });
 
 
-    // used when username/password is sent in request body. used in CLI tool login route
+    // used when username/password is sent in request body. used in CLI login & oauth2 login route
     passport.use(new LocalStrategy(function (username, password, callback) {
 
         // TODO we should only do this for dashboard logins

src/routes/apps.js

@@ -32,6 +32,7 @@ exports = module.exports = {
 
     stopApp: stopApp,
     startApp: startApp,
+    restartApp: restartApp,
     exec: exec,
     execWebSocket: execWebSocket,
 
@@ -349,19 +350,10 @@ function repairApp(req, res, next) {
 
     const data = req.body;
 
-    if (data.backupId && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
-    if (data.backupFormat && typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string or null'));
-
-    if (data.location && typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (data.domain && typeof data.domain !== 'string') return next(new HttpError(400, 'domain is required'));
-
-    if ('alternateDomains' in data) {
-        if (!Array.isArray(data.alternateDomains)) return next(new HttpError(400, 'alternateDomains must be an array'));
-        if (data.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
+    if ('manifest' in data) {
+        if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'backupId must be an object'));
     }
 
-    if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
-
     apps.repair(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -377,10 +369,9 @@ function restoreApp(req, res, next) {
 
     debug('Restore app id:%s', req.params.id);
 
-    if (!('backupId' in req.body)) return next(new HttpError(400, 'backupId is required'));
-    if (data.backupId !== null && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
+    if (!data.backupId || typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be non-empty string'));
 
-    apps.restore(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
+    apps.restore(req.params.id, data.backupId, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -395,8 +386,23 @@ function importApp(req, res, next) {
 
     debug('Importing app id:%s', req.params.id);
 
-    if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string'));
-    if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
+    if ('backupId' in data) { // if not provided, we import in-place
+        if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string'));
+        if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
+
+        if ('backupConfig' in data && typeof data.backupConfig !== 'object') return next(new HttpError(400, 'backupConfig must be an object'));
+
+        const backupConfig = req.body.backupConfig;
+
+        if (req.body.backupConfig) {
+            if (typeof backupConfig.provider !== 'string') return next(new HttpError(400, 'provider is required'));
+            if ('key' in backupConfig && typeof backupConfig.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+            if ('acceptSelfSignedCerts' in backupConfig && typeof backupConfig.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
+
+            // testing backup config can take sometime
+            req.clearTimeout();
+        }
+    }
 
     apps.importApp(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -475,6 +481,18 @@ function stopApp(req, res, next) {
     });
 }
 
+function restartApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    debug('Restart app id:%s', req.params.id);
+
+    apps.restart(req.params.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, { taskId: result.taskId }));
+    });
+}
+
 function updateApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
     assert.strictEqual(typeof req.body, 'object');

src/routes/cloudron.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     isRebootRequired: isRebootRequired,
     getConfig: getConfig,
     getDisks: getDisks,
+    getMemory: getMemory,
     getUpdateInfo: getUpdateInfo,
     update: update,
     checkForUpdates: checkForUpdates,
@@ -23,11 +24,11 @@ let assert = require('assert'),
     BoxError = require('../boxerror.js'),
     cloudron = require('../cloudron.js'),
     custom = require('../custom.js'),
-    disks = require('../disks.js'),
     externalLdap = require('../externalldap.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     sysinfo = require('../sysinfo.js'),
+    system = require('../system.js'),
     updater = require('../updater.js'),
     updateChecker = require('../updatechecker.js');
 
@@ -55,7 +56,15 @@ function getConfig(req, res, next) {
 }
 
 function getDisks(req, res, next) {
-    disks.getDisks(function (error, result) {
+    system.getDisks(function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function getMemory(req, res, next) {
+    system.getMemory(function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, result));

src/routes/domains.js

@@ -69,7 +69,7 @@ function add(req, res, next) {
     domains.add(req.body.domain, data, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
-        next(new HttpSuccess(201, { domain: req.body.domain, config: req.body.config }));
+        next(new HttpSuccess(201, {}));
     });
 }
 

src/routes/profile.js

@@ -3,6 +3,9 @@
 exports = module.exports = {
     get: get,
     update: update,
+    getAvatar: getAvatar,
+    setAvatar: setAvatar,
+    clearAvatar: clearAvatar,
     changePassword: changePassword,
     setTwoFactorAuthenticationSecret: setTwoFactorAuthenticationSecret,
     enableTwoFactorAuthentication: enableTwoFactorAuthentication,
@@ -12,14 +15,21 @@ exports = module.exports = {
 var assert = require('assert'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
+    fs = require('fs'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    path = require('path'),
+    paths = require('../paths.js'),
+    safe = require('safetydance'),
     users = require('../users.js'),
+    settings = require('../settings.js'),
     _ = require('underscore');
 
 function get(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
+    const emailHash = require('crypto').createHash('md5').update(req.user.email).digest('hex');
+
     next(new HttpSuccess(200, {
         id: req.user.id,
         username: req.user.username,
@@ -28,7 +38,8 @@ function get(req, res, next) {
         displayName: req.user.displayName,
         twoFactorAuthenticationEnabled: req.user.twoFactorAuthenticationEnabled,
         admin: req.user.admin,
-        source: req.user.source
+        source: req.user.source,
+        avatarUrl: fs.existsSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id)) ? `${settings.adminOrigin()}/api/v1/profile/avatar/${req.user.id}` : `https://www.gravatar.com/avatar/${emailHash}.jpg`
     }));
 }
 
@@ -49,6 +60,28 @@ function update(req, res, next) {
     });
 }
 
+function setAvatar(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    if (!req.files.avatar) return next(new HttpError(400, 'avatar is missing'));
+
+    if (!safe.fs.renameSync(req.files.avatar.path, path.join(paths.PROFILE_ICONS_DIR, req.user.id))) return next(new HttpError(500, safe.error));
+
+    next(new HttpSuccess(202, {}));
+}
+
+function clearAvatar(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    safe.fs.unlinkSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id));
+
+    next(new HttpSuccess(202, {}));
+}
+
+function getAvatar(req, res) {
+    res.sendFile(path.join(paths.PROFILE_ICONS_DIR, req.params.identifier));
+}
+
 function changePassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.user, 'object');

src/routes/provision.js

@@ -9,14 +9,15 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    appstore = require('../appstore.js'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:routes/setup'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     provision = require('../provision.js'),
-    settings = require('../settings.js'),
-    superagent = require('superagent');
+    request = require('request'),
+    settings = require('../settings.js');
 
 function providerTokenAuth(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
@@ -24,11 +25,11 @@ function providerTokenAuth(req, res, next) {
     if (settings.provider() === 'ami') {
         if (typeof req.body.providerToken !== 'string' || !req.body.providerToken) return next(new HttpError(400, 'providerToken must be a non empty string'));
 
-        superagent.get('http://169.254.169.254/latest/meta-data/instance-id').timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return next(new HttpError(500, error));
-            if (result.statusCode !== 200) return next(new HttpError(500, 'Unable to get meta data'));
+        request.get('http://169.254.169.254/latest/meta-data/instance-id', { timeout: 30 * 1000 }, function (error, result) {
+            if (error) return next(new HttpError(422, `Network error getting EC2 metadata: ${error.message}`));
+            if (result.statusCode !== 200) return next(new HttpError(422, `Unable to get EC2 meta data. statusCode: ${result.statusCode}`));
 
-            if (result.text !== req.body.providerToken) return next(new HttpError(401, 'Invalid providerToken'));
+            if (result.body !== req.body.providerToken) return next(new HttpError(422, 'Instance ID does not match'));
 
             next();
         });
@@ -62,6 +63,8 @@ function setup(req, res, next) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
+
+        appstore.trackFinishedSetup(dnsConfig.domain);
     });
 }
 
@@ -116,5 +119,10 @@ function getStatus(req, res, next) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, status));
+
+        // check if Cloudron is not in setup state nor activated and let appstore know of the attempt
+        if (!status.activated && !status.setup.active && !status.restore.active) {
+            appstore.trackBeginSetup(status.provider);
+        }
     });
 }

src/routes/support.js

@@ -9,6 +9,7 @@ exports = module.exports = {
 
 var appstore = require('../appstore.js'),
     assert = require('assert'),
+    auditSource = require('../auditsource.js'),
     custom = require('../custom.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
@@ -29,7 +30,7 @@ function createTicket(req, res, next) {
     if (req.body.appId && typeof req.body.appId !== 'string') return next(new HttpError(400, 'appId must be string'));
     if (req.body.altEmail && typeof req.body.altEmail !== 'string') return next(new HttpError(400, 'altEmail must be string'));
 
-    appstore.createTicket(_.extend({ }, req.body, { email: req.user.email, displayName: req.user.displayName }), function (error) {
+    appstore.createTicket(_.extend({ }, req.body, { email: req.user.email, displayName: req.user.displayName }), auditSource.fromRequest(req), function (error) {
         if (error) return next(new HttpError(503, `Error contacting cloudron.io: ${error.message}. Please email ${custom.spec().support.email}`));
 
         next(new HttpSuccess(201, { message: `An email for sent to ${custom.spec().support.email}. We will get back shortly!` }));
@@ -43,7 +44,7 @@ function enableRemoteSupport(req, res, next) {
 
     if (typeof req.body.enable !== 'boolean') return next(new HttpError(400, 'enabled is required'));
 
-    support.enableRemoteSupport(req.body.enable, function (error) {
+    support.enableRemoteSupport(req.body.enable, auditSource.fromRequest(req), function (error) {
         if (error) return next(new HttpError(503, 'Error enabling remote support. Try running "cloudron-support --enable-ssh" on the server'));
 
         next(new HttpSuccess(202, {}));

src/routes/test/apps-test.js

@@ -11,7 +11,7 @@ let apps = require('../../apps.js'),
     constants = require('../../constants.js'),
     crypto = require('crypto'),
     database = require('../../database.js'),
-    docker = require('../../docker.js').connection,
+    Docker = require('dockerode'),
     expect = require('expect.js'),
     fs = require('fs'),
     hat = require('../../hat.js'),
@@ -33,6 +33,8 @@ let apps = require('../../apps.js'),
 
 var SERVER_URL = 'http://localhost:' + constants.PORT;
 
+const docker = new Docker({ socketPath: '/var/run/docker.sock' });
+
 // Test image information
 var TEST_IMAGE_REPO = 'cloudron/test';
 var TEST_IMAGE_TAG = '25.19.0';
@@ -1568,6 +1570,34 @@ describe('App API', function () {
                     });
             });
         });
+
+        it('can restart app', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/restart')
+                .query({ access_token: token })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(202);
+                    taskId = res.body.taskId;
+                    done();
+                });
+        });
+
+        it('wait for app to restart', function (done) {
+            waitForTask(taskId, function () { setTimeout(done, 12000); }); // give app 12 seconds (to die and start)
+        });
+
+        it('did restart the app', function (done) {
+            apps.get(APP_ID, function (error, app) {
+                if (error) return done(error);
+
+                superagent.get('http://localhost:' + app.httpPort + APP_MANIFEST.healthCheckPath)
+                    .end(function (err, res) {
+                        if (res && res.statusCode === 200) return done();
+                        done(new Error('app is not running'));
+                    });
+            });
+        });
+
+
     });
 
     describe('uninstall', function () {

src/routes/test/cloudron-test.js

@@ -186,7 +186,6 @@ describe('Cloudron', function () {
                     expect(result.body.webServerOrigin).to.eql('https://cloudron.io');
                     expect(result.body.adminFqdn).to.eql(settings.adminFqdn());
                     expect(result.body.version).to.eql(constants.VERSION);
-                    expect(result.body.memory).to.eql(os.totalmem());
                     expect(result.body.cloudronName).to.be.a('string');
 
                     done();
@@ -271,4 +270,75 @@ describe('Cloudron', function () {
             req.on('error', done);
         });
     });
+
+    describe('misc routes', function () {
+        before(function (done) {
+            async.series([
+                setup,
+
+                function (callback) {
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                        .query({ setupToken: 'somesetuptoken' })
+                        .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                        .end(function (error, result) {
+                            expect(result).to.be.ok();
+
+                            // stash token for further use
+                            token = result.body.token;
+
+                            callback();
+                        });
+                },
+
+                function (callback) {
+                    superagent.post(SERVER_URL + '/api/v1/users')
+                        .query({ access_token: token })
+                        .send({ username: USERNAME_1, email: EMAIL_1, invite: false })
+                        .end(function (error, result) {
+                            expect(result).to.be.ok();
+                            expect(result.statusCode).to.eql(201);
+
+                            token_1 = hat(8 * 32);
+                            userId_1 = result.body.id;
+
+                            // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+                            tokendb.add({ id: 'tid-1', accessToken: token_1, identifier: userId_1, clientId: 'test-client-id', expires: Date.now() + 100000, scope: 'cloudron', name: '' }, callback);
+                        });
+                }
+            ], done);
+        });
+
+        after(cleanup);
+
+        describe('memory', function () {
+            it('cannot get without token', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/memory')
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(401);
+                        done();
+                    });
+            });
+
+            it('succeeds (admin)', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/memory')
+                    .query({ access_token: token })
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(200);
+                        expect(result.body.memory).to.eql(os.totalmem());
+                        expect(result.body.swap).to.be.a('number');
+
+                        done();
+                    });
+            });
+
+            it('fails (non-admin)', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/memory')
+                    .query({ access_token: token_1 })
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(403);
+                        done();
+                    });
+            });
+        });
+    });
 });

src/routes/test/server-test.js

@@ -148,7 +148,7 @@ describe('REST API', function () {
             });
     });
 
-    it('dns setup twice succeeds', function (done) {
+    xit('dns setup twice succeeds', function (done) {
         superagent.post(SERVER_URL + '/api/v1/cloudron/setup')
             .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, DOMAIN, config: {} }, tlsConfig: { provider: 'fallback' } })
             .end(function (error, result) {

src/routes/users.js

@@ -37,17 +37,7 @@ function create(req, res, next) {
     users.create(username, password, email, displayName, { invitor: req.user, admin: req.body.admin }, auditSource.fromRequest(req), function (error, user) {
         if (error) return next(BoxError.toHttpError(error));
 
-        var userInfo = {
-            id: user.id,
-            username: user.username,
-            displayName: user.displayName,
-            email: user.email,
-            fallbackEmail: user.fallbackEmail,
-            groupIds: [ ],
-            resetToken: user.resetToken
-        };
-
-        next(new HttpSuccess(201, userInfo));
+        next(new HttpSuccess(201, users.removePrivateFields(user)));
     });
 }
 

src/server.js

@@ -49,7 +49,16 @@ function initializeExpressSync() {
     app.enable('trust proxy');
 
     if (process.env.BOX_ENV !== 'test') {
-        app.use(middleware.morgan('Box :method :url :status :response-time ms - :res[content-length]', {
+        app.use(middleware.morgan(function (tokens, req, res) {
+            return [
+                'Box',
+                tokens.method(req, res),
+                tokens.url(req, res).replace(/(access_token=)[^\&]+/, '$1' + '<redacted>'),
+                tokens.status(req, res),
+                tokens['response-time'](req, res), 'ms', '-',
+                tokens.res(req, res, 'content-length')
+            ].join(' ');
+        }, {
             immediate: false,
             // only log failed requests by default
             skip: function (req, res) { return res.statusCode < 400; }
@@ -95,7 +104,7 @@ function initializeExpressSync() {
         .use(router)
         .use(middleware.lastMile());
 
-    // NOTE: these limits have to be in sync with nginx limits
+    // NOTE: routes that use multi-part have to be whitelisted in the reverse proxy
     var FILE_SIZE_LIMIT = '256mb', // max file size that can be uploaded (see also client_max_body_size in nginx)
         FILE_TIMEOUT = 60 * 1000; // increased timeout for file uploads (1 min)
 
@@ -144,6 +153,7 @@ function initializeExpressSync() {
     router.post('/api/v1/cloudron/reboot', cloudronScope, routes.cloudron.reboot);
     router.get ('/api/v1/cloudron/graphs', cloudronScope, routes.graphs.getGraphs);
     router.get ('/api/v1/cloudron/disks', cloudronScope, routes.cloudron.getDisks);
+    router.get ('/api/v1/cloudron/memory', cloudronScope, routes.cloudron.getMemory);
     router.get ('/api/v1/cloudron/logs/:unit', cloudronScope, routes.cloudron.getLogs);
     router.get ('/api/v1/cloudron/logstream/:unit', cloudronScope, routes.cloudron.getLogStream);
     router.get ('/api/v1/cloudron/eventlog', cloudronScope, routes.eventlog.list);
@@ -174,6 +184,9 @@ function initializeExpressSync() {
     // working off the user behind the provided token
     router.get ('/api/v1/profile', profileScope, routes.profile.get);
     router.post('/api/v1/profile', profileScope, routes.profile.update);
+    router.get ('/api/v1/profile/avatar/:identifier', routes.profile.getAvatar); // this is not scoped so it can used directly in img tag
+    router.post('/api/v1/profile/avatar', profileScope, multipart, routes.profile.setAvatar);
+    router.del ('/api/v1/profile/avatar', profileScope, routes.profile.clearAvatar);
     router.post('/api/v1/profile/password', profileScope, routes.users.verifyPassword, routes.profile.changePassword);
     router.post('/api/v1/profile/twofactorauthentication', profileScope, routes.profile.setTwoFactorAuthenticationSecret);
     router.post('/api/v1/profile/twofactorauthentication/enable', profileScope, routes.profile.enableTwoFactorAuthentication);
@@ -255,8 +268,8 @@ function initializeExpressSync() {
     router.post('/api/v1/apps/:id/configure/env', appsManageScope, routes.apps.setEnvironment);
     router.post('/api/v1/apps/:id/configure/data_dir', appsManageScope, routes.apps.setDataDir);
     router.post('/api/v1/apps/:id/configure/location', appsManageScope, routes.apps.setLocation);
-    router.post('/api/v1/apps/:id/configure/repair', appsManageScope, routes.apps.repairApp);
 
+    router.post('/api/v1/apps/:id/repair', appsManageScope, routes.apps.repairApp);
     router.post('/api/v1/apps/:id/update',    appsManageScope, routes.apps.updateApp);
     router.post('/api/v1/apps/:id/restore',   appsManageScope, routes.apps.restoreApp);
     router.post('/api/v1/apps/:id/import',    appsManageScope, routes.apps.importApp);
@@ -264,6 +277,7 @@ function initializeExpressSync() {
     router.get ('/api/v1/apps/:id/backups',   appsManageScope, routes.apps.listBackups);
     router.post('/api/v1/apps/:id/stop',      appsManageScope, routes.apps.stopApp);
     router.post('/api/v1/apps/:id/start',     appsManageScope, routes.apps.startApp);
+    router.post('/api/v1/apps/:id/restart',   appsManageScope, routes.apps.restartApp);
     router.get ('/api/v1/apps/:id/logstream', appsManageScope, routes.apps.getLogStream);
     router.get ('/api/v1/apps/:id/logs',      appsManageScope, routes.apps.getLogs);
     router.get ('/api/v1/apps/:id/exec',      appsManageScope, routes.apps.exec);

src/settings.js

@@ -498,6 +498,8 @@ function setSysinfoConfig(sysinfoConfig, callback) {
     assert.strictEqual(typeof sysinfoConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    if (isDemo()) return callback(new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode'));
+
     sysinfo.testConfig(sysinfoConfig, function (error) {
         if (error) return callback(error);
 

src/shell.js

@@ -7,6 +7,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
     debug = require('debug')('box:shell'),
     once = require('once'),
@@ -60,7 +61,7 @@ function spawn(tag, file, args, options, callback) {
         if (code || signal) debug(tag + ' code: %s, signal: %s', code, signal);
         if (code === 0) return callback(null);
 
-        var e = new Error(util.format(tag + ' exited with error %s signal %s', code, signal));
+        let e = new BoxError(BoxError.SPAWN_ERROR, `${tag} exited with code ${code} signal ${signal}`);
         e.code = code;
         e.signal = signal;
         callback(e);
@@ -68,7 +69,8 @@ function spawn(tag, file, args, options, callback) {
 
     cp.on('error', function (error) {
         debug(tag + ' code: %s, signal: %s', error.code, error.signal);
-        callback(error);
+        let e = new BoxError(BoxError.SPAWN_ERROR, `${tag} errored with code ${error.code} message ${error.message}`);
+        callback(e);
     });
 
     return cp;

src/storage/gcs.js

@@ -115,27 +115,25 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac
         query.maxResults = batchSize;
     }
 
-    async.forever(function listAndDownload(foreverCallback) {
-        bucket.getFiles(query, function (error, files, nextQuery) {
-            if (error) return foreverCallback(error);
+    let done = false;
 
-            if (files.length === 0) return foreverCallback(new Error('Done'));
+    async.whilst(() => !done, function listAndDownload(whilstCallback) {
+        bucket.getFiles(query, function (error, files, nextQuery) {
+            if (error) return whilstCallback(error);
+
+            if (files.length === 0) { done = true; return whilstCallback(); }
 
             const entries = files.map(function (f) { return { fullPath: f.name }; });
             iteratorCallback(entries, function (error) {
-                if (error) return foreverCallback(error);
-                if (!nextQuery) return foreverCallback(new Error('Done'));
+                if (error) return whilstCallback(error);
+                if (!nextQuery) { done = true; return whilstCallback(); }
 
                 query = nextQuery;
 
-                foreverCallback();
+                whilstCallback();
             });
         });
-    }, function (error) {
-        if (error.message === 'Done') return callback(null);
-
-        callback(error);
-    });
+    }, callback);
 }
 
 function copy(apiConfig, oldFilePath, newFilePath) {

src/storage/interface.js

@@ -28,6 +28,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
     EventEmitter = require('events');
 
 function removePrivateFields(apiConfig) {
@@ -35,6 +36,7 @@ function removePrivateFields(apiConfig) {
     return apiConfig;
 }
 
+// eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
     // in-place injection of tokens and api keys which came in with domains.SECRET_PLACEHOLDER
 }
@@ -48,7 +50,7 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
     // Result: none
     // sourceStream errors are handled upstream
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upload is not implemented'));
 }
 
 function download(apiConfig, backupFilePath, callback) {
@@ -57,7 +59,7 @@ function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     // Result: download stream
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'download is not implemented'));
 }
 
 function downloadDir(apiConfig, backupFilePath, destDir) {
@@ -87,7 +89,7 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
     assert.strictEqual(typeof iteratorCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'listDir is not implemented'));
 }
 
 function remove(apiConfig, filename, callback) {
@@ -97,7 +99,7 @@ function remove(apiConfig, filename, callback) {
 
     // Result: none
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'remove is not implemented'));
 }
 
 function removeDir(apiConfig, pathPrefix) {
@@ -106,7 +108,7 @@ function removeDir(apiConfig, pathPrefix) {
 
     // Result: none
     var events = new EventEmitter();
-    process.nextTick(function () { events.emit('done', new Error('not implemented')); });
+    process.nextTick(function () { events.emit('done', new BoxError(BoxError.NOT_IMPLEMENTED, 'removeDir is not implemented')); });
     return events;
 }
 
@@ -116,6 +118,6 @@ function testConfig(apiConfig, callback) {
 
     // Result: none - first callback argument error if config does not pass the test
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'testConfig is not implemented'));
 }
 

src/storage/noop.js

@@ -17,6 +17,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
     debug = require('debug')('box:storage/noop'),
     EventEmitter = require('events');
 
@@ -38,7 +39,7 @@ function download(apiConfig, backupFilePath, callback) {
 
     debug('download: %s', backupFilePath);
 
-    callback(new Error('Cannot download from noop backend'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'Cannot download from noop backend'));
 }
 
 function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
@@ -60,7 +61,7 @@ function downloadDir(apiConfig, backupFilePath, destDir) {
     process.nextTick(function () {
         debug('downloadDir: %s -> %s', backupFilePath, destDir);
 
-        events.emit('done', new Error('Cannot download from noop backend'));
+        events.emit('done', new BoxError(BoxError.NOT_IMPLEMENTED, 'Cannot download from noop backend'));
     });
     return events;
 }
@@ -109,5 +110,6 @@ function removePrivateFields(apiConfig) {
     return apiConfig;
 }
 
+// eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
 }

src/storage/s3.js

@@ -161,29 +161,27 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
             MaxKeys: batchSize
         };
 
-        async.forever(function listAndDownload(foreverCallback) {
-            s3.listObjects(listParams, function (error, listData) {
-                if (error) return foreverCallback(error);
+        let done = false;
 
-                if (listData.Contents.length === 0) return foreverCallback(new Error('Done'));
+        async.whilst(() => !done, function listAndDownload(whilstCallback) {
+            s3.listObjects(listParams, function (error, listData) {
+                if (error) return whilstCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message || error.code));
+
+                if (listData.Contents.length === 0) { done = true; return whilstCallback(); }
 
                 const entries = listData.Contents.map(function (c) { return { fullPath: c.Key, size: c.Size }; });
 
                 iteratorCallback(entries, function (error) {
-                    if (error) return foreverCallback(error);
+                    if (error) return whilstCallback(error);
 
-                    if (!listData.IsTruncated) return foreverCallback(new Error('Done'));
+                    if (!listData.IsTruncated) { done = true; return whilstCallback(); }
 
                     listParams.Marker = listData.Contents[listData.Contents.length - 1].Key; // NextMarker is returned only with delimiter
 
-                    foreverCallback();
+                    whilstCallback();
                 });
             });
-        }, function (error) {
-            if (error.message === 'Done') return callback(null);
-
-            callback(error);
-        });
+        }, callback);
     });
 }
 
@@ -351,9 +349,9 @@ function remove(apiConfig, filename, callback) {
 
         // deleteObjects does not return error if key is not found
         s3.deleteObjects(deleteParams, function (error) {
-            if (error) debug(`remove: Unable to remove ${deleteParams.Key}. error: ${error.message}`);
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message || error.code}`)); // DO sets 'code'
 
-            callback(error);
+            callback(null);
         });
     });
 }
@@ -388,9 +386,12 @@ function removeDir(apiConfig, pathPrefix) {
 
                 // deleteObjects does not return error if key is not found
                 s3.deleteObjects(deleteParams, function (error /*, deleteData */) {
-                    if (error) events.emit('progress', `Unable to remove ${deleteParams.Key} ${error.message}`);
+                    if (error) {
+                        events.emit('progress', `Unable to remove ${deleteParams.Key} ${error.message || error.code}`);
+                        return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to remove ${deleteParams.Key}. error: ${error.message || error.code}`)); // DO sets 'code'
+                    }
 
-                    iteratorCallback(error);
+                    iteratorCallback(null);
                 });
             }, done);
         }, function (error) {

src/support.js

@@ -8,11 +8,12 @@ exports = module.exports = {
 let assert = require('assert'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    shell = require('./shell.js'),
+    eventlog = require('./eventlog.js'),
     once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
-    settings = require('./settings.js');
+    settings = require('./settings.js'),
+    shell = require('./shell.js');
 
 // the logic here is also used in the cloudron-support tool
 const AUTHORIZED_KEYS_CMD = path.join(__dirname, 'scripts/remotesupport.sh');
@@ -48,13 +49,17 @@ function getRemoteSupport(callback) {
     cp.stdout.on('data', (data) => result = result + data.toString('utf8'));
 }
 
-function enableRemoteSupport(enable, callback) {
+function enableRemoteSupport(enable, auditSource, callback) {
+    assert.strictEqual(typeof enable, 'boolean');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     let si = sshInfo();
     shell.sudo('support', [ AUTHORIZED_KEYS_CMD, enable ? 'enable' : 'disable', si.filePath, si.user ], {}, function (error) {
         if (error) callback(new BoxError(BoxError.FS_ERROR, error));
 
+        eventlog.add(eventlog.ACTION_SUPPORT_SSH, auditSource, { enable });
+
         callback();
     });
 }

src/syncer.js

@@ -2,6 +2,7 @@
 
 var assert = require('assert'),
     async = require('async'),
+    BoxError = require('./boxerror.js'),
     DataLayout = require('./datalayout.js'),
     debug = require('debug')('box:syncer'),
     fs = require('fs'),
@@ -85,7 +86,7 @@ function sync(dataLayout, taskProcessor, concurrency, callback) {
     }
 
     var newCacheFd = safe.fs.openSync(newCacheFile, 'w'); // truncates any existing file
-    if (newCacheFd === -1) return callback(new Error('Error opening new cache file: ' + safe.error.message));
+    if (newCacheFd === -1) return callback(new BoxError(BoxError.FS_ERROR, 'Error opening new cache file: ' + safe.error.message));
 
     function advanceCache(entryPath) {
         var lastRemovedDir = null;

src/sysinfo/interface.js

@@ -11,13 +11,14 @@ exports = module.exports = {
     testConfig
 };
 
-var assert = require('assert');
+var assert = require('assert'),
+    BoxError = require('../boxerror.js');
 
 function getServerIp(config, callback) {
-    assert.strictEqual(typeo config, 'object');
+    assert.strictEqual(typeof config, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'testConfig is not implemented'));
 }
 
 function testConfig(config, callback) {

src/system.js

@@ -2,7 +2,8 @@
 
 exports = module.exports = {
     getDisks: getDisks,
-    checkDiskSpace: checkDiskSpace
+    checkDiskSpace: checkDiskSpace,
+    getMemory: getMemory
 };
 
 const apps = require('./apps.js'),
@@ -13,7 +14,9 @@ const apps = require('./apps.js'),
     df = require('@sindresorhus/df'),
     docker = require('./docker.js'),
     notifications = require('./notifications.js'),
-    paths = require('./paths.js');
+    os = require('os'),
+    paths = require('./paths.js'),
+    safe = require('safetydance');
 
 function getDisks(callback) {
     assert.strictEqual(typeof callback, 'function');
@@ -94,3 +97,15 @@ function checkDiskSpace(callback) {
         notifications.alert(notifications.ALERT_DISK_SPACE, 'Server is running out of disk space', oos ? JSON.stringify(disks.disks, null, 4) : '', callback);
     });
 }
+
+function getMemory(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    const stdout = safe.child_process.execSync('swapon --noheadings --raw --bytes --show=SIZE', { encoding: 'utf8' });
+    const swap = !stdout ? 0 : stdout.trim().split('\n').map(x => parseInt(x, 10) || 0).reduce((acc, cur) => acc + cur);
+
+    callback(null, {
+        memory: os.totalmem(),
+        swap: swap
+    });
+}

src/tasks.js

@@ -125,7 +125,7 @@ function add(type, args, callback) {
     assert(Array.isArray(args));
     assert.strictEqual(typeof callback, 'function');
 
-    taskdb.add({ type: type, percent: 0, message: 'Starting ...', args: args }, function (error, taskId) {
+    taskdb.add({ type: type, percent: 1, message: 'Queued', args: args }, function (error, taskId) {
         if (error) return callback(error);
 
         callback(null, taskId);
@@ -166,7 +166,7 @@ function startTask(taskId, options, callback) {
             } else if (!error && task.error) {
                 taskError = task.error;
             } else if (!task) { // db got cleared in tests
-                taskError = new Error(`No such task ${taskId}`);
+                taskError = new BoxError(BoxError.NOT_FOUND, `No such task ${taskId}`);
             }
 
             delete gTasks[taskId];

src/test/system-test.js

@@ -8,8 +8,8 @@
 
 var async = require('async'),
     database = require('../database.js'),
-    disks = require('../disks.js'),
-    expect = require('expect.js');
+    expect = require('expect.js'),
+    system = require('../system.js');
 
 function setup(done) {
     async.series([
@@ -25,12 +25,12 @@ function cleanup(done) {
     ], done);
 }
 
-describe('Disks', function () {
+describe('System', function () {
     before(setup);
     after(cleanup);
 
     it('can get disks', function (done) {
-        disks.getDisks(function (error, disks) {
+        system.getDisks(function (error, disks) {
             expect(!error).to.be.ok();
             expect(disks).to.be.ok();
             done();
@@ -38,10 +38,19 @@ describe('Disks', function () {
     });
 
     it('can check for disk space', function (done) {
-        disks.checkDiskSpace(function (error) {
+        system.checkDiskSpace(function (error) {
             expect(!error).to.be.ok();
             done();
         });
     });
+
+    it('can get memory', function (done) {
+        system.getMemory(function (error, memory) {
+            expect(!error).to.be.ok();
+            expect(memory.memory).to.be.ok();
+            expect(memory.swap).to.be.ok();
+            done();
+        });
+    });
 });
 

src/updatechecker.js

@@ -105,8 +105,8 @@ function checkAppUpdates(callback) {
                         return iteratorDone();
                     }
 
-                    const updateIsBlocked = apps.canAutoupdateApp(app, updateInfo.manifest);
-                    if (autoupdatesEnabled && !updateIsBlocked) return iteratorDone();
+                    const canAutoupdateApp = apps.canAutoupdateApp(app, updateInfo.manifest);
+                    if (autoupdatesEnabled && canAutoupdateApp) return iteratorDone();
 
                     debug('Notifying of app update for %s from %s to %s', app.id, app.manifest.version, updateInfo.manifest.version);
                     notificationPending.push({