more changes

(cherry picked from commit eafd72b4e7)

.eslintrc.json

@@ -5,7 +5,7 @@
     },
     "extends": "eslint:recommended",
     "parserOptions": {
-        "ecmaVersion": 2020
+        "ecmaVersion": 8
     },
     "rules": {
         "indent": [

CHANGES

@@ -2299,7 +2299,6 @@
 * Fix issue where old nginx configs where not removed before upgrade
 
 [6.3.5]
-* Fix permission issues with sshfs
 * filemanager: reset selection if directory has changed
 * branding: fix error highlight with empty cloudron name
 * better text instead of "Cloudron in the wild"
@@ -2312,54 +2311,3 @@
 * eventlog: fix issue where old events are not periodically removed
 * ssfs: fix chown
 
-[6.3.6]
-* Fix broken reboot button
-* app updated notification shown despite failure
-* Update translation for sso login information
-* Hide groups/tags/state filter in app listing for normal users
-* filemanager: Ensure breadcrumbs and hash are correctly updated on folder navigation
-* cloudron-setup: check if nginx/docker is already installed
-* Use the addresses of all available interfaces for port 53 binding
-* refresh config on appstore login
-* password reset: check 2fa when enabled
-
-[7.0.0]
-* Do not use Gravatar as the default but only an option
-* redis: suppress password warning
-* setup UI: fix dark mode
-* wellknown: response to .wellknown/matrix/client
-* purpose field is not required anymore during appstore signup
-* sftp: fix symlink deletion
-* Show correct/new app version info in updated finished notification
-* Make new login email translatable
-* Hide ticket form if cloudron.io mail is not verified
-* Refactor code to use async/await
-* postgresql: bump shm size and disable parallel queries
-* update nodejs to 14.17.6
-* external ldap: If we detect a local user with the same username as found on LDAP/AD we map it
-* add basic eventlog for apps in app view
-* Enable sshfs/cifs/nfs in app import UI
-* Require password for fallback email change
-* Make password reset logic translatable
-* support: only verified email address can open support tickets
-* Logout users without 2FA when mandatory 2fa is enabled
-* notifications: better oom message for redis
-* Add way to impersonate users for presetup
-* mail: open up port 465 for mail submission (TLS)
-* Implement operator role for apps
-* sftp: normal users do not have SFTP access anymore. Use operator role instead
-* eventlog: add service rebuild/restart/configure events
-* upcloud: add object storage integration
-* Each app can now have a custom crontab
-* services: add recovery mode
-* postgresql: fix restore issue with long table names
-* recvmail: make the addon work again
-* mail: update solr to 8.10.0
-* mail: POP3 support
-* update docker to 20.10.7
-* volumes: add remount button
-* mail: add spam eventlog filter type
-* mail: configure dnsbl
-* mail: add duplication detection for lists
-* mail: add SRS for Sieve Forwarding
-

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2021 Cloudron UG
+Copyright (c) 2020 Cloudron UG
 
 With regard to the Cloudron Software:
 

baseimage/initializeBaseUbuntuImage.sh

@@ -32,7 +32,6 @@ debconf-set-selections <<< 'mysql-server mysql-server/root_password_again passwo
 
 gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
 mysql_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "mysql-server-8.0" || echo "mysql-server-5.7")
-ntpd_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "systemd-timesyncd" || echo "")
 apt-get -y install --no-install-recommends \
     acl \
     apparmor \
@@ -50,7 +49,6 @@ apt-get -y install --no-install-recommends \
     logrotate \
     $mysql_package \
     nfs-common \
-    $ntpd_package \
     openssh-server \
     pwgen \
     resolvconf \
@@ -76,7 +74,7 @@ apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends
 cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
 echo "==> Installing node.js"
-readonly node_version=14.17.6
+readonly node_version=14.15.4
 mkdir -p /usr/local/node-${node_version}
 curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-${node_version}
 ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
@@ -92,8 +90,8 @@ mkdir -p /etc/systemd/system/docker.service.d
 echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
 
 # there are 3 packages for docker - containerd, CLI and the daemon
-readonly docker_version=20.10.7
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.6-1_amd64.deb" -o /tmp/containerd.deb
+readonly docker_version=20.10.3
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
 curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
 curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 # apt install with install deps (as opposed to dpkg -i)

box.js

@@ -2,74 +2,65 @@
 
 'use strict';
 
-const dockerProxy = require('./src/dockerproxy.js'),
+let async = require('async'),
+    dockerProxy = require('./src/dockerproxy.js'),
     fs = require('fs'),
     ldap = require('./src/ldap.js'),
     paths = require('./src/paths.js'),
     proxyAuth = require('./src/proxyauth.js'),
-    safe = require('safetydance'),
     server = require('./src/server.js');
 
-let logFd;
+const NOOP_CALLBACK = function () { };
 
-async function setupLogging() {
-    if (process.env.BOX_ENV === 'test') return;
+function setupLogging(callback) {
+    if (process.env.BOX_ENV === 'test') return callback();
 
-    logFd = fs.openSync(paths.BOX_LOG_FILE, 'a');
-    // we used to write using a stream before but it caches internally and there is no way to flush it when things crash
-    process.stdout.write = process.stderr.write = function (...args) {
-        const callback = typeof args[args.length-1] === 'function' ? args.pop() : function () {}; // callback is required for fs.write
-        fs.write.apply(fs, [logFd, ...args, callback]);
-    };
+    const logfileStream = fs.createWriteStream(paths.BOX_LOG_FILE, { flags:'a' });
+    process.stdout.write = process.stderr.write = logfileStream.write.bind(logfileStream);
+
+    callback();
 }
 
-// this is also used as the 'uncaughtException' handler which can only have synchronous functions
-function exitSync(status) {
-    if (status.error) fs.write(logFd, status.error.stack + '\n', function () {});
-    fs.fsyncSync(logFd);
-    fs.closeSync(logFd);
-    process.exit(status.code);
-}
-
-async function startServers() {
-    await setupLogging();
-    await server.start(); // do this first since it also inits the database
-    await proxyAuth.start();
-    await ldap.start();
-    await dockerProxy.start();
-}
-
-async function main() {
-    const [error] = await safe(startServers());
-    if (error) return exitSync({ error: new Error(`Error starting server: ${JSON.stringify(error)}`), code: 1 });
+async.series([
+    setupLogging,
+    server.start, // do this first since it also inits the database
+    proxyAuth.start,
+    ldap.start,
+    dockerProxy.start
+], function (error) {
+    if (error) {
+        console.log('Error starting server', error);
+        process.exit(1);
+    }
 
     // require those here so that logging handler is already setup
     require('supererror');
     const debug = require('debug')('box:box');
 
-    process.on('SIGINT', async function () {
+    process.on('SIGINT', function () {
         debug('Received SIGINT. Shutting down.');
 
-        await proxyAuth.stop();
-        await server.stop();
-        await ldap.stop();
-        await dockerProxy.stop();
+        proxyAuth.stop(NOOP_CALLBACK);
+        server.stop(NOOP_CALLBACK);
+        ldap.stop(NOOP_CALLBACK);
+        dockerProxy.stop(NOOP_CALLBACK);
         setTimeout(process.exit.bind(process), 3000);
     });
 
-    process.on('SIGTERM', async function () {
+    process.on('SIGTERM', function () {
         debug('Received SIGTERM. Shutting down.');
 
-        await proxyAuth.stop();
-        await server.stop();
-        await ldap.stop();
-        await dockerProxy.stop();
+        proxyAuth.stop(NOOP_CALLBACK);
+        server.stop(NOOP_CALLBACK);
+        ldap.stop(NOOP_CALLBACK);
+        dockerProxy.stop(NOOP_CALLBACK);
         setTimeout(process.exit.bind(process), 3000);
     });
 
-    process.on('uncaughtException', (error) => exitSync({ error, code: 1 }));
+    process.on('uncaughtException', function (error) {
+        console.error((error && error.stack) ? error.stack : error);
+        setTimeout(process.exit.bind(process, 1), 3000);
+    });
 
     console.log(`Cloudron is up and running. Logs are at ${paths.BOX_LOG_FILE}`); // this goes to journalctl
-}
-
-main();
+});

crashnotifierservice.js

@@ -2,21 +2,27 @@
 
 'use strict';
 
-const database = require('./src/database.js');
+var database = require('./src/database.js');
 
-const crashNotifier = require('./src/crashnotifier.js');
+var crashNotifier = require('./src/crashnotifier.js');
 
 // This is triggered by systemd with the crashed unit name as argument
-async function main() {
+function main() {
     if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <unitName>');
 
-    const unitName = process.argv[2];
+    var unitName = process.argv[2];
     console.log('Started crash notifier for', unitName);
 
     // eventlog api needs the db
-    await database.initialize();
+    database.initialize(function (error) {
+        if (error) return console.error('Cannot connect to database. Unable to send crash log.', error);
 
-    await crashNotifier.sendFailureLogs(unitName);
+        crashNotifier.sendFailureLogs(unitName, function (error) {
+            if (error) console.error(error);
+
+            process.exit();
+        });
+    });
 }
 
 main();

migrations/20210707154724-users-migrate-avatar.js

@@ -1,13 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('UPDATE users SET avatar="gravatar" WHERE avatar IS NULL', function (error) {
-        if (error) return callback(error);
-
-        db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB NOT NULL', callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB', callback);
-};

migrations/20210729191138-domains-update-wellKnown-matrix-client.js

@@ -1,30 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    safe = require('safetydance');
-
-exports.up = function(db, callback) {
-    db.all('SELECT * from domains', [], function (error, results) {
-        if (error) return callback(error);
-
-        async.eachSeries(results, function (r, iteratorDone) {
-            if (!r.wellKnownJson) return iteratorDone();
-
-            const wellKnown = safe.JSON.parse(r.wellKnownJson);
-            if (!wellKnown) return iteratorDone();
-            const matrixHostname = JSON.parse(wellKnown['matrix/server'])['m.server'];
-
-            wellKnown['matrix/client'] = JSON.stringify({
-                'm.homeserver': {
-                    'base_url': 'https://' + matrixHostname
-                }
-            });
-
-            db.runSql('UPDATE domains SET wellKnownJson=? WHERE domain=?', [ JSON.stringify(wellKnown), r.domain ], iteratorDone);
-        }, callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20210809203855-appAddonConfigs-alter-value.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE appAddonConfigs MODIFY value TEXT NOT NULL', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE appAddonConfigs MODIFY value VARCHAR(512)', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20210820172800-users-alter-loginLocationsJson.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users MODIFY loginLocationsJson MEDIUMTEXT', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users MODIFY loginLocationsJson TEXT', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20210921022843-apps-add-operatorsJson.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN operatorsJson TEXT', callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN operatorsJson', callback);
-};

migrations/20210927211658-apps-add-crontab.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN crontab TEXT', callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN crontab', callback);
-};

migrations/20211001095256-users-add-inviteToken.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN inviteToken VARCHAR(128) DEFAULT ""', callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN inviteToken', callback);
-};

migrations/20211001160903-apps-add-inbox.js

@@ -1,19 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN enableInbox BOOLEAN DEFAULT 0'),
-        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxName VARCHAR(128)'),
-        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxDomain VARCHAR(128)'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN enableInbox'),
-        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxName'),
-        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxDomain'),
-    ], callback);
-};

migrations/20211006200150-domains-ensure-fallbackCertificate.js

@@ -1,33 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    reverseProxy = require('../src/reverseproxy.js'),
-    safe = require('safetydance');
-
-const NGINX_CERT_DIR = '/home/yellowtent/platformdata/nginx/cert';
-
-// ensure fallbackCertificate of domains are present in database and the cert dir. it seems a bad migration lost them.
-// https://forum.cloudron.io/topic/5683/data-argument-must-be-of-type-received-null-error-during-restore-process
-exports.up = function(db, callback) {
-    db.all('SELECT * FROM domains', [ ], function (error, domains) {
-        if (error) return callback(error);
-
-        async.eachSeries(domains, async function (domain, iteratorDone) {
-            let fallbackCertificate = safe.JSON.parse(domain.fallbackCertificateJson);
-            if (!fallbackCertificate || !fallbackCertificate.cert || !fallbackCertificate.key) {
-                let error;
-                [error, fallbackCertificate] = await safe(reverseProxy.generateFallbackCertificate(domain.domain));
-                if (error) return iteratorDone(error);
-            }
-
-            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.cert`, fallbackCertificate.cert, 'utf8')) return iteratorDone(safe.error);
-            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.key`, fallbackCertificate.key, 'utf8')) return iteratorDone(safe.error);
-
-            db.runSql('UPDATE domains SET fallbackCertificateJson=? WHERE domain=?', [ JSON.stringify(fallbackCertificate), domain.domain ], iteratorDone);
-        }, callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20211008171126-mailboxes-add-enablePop3.js

@@ -1,16 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE mailboxes ADD COLUMN enablePop3 BOOLEAN DEFAULT 0', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE mailboxes DROP COLUMN enablePop3', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20211012020255-mail-add-dkimKeyJson.js

@@ -1,42 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    fs = require('fs'),
-    path = require('path');
-
-const MAIL_DATA_DIR = '/home/yellowtent/boxdata/mail';
-const DKIM_DIR = `${MAIL_DATA_DIR}/dkim`;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE mail ADD COLUMN dkimKeyJson MEDIUMTEXT', function (error) {
-        if (error) return callback(error);
-
-        fs.readdir(DKIM_DIR, function (error, filenames) {
-            if (error && error.code === 'ENOENT') return callback();
-            if (error) return callback(error);
-
-            async.eachSeries(filenames, function (filename, iteratorCallback) {
-                const domain = filename;
-                const publicKey = fs.readFileSync(path.join(DKIM_DIR, domain, 'public'), 'utf8');
-                const privateKey = fs.readFileSync(path.join(DKIM_DIR, domain, 'private'), 'utf8');
-
-                const dkimKey = {
-                    publicKey,
-                    privateKey
-                };
-
-                db.runSql('UPDATE mail SET dkimKeyJson=? WHERE domain=?', [ JSON.stringify(dkimKey), domain ], iteratorCallback);
-            }, function (error) {
-                if (error) return callback(error);
-
-                fs.rmdir(DKIM_DIR, { recursive: true }, callback);
-            });
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.run(db, 'ALTER TABLE mail DROP COLUMN dkimKeyJson')
-    ], callback);
-};

migrations/schema.sql

@@ -28,12 +28,11 @@ CREATE TABLE IF NOT EXISTS users(
     twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
     source VARCHAR(128) DEFAULT "",
     role VARCHAR(32),
-    inviteToken VARCHAR(128) DEFAULT "",
     resetToken VARCHAR(128) DEFAULT "",
     resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     active BOOLEAN DEFAULT 1,
-    avatar MEDIUMBLOB NOT NULL,
-    loginLocationsJson MEDIUMTEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
+    avatar MEDIUMBLOB,
+    locationJson TEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
 
     INDEX creationTime_index (creationTime),
     PRIMARY KEY(id));
@@ -86,9 +85,6 @@ CREATE TABLE IF NOT EXISTS apps(
     enableMailbox BOOLEAN DEFAULT 1, // whether sendmail addon is enabled
     mailboxName VARCHAR(128), // mailbox of this app
     mailboxDomain VARCHAR(128), // mailbox domain of this apps
-    enableInbox BOOLEAN DEFAULT 0, // whether recvmail addon is enabled
-    inboxName VARCHAR(128), // mailbox of this app
-    inboxDomain VARCHAR(128), // mailbox domain of this apps
     label VARCHAR(128),     // display name
     tagsJson VARCHAR(2048), // array of tags
     dataDir VARCHAR(256) UNIQUE,
@@ -121,7 +117,7 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
     appId VARCHAR(128) NOT NULL,
     addonId VARCHAR(32) NOT NULL,
     name VARCHAR(128) NOT NULL,
-    value TEXT NOT NULL,
+    value VARCHAR(512) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id));
 
 CREATE TABLE IF NOT EXISTS appEnvVars(
@@ -180,7 +176,6 @@ CREATE TABLE IF NOT EXISTS mail(
     relayJson TEXT,
     bannerJson TEXT,
 
-    dkimKeyJson MEDIUMTEXT,
     dkimSelector VARCHAR(128) NOT NULL DEFAULT "cloudron",
 
     FOREIGN KEY(domain) REFERENCES domains(domain),
@@ -207,7 +202,6 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     domain VARCHAR(128),
     active BOOLEAN DEFAULT 1,
-    enablePop3 BOOLEAN DEFAULT 0,
 
     FOREIGN KEY(domain) REFERENCES mail(domain),
     FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
@@ -228,7 +222,6 @@ CREATE TABLE IF NOT EXISTS subdomains(
 CREATE TABLE IF NOT EXISTS tasks(
     id int NOT NULL AUTO_INCREMENT,
     type VARCHAR(32) NOT NULL,
-    argsJson TEXT,
     percent INTEGER DEFAULT 0,
     message TEXT,
     errorJson TEXT,

package.json

@@ -11,11 +11,11 @@
     "url": "https://git.cloudron.io/cloudron/box.git"
   },
   "dependencies": {
-    "@google-cloud/dns": "^2.2.0",
+    "@google-cloud/dns": "^2.1.0",
     "@google-cloud/storage": "^5.8.5",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
     "async": "^3.2.0",
-    "aws-sdk": "^2.936.0",
+    "aws-sdk": "^2.906.0",
     "basic-auth": "^2.0.1",
     "body-parser": "^1.19.0",
     "cloudron-manifestformat": "^5.10.2",
@@ -33,11 +33,11 @@
     "ejs": "^3.1.6",
     "ejs-cli": "^2.2.1",
     "express": "^4.17.1",
-    "ipaddr.js": "^2.0.1",
+    "ipaddr.js": "^2.0.0",
     "js-yaml": "^4.1.0",
     "json": "^11.0.0",
     "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.3.0",
+    "ldapjs": "^2.2.4",
     "lodash": "^4.17.21",
     "lodash.chunk": "^4.2.0",
     "mime": "^2.5.2",
@@ -45,9 +45,9 @@
     "moment-timezone": "^0.5.33",
     "morgan": "^1.10.0",
     "multiparty": "^4.2.2",
-    "mustache-express": "^1.3.1",
+    "mustache-express": "^1.3.0",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.6.2",
+    "nodemailer": "^6.6.0",
     "nodemailer-smtp-transport": "^2.7.4",
     "once": "^1.4.0",
     "pretty-bytes": "^5.6.0",
@@ -56,8 +56,9 @@
     "qrcode": "^1.4.4",
     "readdirp": "^3.6.0",
     "request": "^2.88.2",
+    "rimraf": "^3.0.2",
     "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^2.2.0",
+    "safetydance": "^2.0.1",
     "semver": "^7.3.5",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
@@ -70,17 +71,17 @@
     "underscore": "^1.13.1",
     "uuid": "^8.3.2",
     "validator": "^13.6.0",
-    "ws": "^7.5.1",
+    "ws": "^7.4.5",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.4.1",
     "js2xmlparser": "^4.0.1",
-    "mocha": "^9.0.1",
+    "mocha": "^8.4.0",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^13.1.0",
-    "node-sass": "^6.0.1",
+    "nock": "^13.0.11",
+    "node-sass": "^6.0.0",
     "recursive-readdir": "^2.2.2"
   },
   "scripts": {

runTests

@@ -22,7 +22,7 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/box boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
 mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update
 sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
 
@@ -37,8 +37,7 @@ openssl req -x509 -newkey rsa:2048 -keyout platformdata/nginx/cert/host.key -out
 # clear out any containers if FAST is unset
 if [[ -z ${FAST+x} ]]; then
     echo "=> Delete all docker containers first"
-    docker ps -qa --filter "label=isCloudronManaged" | xargs --no-run-if-empty docker rm -f
-    docker rm -f mysql-server
+    docker ps -qa | xargs --no-run-if-empty docker rm -f
     echo "==> To skip this run with: FAST=1 ./runTests"
 else
     echo "==> WARNING!! Skipping docker container cleanup, the database might not be pristine!"
@@ -63,9 +62,6 @@ while ! mysqladmin ping -h"${MYSQL_IP}" --silent; do
     sleep 1
 done
 
-echo "=> Ensure local base image"
-docker pull cloudron/base:3.0.0@sha256:455c70428723e3a823198c57472785437eb6eab082e79b3ff04ea584faf46e92
-
 echo "=> Create iptables blocklist"
 sudo ipset create cloudron_blocklist hash:net || true
 

scripts/cloudron-setup

@@ -41,11 +41,6 @@ if [[ "${disk_size_gb}" -lt "${MINIMUM_DISK_SIZE_GB}" ]]; then
     exit 1
 fi
 
-if [[ "$(uname -m)" != "x86_64" ]]; then
-    echo "Error: Cloudron only supports amd64/x86_64"
-    exit 1
-fi
-
 # do not use is-active in case box service is down and user attempts to re-install
 if systemctl cat box.service >/dev/null 2>&1; then
     echo "Error: Cloudron is already installed. To reinstall, start afresh"
@@ -104,11 +99,6 @@ if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubu
     exit 1
 fi
 
-if which nginx >/dev/null || which docker >/dev/null || which node > /dev/null; then
-    echo "Error: Some packages like nginx/docker/nodejs are already installed. Cloudron requires specific versions of these packages and will install them as part of it's installation. Please start with a fresh Ubuntu install and run this script again." > /dev/stderr
-    exit 1
-fi
-
 # Install MOTD file for stack script style installations. this is removed by the trap exit handler. Heredoc quotes prevents parameter expansion
 cat > /etc/update-motd.d/91-cloudron-install-in-progress <<'EOF'
 #!/bin/bash

scripts/cloudron-support

@@ -39,9 +39,11 @@ while true; do
     --owner-login)
         admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY creationTime LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
-        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/null)
-        mysql -NB -uroot -ppassword -e "INSERT INTO box.settings (name, value) VALUES ('ghosts_config', '{\"${admin_username}\":\"${admin_password}\"}') ON DUPLICATE KEY UPDATE name='ghosts_config', value='{\"${admin_username}\":\"${admin_password}\"}'" 2>/dev/null
-        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once."
+        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/nul)
+        ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
+        printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > "${ghost_file}"
+        chown yellowtent:yellowtent "${ghost_file}" && chmod o-r,g-r "${ghost_file}"
+        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
         exit 0
         ;;
     --) break;;

scripts/createReleaseTarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v14.17.6" ]]; then
-    echo "This script requires node 14.17.6"
+if [[ "$(node --version)" != "v14.15.4" ]]; then
+    echo "This script requires node 14.15.4"
     exit 1
 fi
 

scripts/installer.sh

@@ -73,10 +73,10 @@ log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION
 
 log "updating docker"
 
-readonly docker_version=20.10.7
+readonly docker_version=20.10.3
 if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
     # there are 3 packages for docker - containerd, CLI and the daemon
-    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.6-1_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
@@ -115,13 +115,13 @@ if ! which sshfs; then
 fi
 
 log "updating node"
-readonly node_version=14.17.6
+readonly node_version=14.15.4
 if [[ "$(node --version)" != "v${node_version}" ]]; then
     mkdir -p /usr/local/node-${node_version}
     $curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-${node_version}
     ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
     ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-14.15.4
+    rm -rf /usr/local/node-10.18.1
 fi
 
 # this is here (and not in updater.js) because rebuild requires the above node

setup/start.sh

@@ -16,7 +16,7 @@ readonly HOME_DIR="/home/${USER}"
 readonly BOX_SRC_DIR="${HOME_DIR}/box"
 readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata"
 readonly APPS_DATA_DIR="${HOME_DIR}/appsdata"
-readonly BOX_DATA_DIR="${HOME_DIR}/boxdata/box"
+readonly BOX_DATA_DIR="${HOME_DIR}/boxdata"
 readonly MAIL_DATA_DIR="${HOME_DIR}/boxdata/mail"
 
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -42,7 +42,7 @@ docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron |
 
 mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
-mkdir -p "${MAIL_DATA_DIR}"
+mkdir -p "${MAIL_DATA_DIR}/dkim"
 
 # keep these in sync with paths.js
 log "Ensuring directories"
@@ -52,8 +52,7 @@ mkdir -p "${PLATFORM_DATA_DIR}/mysql"
 mkdir -p "${PLATFORM_DATA_DIR}/postgresql"
 mkdir -p "${PLATFORM_DATA_DIR}/mongodb"
 mkdir -p "${PLATFORM_DATA_DIR}/redis"
-mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner" \
-         "${PLATFORM_DATA_DIR}/addons/mail/dkim"
+mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner"
 mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
 mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
 mkdir -p "${PLATFORM_DATA_DIR}/acme"
@@ -71,6 +70,10 @@ mkdir -p "${PLATFORM_DATA_DIR}/firewall"
 mkdir -p /var/backups
 chmod 777 /var/backups
 
+# can be removed after 6.3
+[[ -f "${BOX_DATA_DIR}/updatechecker.json" ]] && mv "${BOX_DATA_DIR}/updatechecker.json" "${PLATFORM_DATA_DIR}/update/updatechecker.json"
+rm -rf "${BOX_DATA_DIR}/well-known"
+
 log "Configuring journald"
 sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
     -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
@@ -81,14 +84,13 @@ sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
 sed -e "s/^WatchdogSec=.*$/WatchdogSec=3min/" \
     -i /lib/systemd/system/systemd-journald.service
 
-usermod -a -G systemd-journal ${USER} # Give user access to system logs
-if [[ ! -d /var/log/journal ]]; then # in some images, this directory is not created making system log to /run/systemd instead
-    mkdir -p /var/log/journal
-    chown root:systemd-journal /var/log/journal
-    chmod g+s /var/log/journal  # sticky bit for group propagation
-fi
+# Give user access to system logs
+usermod -a -G systemd-journal ${USER}
+mkdir -p /var/log/journal  # in some images, this directory is not created making system log to /run/systemd instead
+chown root:systemd-journal /var/log/journal
 systemctl daemon-reload
 systemctl restart systemd-journald
+setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
 
 # Give user access to nginx logs (uses adm group)
 usermod -a -G adm ${USER}
@@ -231,9 +233,11 @@ chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 chown "${USER}:${USER}" "${APPS_DATA_DIR}"
 
-chown "${USER}:${USER}" -R "${BOX_DATA_DIR}"
-# do not chown the boxdata/mail directory entirely; dovecot gets upset
+# do not chown the boxdata/mail directory; dovecot gets upset
+chown "${USER}:${USER}" "${BOX_DATA_DIR}"
+find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${MAIL_DATA_DIR}" -exec chown -R "${USER}:${USER}" {} \;
 chown "${USER}:${USER}" "${MAIL_DATA_DIR}"
+chown "${USER}:${USER}" -R "${MAIL_DATA_DIR}/dkim" # this is owned by box currently since it generates the keys
 
 log "Starting Cloudron"
 systemctl start box

setup/start/cloudron-firewall.sh

@@ -81,8 +81,8 @@ for port in 2525 4190 9993; do
     iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
 done
 
-# msa, ldap, imap, sieve, pop3
-for port in 2525 3002 4190 9993 9995; do
+# msa, ldap, imap, sieve
+for port in 2525 3002 4190 9993; do
     iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
 done
 

setup/start/cloudron-motd

@@ -4,7 +4,7 @@
 
 printf "**********************************************************************\n\n"
 
-if [[ -z "$(ls -A /home/yellowtent/platformdata/addons/mail/dkim)" ]]; then
+if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
     if [[ -f /tmp/.cloudron-motd-cache ]]; then
         ip=$(cat /tmp/.cloudron-motd-cache)
     elif ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then

setup/start/sudoers

@@ -59,5 +59,3 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/addmount.sh
 Defaults!/home/yellowtent/box/src/scripts/rmmount.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmount.sh
 
-Defaults!/home/yellowtent/box/src/scripts/remountmount.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/remountmount.sh

src/accesscontrol.js

@@ -8,7 +8,10 @@ const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     safe = require('safetydance'),
     tokens = require('./tokens.js'),
-    users = require('./users.js');
+    users = require('./users.js'),
+    util = require('util');
+
+const userGet = util.promisify(users.get);
 
 async function verifyToken(accessToken) {
     assert.strictEqual(typeof accessToken, 'string');
@@ -16,8 +19,10 @@ async function verifyToken(accessToken) {
     const token = await tokens.getByAccessToken(accessToken);
     if (!token) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'No such token');
 
-    const user = await users.get(token.identifier);
-    if (!user) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not found');
+    const [error, user] = await safe(userGet(token.identifier));
+    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not found');
+    if (error) throw error;
+
     if (!user.active) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not active');
 
     await safe(tokens.update(token.id, { lastUsedTime: new Date() })); // ignore any error

src/acme2.js

@@ -9,10 +9,11 @@ exports = module.exports = {
 };
 
 const assert = require('assert'),
+    async = require('async'),
     BoxError = require('./boxerror.js'),
     crypto = require('crypto'),
     debug = require('debug')('box:cert/acme2'),
-    dns = require('./dns.js'),
+    domains = require('./domains.js'),
     fs = require('fs'),
     os = require('os'),
     path = require('path'),
@@ -398,11 +399,17 @@ Acme2.prototype.prepareDnsChallenge = async function (hostname, domain, authoriz
 
     debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
 
-    await dns.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
+    return new Promise((resolve, reject) => {
+        domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
+            if (error) return reject(error);
 
-    await dns.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 });
+            domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 }, function (error) {
+                if (error) return reject(error);
 
-    return challenge;
+                resolve(challenge);
+            });
+        });
+    });
 };
 
 Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challenge) {
@@ -419,7 +426,13 @@ Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challeng
 
     debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
 
-    await dns.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
+    return new Promise((resolve, reject) => {
+        domains.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
+            if (error) return reject(error);
+
+            resolve(null);
+        });
+    });
 };
 
 Acme2.prototype.prepareChallenge = async function (hostname, domain, authorizationUrl, acmeChallengesDir) {
@@ -509,7 +522,7 @@ Acme2.prototype.getCertificate = async function (vhost, domain, paths) {
     debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
 
     if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
-        vhost = dns.makeWildcard(vhost);
+        vhost = domains.makeWildcard(vhost);
         debug(`getCertificate: will get wildcard cert for ${vhost}`);
     }
 
@@ -517,17 +530,18 @@ Acme2.prototype.getCertificate = async function (vhost, domain, paths) {
     await this.acmeFlow(vhost, domain, paths);
 };
 
-async function getCertificate(vhost, domain, paths, options) {
+function getCertificate(vhost, domain, paths, options, callback) {
     assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof paths, 'object');
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     let attempt = 1;
-    await promiseRetry({ times: 3, interval: 0 }, async function () {
+    async.retry({ times: 3, interval: 0 }, function (retryCallback) {
         debug(`getCertificate: attempt ${attempt++}`);
 
-        const acme = new Acme2(options || { });
-        return await acme.getCertificate(vhost, domain, paths);
-    });
+        let acme = new Acme2(options || { });
+        acme.getCertificate(vhost, domain, paths).then(callback).catch(retryCallback);
+    }, callback);
 }

src/addonconfigs.js

@@ -1,81 +0,0 @@
-'use strict';
-
-exports = module.exports =  {
-    get,
-    set,
-    unset,
-
-    getByAppId,
-    getByName,
-    unsetByAppId,
-    getAppIdByValue,
-};
-
-const assert = require('assert'),
-    database = require('./database.js');
-
-async function set(appId, addonId, env) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert(Array.isArray(env));
-
-    await unset(appId, addonId);
-    if (env.length === 0) return;
-
-    const query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
-    const args = [ ], queryArgs = [ ];
-    for (let i = 0; i < env.length; i++) {
-        args.push(appId, addonId, env[i].name, env[i].value);
-        queryArgs.push('(?, ?, ?, ?)');
-    }
-
-    await database.query(query + queryArgs.join(','), args);
-}
-
-async function unset(appId, addonId) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-
-    await database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
-}
-
-async function unsetByAppId(appId) {
-    assert.strictEqual(typeof appId, 'string');
-
-    await database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ]);
-}
-
-async function get(appId, addonId) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-
-    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
-    return results;
-}
-
-async function getByAppId(appId) {
-    assert.strictEqual(typeof appId, 'string');
-
-    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ]);
-    return results;
-}
-
-async function getAppIdByValue(addonId, namePattern, value) {
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-    assert.strictEqual(typeof value, 'string');
-
-    const results = await database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ]);
-    if (results.length === 0) return null;
-    return results[0].appId;
-}
-
-async function getByName(appId, addonId, namePattern) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-
-    const results = await database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ]);
-    if (results.length === 0) return null;
-    return results[0].value;
-}

src/appdb.js

@@ -0,0 +1,597 @@
+'use strict';
+
+exports = module.exports = {
+    get,
+    add,
+    exists,
+    del,
+    update,
+    getAll,
+    getPortBindings,
+    delPortBinding,
+
+    setAddonConfig,
+    getAddonConfig,
+    getAddonConfigByAppId,
+    getAddonConfigByName,
+    unsetAddonConfig,
+    unsetAddonConfigByAppId,
+    getAppIdByAddonConfigValue,
+    getByIpAddress,
+
+    getIcons,
+
+    setHealth,
+    setTask,
+    getAppStoreIds,
+
+    // subdomain table types
+    SUBDOMAIN_TYPE_PRIMARY: 'primary',
+    SUBDOMAIN_TYPE_REDIRECT: 'redirect',
+    SUBDOMAIN_TYPE_ALIAS: 'alias',
+
+    _clear: clear
+};
+
+const assert = require('assert'),
+    async = require('async'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    safe = require('safetydance'),
+    util = require('util');
+
+const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
+    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
+    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson',
+    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp',
+    'apps.creationTime', 'apps.updateTime', 'apps.enableMailbox', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
+    'apps.dataDir', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
+
+const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
+    result.manifest = safe.JSON.parse(result.manifestJson);
+    delete result.manifestJson;
+
+    assert(result.tagsJson === null || typeof result.tagsJson === 'string');
+    result.tags = safe.JSON.parse(result.tagsJson) || [];
+    delete result.tagsJson;
+
+    assert(result.reverseProxyConfigJson === null || typeof result.reverseProxyConfigJson === 'string');
+    result.reverseProxyConfig = safe.JSON.parse(result.reverseProxyConfigJson) || {};
+    delete result.reverseProxyConfigJson;
+
+    assert(result.hostPorts === null || typeof result.hostPorts === 'string');
+    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');
+
+    result.portBindings = { };
+    let hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
+    let environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
+    let portTypes = result.portTypes === null ? [ ] : result.portTypes.split(',');
+
+    delete result.hostPorts;
+    delete result.environmentVariables;
+    delete result.portTypes;
+
+    for (let i = 0; i < environmentVariables.length; i++) {
+        result.portBindings[environmentVariables[i]] = { hostPort: parseInt(hostPorts[i], 10), type: portTypes[i] };
+    }
+
+    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
+    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
+    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
+    delete result.accessRestrictionJson;
+
+    result.sso = !!result.sso;
+    result.enableBackup = !!result.enableBackup;
+    result.enableAutomaticUpdate = !!result.enableAutomaticUpdate;
+    result.enableMailbox = !!result.enableMailbox;
+    result.proxyAuth = !!result.proxyAuth;
+    result.hasIcon = !!result.hasIcon;
+    result.hasAppStoreIcon = !!result.hasAppStoreIcon;
+
+    assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
+    result.debugMode = safe.JSON.parse(result.debugModeJson);
+    delete result.debugModeJson;
+
+    assert(result.servicesConfigJson === null || typeof result.servicesConfigJson === 'string');
+    result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
+    delete result.servicesConfigJson;
+
+    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
+    delete result.subdomains;
+    delete result.domains;
+    delete result.subdomainTypes;
+
+    result.alternateDomains = [];
+    result.aliasDomains = [];
+    for (let i = 0; i < subdomainTypes.length; i++) {
+        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
+            result.location = subdomains[i];
+            result.domain = domains[i];
+        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
+            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
+        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
+            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
+        }
+    }
+
+    let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
+    delete result.envNames;
+    delete result.envValues;
+    result.env = {};
+    for (let i = 0; i < envNames.length; i++) { // NOTE: envNames is [ null ] when env of an app is empty
+        if (envNames[i]) result.env[envNames[i]] = envValues[i];
+    }
+
+    let volumeIds = JSON.parse(result.volumeIds);
+    delete result.volumeIds;
+    let volumeReadOnlys = JSON.parse(result.volumeReadOnlys);
+    delete result.volumeReadOnlys;
+
+    result.mounts = volumeIds[0] === null ? [] : volumeIds.map((v, idx) => { return { volumeId: v, readOnly: !!volumeReadOnlys[idx] }; }); // NOTE: volumeIds is [null] when volumes of an app is empty
+
+    result.error = safe.JSON.parse(result.errorJson);
+    delete result.errorJson;
+
+    result.taskId = result.taskId ? String(result.taskId) : null;
+}
+
+// each query simply join apps table with another table by id. we then join the full result together
+const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
+const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
+const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
+const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
+const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
+    + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
+    + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
+    + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
+    + ` LEFT JOIN (${MOUNTS_QUERY}) AS q4 on q4.id = apps.id`;
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`${APPS_QUERY} WHERE apps.id = ?`, [ id ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function getByIpAddress(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`${APPS_QUERY} WHERE apps.containerIp = ?`, [ ip ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function add(id, appStoreId, manifest, location, domain, portBindings, data, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof appStoreId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof manifest.version, 'string');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    portBindings = portBindings || { };
+
+    var manifestJson = JSON.stringify(manifest);
+
+    const accessRestriction = data.accessRestriction || null;
+    const accessRestrictionJson = JSON.stringify(accessRestriction);
+    const memoryLimit = data.memoryLimit || 0;
+    const cpuShares = data.cpuShares || 512;
+    const installationState = data.installationState;
+    const runState = data.runState;
+    const sso = 'sso' in data ? data.sso : null;
+    const debugModeJson = data.debugMode ? JSON.stringify(data.debugMode) : null;
+    const env = data.env || {};
+    const label = data.label || null;
+    const tagsJson = data.tags ? JSON.stringify(data.tags) : null;
+    const mailboxName = data.mailboxName || null;
+    const mailboxDomain = data.mailboxDomain || null;
+    const reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null;
+    const servicesConfigJson = data.servicesConfig ? JSON.stringify(data.servicesConfig) : null;
+    const enableMailbox = data.enableMailbox || false;
+    const icon = data.icon || null;
+
+    let queries = [];
+
+    queries.push({
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
+            + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox) '
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
+            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox ]
+    });
+
+    queries.push({
+        query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+        args: [ id, domain, location, exports.SUBDOMAIN_TYPE_PRIMARY ]
+    });
+
+    Object.keys(portBindings).forEach(function (env) {
+        queries.push({
+            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, type, appId) VALUES (?, ?, ?, ?)',
+            args: [ env, portBindings[env].hostPort, portBindings[env].type, id ]
+        });
+    });
+
+    Object.keys(env).forEach(function (name) {
+        queries.push({
+            query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
+            args: [ id, name, env[name] ]
+        });
+    });
+
+    if (data.alternateDomains) {
+        data.alternateDomains.forEach(function (d) {
+            queries.push({
+                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]
+            });
+        });
+    }
+
+    if (data.aliasDomains) {
+        data.aliasDomains.forEach(function (d) {
+            queries.push({
+                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
+            });
+        });
+    }
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'no such domain'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function exists(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        return callback(null, result.length !== 0);
+    });
+}
+
+function getPortBindings(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        var portBindings = { };
+        for (let i = 0; i < results.length; i++) {
+            portBindings[results[i].environmentVariable] = { hostPort: results[i].hostPort, type: results[i].type };
+        }
+
+        callback(null, portBindings);
+    });
+}
+
+function getIcons(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT icon, appStoreIcon FROM apps WHERE id = ?', [ id ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, { icon: results[0].icon, appStoreIcon: results[0].appStoreIcon });
+    });
+}
+
+function delPortBinding(hostPort, type, callback) {
+    assert.strictEqual(typeof hostPort, 'number');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appPortBindings WHERE hostPort=? AND type=?', [ hostPort, type ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [
+        { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
+        { query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
+    ];
+
+    database.transaction(queries, function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results[5].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        database.query.bind(null, 'DELETE FROM subdomains'),
+        database.query.bind(null, 'DELETE FROM appPortBindings'),
+        database.query.bind(null, 'DELETE FROM appAddonConfigs'),
+        database.query.bind(null, 'DELETE FROM appEnvVars'),
+        database.query.bind(null, 'DELETE FROM apps')
+    ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        return callback(null);
+    });
+}
+
+function update(id, app, callback) {
+    // ts is useful as a versioning mechanism (for example, icon changed). update the timestamp explicity in code instead of db.
+    // this way health and healthTime can be updated without changing ts
+    app.ts = new Date();
+    updateWithConstraints(id, app, '', callback);
+}
+
+function updateWithConstraints(id, app, constraints, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof constraints, 'string');
+    assert.strictEqual(typeof callback, 'function');
+    assert(!('portBindings' in app) || typeof app.portBindings === 'object');
+    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
+    assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
+    assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
+    assert(!('tags' in app) || Array.isArray(app.tags));
+    assert(!('env' in app) || typeof app.env === 'object');
+
+    var queries = [ ];
+
+    if ('portBindings' in app) {
+        var portBindings = app.portBindings || { };
+        // replace entries by app id
+        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
+        Object.keys(portBindings).forEach(function (env) {
+            var values = [ portBindings[env].hostPort, portBindings[env].type, env, id ];
+            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, type, environmentVariable, appId) VALUES(?, ?, ?, ?)', args: values });
+        });
+    }
+
+    if ('env' in app) {
+        queries.push({ query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] });
+
+        Object.keys(app.env).forEach(function (name) {
+            queries.push({
+                query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
+                args: [ id, name, app.env[name] ]
+            });
+        });
+    }
+
+    if ('location' in app && 'domain' in app) { // must be updated together as they are unique together
+        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ]}); // all locations of an app must be updated together
+        queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, app.domain, app.location, exports.SUBDOMAIN_TYPE_PRIMARY ]});
+
+        if ('alternateDomains' in app) {
+            app.alternateDomains.forEach(function (d) {
+                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
+            });
+        }
+
+        if ('aliasDomains' in app) {
+            app.aliasDomains.forEach(function (d) {
+                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
+            });
+        }
+    }
+
+    if ('mounts' in app) {
+        queries.push({ query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ]});
+        app.mounts.forEach(function (m) {
+            queries.push({ query: 'INSERT INTO appMounts (appId, volumeId, readOnly) VALUES (?, ?, ?)', args: [ id, m.volumeId, m.readOnly ]});
+        });
+    }
+
+    var fields = [ ], values = [ ];
+    for (let p in app) {
+        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig') {
+            fields.push(`${p}Json = ?`);
+            values.push(JSON.stringify(app[p]));
+        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
+            fields.push(p + ' = ?');
+            values.push(app[p]);
+        }
+    }
+
+    if (values.length !== 0) {
+        values.push(id);
+        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
+    }
+
+    database.transaction(queries, function (error, results) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results[results.length - 1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        return callback(null);
+    });
+}
+
+function setHealth(appId, health, healthTime, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof health, 'string');
+    assert(util.types.isDate(healthTime));
+    assert.strictEqual(typeof callback, 'function');
+
+    const values = { health, healthTime };
+
+    updateWithConstraints(appId, values, '', callback);
+}
+
+function setTask(appId, values, options, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof values, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    values.ts = new Date();
+
+    if (!options.requireNullTaskId) return updateWithConstraints(appId, values, '', callback);
+
+    if (options.requiredState === null) {
+        updateWithConstraints(appId, values, 'AND taskId IS NULL', callback);
+    } else {
+        updateWithConstraints(appId, values, `AND taskId IS NULL AND installationState = "${options.requiredState}"`, callback);
+    }
+}
+
+function getAppStoreIds(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT id, appStoreId FROM apps', function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAddonConfig(appId, addonId, env, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert(Array.isArray(env));
+    assert.strictEqual(typeof callback, 'function');
+
+    unsetAddonConfig(appId, addonId, function (error) {
+        if (error) return callback(error);
+
+        if (env.length === 0) return callback(null);
+
+        var query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
+        var args = [ ], queryArgs = [ ];
+        for (var i = 0; i < env.length; i++) {
+            args.push(appId, addonId, env[i].name, env[i].value);
+            queryArgs.push('(?, ?, ?, ?)');
+        }
+
+        database.query(query + queryArgs.join(','), args, function (error) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            return callback(null);
+        });
+    });
+}
+
+function unsetAddonConfig(appId, addonId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function unsetAddonConfigByAppId(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getAddonConfig(appId, addonId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getAddonConfigByAppId(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getAppIdByAddonConfigValue(addonId, namePattern, value, callback) {
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        callback(null, results[0].appId);
+    });
+}
+
+function getAddonConfigByName(appId, addonId, namePattern, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        callback(null, results[0].value);
+    });
+}

src/apphealthmonitor.js

@@ -1,8 +1,10 @@
 'use strict';
 
-const apps = require('./apps.js'),
+const appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
     assert = require('assert'),
-    AuditSource = require('./auditsource.js'),
+    async = require('async'),
+    auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:apphealthmonitor'),
@@ -15,15 +17,17 @@ exports = module.exports = {
     run
 };
 
+const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
 const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes
 
 const OOM_EVENT_LIMIT = 60 * 60 * 1000; // will only raise 1 oom event every hour
 let gStartTime = null; // time when apphealthmonitor was started
 let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago
 
-async function setHealth(app, health) {
+function setHealth(app, health, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof health, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     // app starts out with null health
     // if it became healthy, we update immediately. this is required for ui to say "running" etc
@@ -38,74 +42,79 @@ async function setHealth(app, health) {
             debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, auditSource.HEALTH_MONITOR, { app: app });
         }
     } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
         if (lastHealth === apps.HEALTH_HEALTHY) {
             debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, auditSource.HEALTH_MONITOR, { app: app });
         }
     } else {
         debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);
-        return;
+        return callback(null);
     }
 
-    const [error] = await safe(apps.setHealth(app.id, health, healthTime));
-    if (error && error.reason === BoxError.NOT_FOUND) return; // app uninstalled?
-    if (error) throw error;
+    appdb.setHealth(app.id, health, healthTime, function (error) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(null); // app uninstalled?
+        if (error) return callback(error);
 
-    app.health = health;
-    app.healthTime = healthTime;
+        app.health = health;
+        app.healthTime = healthTime;
+
+        callback(null);
+    });
 }
 
 
 // callback is called with error for fatal errors and not if health check failed
-async function checkAppHealth(app, options) {
+function checkAppHealth(app, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) return;
+    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) {
+        return callback(null);
+    }
 
     const manifest = app.manifest;
 
-    const [error, data] = await safe(docker.inspect(app.containerId));
-    if (error || !data || !data.State) return await setHealth(app, apps.HEALTH_ERROR);
-    if (data.State.Running !== true) return await setHealth(app, apps.HEALTH_DEAD);
+    docker.inspect(app.containerId, function (error, data) {
+        if (error || !data || !data.State) return setHealth(app, apps.HEALTH_ERROR, callback);
+        if (data.State.Running !== true) return setHealth(app, apps.HEALTH_DEAD, callback);
 
-    // non-appstore apps may not have healthCheckPath
-    if (!manifest.healthCheckPath) return await setHealth(app, apps.HEALTH_HEALTHY);
+        // non-appstore apps may not have healthCheckPath
+        if (!manifest.healthCheckPath) return setHealth(app, apps.HEALTH_HEALTHY, callback);
 
-    const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
-    const [healthCheckError, response] = await safe(superagent
-        .get(healthCheckUrl)
-        .set('Host', app.fqdn) // required for some apache configs with rewrite rules
-        .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
-        .redirects(0)
-        .ok(() => true)
-        .timeout(options.timeout * 1000));
-
-    if (healthCheckError) {
-        await setHealth(app, apps.HEALTH_UNHEALTHY);
-    } else if (response.status > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
-        await setHealth(app, apps.HEALTH_UNHEALTHY);
-    } else {
-        await setHealth(app, apps.HEALTH_HEALTHY);
-    }
+        const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
+        superagent
+            .get(healthCheckUrl)
+            .set('Host', app.fqdn) // required for some apache configs with rewrite rules
+            .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
+            .redirects(0)
+            .timeout(HEALTHCHECK_INTERVAL)
+            .end(function (error, res) {
+                if (error && !error.response) {
+                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
+                } else if (res.statusCode > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
+                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
+                } else {
+                    setHealth(app, apps.HEALTH_HEALTHY, callback);
+                }
+            });
+    });
 }
 
-async function getContainerInfo(containerId) {
-    const result = await docker.inspect(containerId);
+function getContainerInfo(containerId, callback) {
+    docker.inspect(containerId, function (error, result) {
+        if (error) return callback(error);
 
-    const appId = safe.query(result, 'Config.Labels.appId', null);
-    if (appId) return { app: await apps.get(appId) }; // don't get by container id as this can be an exec container
+        const appId = safe.query(result, 'Config.Labels.appId', null);
 
-    if (result.Name.startsWith('/redis-')) {
-        return { app: await apps.get(result.Name.slice('/redis-'.length)), addonName: 'redis' };
-    } else {
-        return { addonName: result.Name.slice(1) }; // addon . Name has a '/' in the beginning for some reason
-    }
+        if (!appId) return callback(null, null /* app */, { name: result.Name.slice(1) }); // addon . Name has a '/' in the beginning for some reason
+
+        apps.get(appId, callback); // don't get by container id as this can be an exec container
+    });
 }
 
 /*
@@ -113,69 +122,81 @@ async function getContainerInfo(containerId) {
         docker run -ti -m 100M cloudron/base:3.0.0 /bin/bash
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
-async function processDockerEvents(options) {
-    assert.strictEqual(typeof options, 'object');
+function processDockerEvents(intervalSecs, callback) {
+    assert.strictEqual(typeof intervalSecs, 'number');
+    assert.strictEqual(typeof callback, 'function');
 
-    const since = ((new Date().getTime() / 1000) - options.intervalSecs).toFixed(0);
+    const since = ((new Date().getTime() / 1000) - intervalSecs).toFixed(0);
     const until = ((new Date().getTime() / 1000) - 1).toFixed(0);
 
-    const stream = await docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) });
-    stream.setEncoding('utf8');
-    stream.on('data', async function (data) { // this is actually ldjson, we only process the first line for now
-        const event = safe.JSON.parse(data);
-        if (!event) return;
-        const containerId = String(event.id);
+    docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
+        if (error) return callback(error);
 
-        const [error, info] = await safe(getContainerInfo(containerId));
-        const program = error ? containerId : (info.addonName || info.app.fqdn);
-        const now = Date.now();
+        stream.setEncoding('utf8');
+        stream.on('data', function (data) {
+            const event = JSON.parse(data);
+            const containerId = String(event.id);
 
-        // do not send mails for dev apps
-        const notifyUser = !(info.app && info.app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
+            getContainerInfo(containerId, function (error, app, addon) {
+                const program = error ? containerId : (app ? app.fqdn : addon.name);
+                const now = Date.now();
+                const notifyUser = !(app && app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
 
-        debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
+                debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
 
-        if (notifyUser) {
-            await eventlog.add(eventlog.ACTION_APP_OOM, AuditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null });
+                // do not send mails for dev apps
+                if (notifyUser) {
+                    // app can be null for addon containers
+                    eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event, containerId, addon: addon || null, app: app || null });
 
-            gLastOomMailTime = now;
-        }
+                    gLastOomMailTime = now;
+                }
+            });
+        });
+
+        stream.on('error', function (error) {
+            debug('Error reading docker events', error);
+            callback();
+        });
+
+        stream.on('end', callback);
+
+        // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
+        setTimeout(stream.destroy.bind(stream), 3000); // https://github.com/apocas/dockerode/issues/179
     });
-
-    stream.on('error', function (error) {
-        debug('Error reading docker events', error);
-    });
-
-    stream.on('end', function () {
-        // debug('Event stream ended');
-    });
-
-    // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
-    setTimeout(stream.destroy.bind(stream), options.timeout); // https://github.com/apocas/dockerode/issues/179
 }
 
-async function processApp(options) {
-    assert.strictEqual(typeof options, 'object');
+function processApp(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    const allApps = await apps.list();
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(error);
 
-    const healthChecks = allApps.map((app) => checkAppHealth(app, options)); // start healthcheck in parallel
+        async.each(allApps, checkAppHealth, function (error) {
+            const alive = allApps
+                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
 
-    await Promise.allSettled(healthChecks); // wait for all promises to finish
+            debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.` + (error ? ` ${error.reason}` : ''));
 
-    const alive = allApps
-        .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
-
-    debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.`);
+            callback(null);
+        });
+    });
 }
 
-async function run(intervalSecs) {
+function run(intervalSecs, callback) {
     assert.strictEqual(typeof intervalSecs, 'number');
+    assert.strictEqual(typeof callback, 'function');
 
     if (constants.TEST) return;
 
     if (!gStartTime) gStartTime = new Date();
 
-    await processApp({ timeout: (intervalSecs - 3) * 1000 });
-    await processDockerEvents({ intervalSecs, timeout: 3000 });
+    async.series([
+        processApp, // this is first because docker.getEvents seems to get 'stuck' sometimes
+        processDockerEvents.bind(null, intervalSecs)
+    ], function (error) {
+        if (error) debug(`run: could not check app health. ${error.message}`);
+
+        callback();
+    });
 }

src/apppasswords.js

@@ -1,88 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get,
-    add,
-    list,
-    del,
-
-    removePrivateFields
-};
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    crypto = require('crypto'),
-    database = require('./database.js'),
-    hat = require('./hat.js'),
-    safe = require('safetydance'),
-    uuid = require('uuid'),
-    _ = require('underscore');
-
-const APP_PASSWORD_FIELDS = [ 'id', 'name', 'userId', 'identifier', 'hashedPassword', 'creationTime' ].join(',');
-
-function validateAppPasswordName(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 char');
-    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'name too long');
-
-    return null;
-}
-
-function removePrivateFields(appPassword) {
-    return _.pick(appPassword, 'id', 'name', 'userId', 'identifier', 'creationTime');
-}
-
-async function get(id) {
-    assert.strictEqual(typeof id, 'string');
-
-    const result = await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE id = ?', [ id ]);
-    if (result.length === 0) return null;
-    return result[0];
-}
-
-async function add(userId, identifier, name) {
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof name, 'string');
-
-    let error = validateAppPasswordName(name);
-    if (error) throw error;
-
-    if (identifier.length < 1) throw new BoxError(BoxError.BAD_FIELD, 'identifier must be atleast 1 char');
-
-    const password = hat(16 * 4);
-    const hashedPassword = crypto.createHash('sha256').update(password).digest('base64');
-
-    const appPassword = {
-        id: 'uid-' + uuid.v4(),
-        name,
-        userId,
-        identifier,
-        password,
-        hashedPassword
-    };
-
-    const query = 'INSERT INTO appPasswords (id, userId, identifier, name, hashedPassword) VALUES (?, ?, ?, ?, ?)';
-    const args = [ appPassword.id, appPassword.userId, appPassword.identifier, appPassword.name, appPassword.hashedPassword ];
-
-    [error] = await safe(database.query(query, args));
-    if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('appPasswords_name_userId_identifier') !== -1) throw new BoxError(BoxError.ALREADY_EXISTS, 'name/app combination already exists');
-    if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.indexOf('userId')) throw new BoxError(BoxError.NOT_FOUND, 'user not found');
-    if (error) throw error;
-
-    return { id: appPassword.id, password: appPassword.password };
-}
-
-async function list(userId) {
-    assert.strictEqual(typeof userId, 'string');
-
-    return await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE userId = ?', [ userId ]);
-}
-
-async function del(id) {
-    assert.strictEqual(typeof id, 'string');
-
-    const result = await database.query('DELETE FROM appPasswords WHERE id = ?', [ id ]);
-    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'password not found');
-}

src/appstore.js

@@ -13,7 +13,7 @@ exports = module.exports = {
     purchaseApp,
     unpurchaseApp,
 
-    createUserToken,
+    getUserToken,
     getSubscription,
     isFreePlan,
 
@@ -23,8 +23,9 @@ exports = module.exports = {
     createTicket
 };
 
-const apps = require('./apps.js'),
+var apps = require('./apps.js'),
     assert = require('assert'),
+    async = require('async'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:appstore'),
@@ -73,86 +74,99 @@ function isAppAllowed(appstoreId, listingConfig) {
     return true;
 }
 
-async function login(email, password, totpToken) {
+function getCloudronToken(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getCloudronToken(function (error, token) {
+        if (error) return callback(error);
+        if (!token) return callback(new BoxError(BoxError.LICENSE_ERROR, 'Missing token'));
+
+        callback(null, token);
+    });
+}
+
+function login(email, password, totpToken, callback) {
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof totpToken, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const data = { email, password, totpToken };
+    var data = {
+        email: email,
+        password: password,
+        totpToken: totpToken
+    };
 
     const url = settings.apiServerOrigin() + '/api/v1/login';
-    const [error, response] = await safe(superagent.post(url)
-        .send(data)
-        .timeout(30 * 1000)
-        .ok(() => true));
+    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${result.statusCode}`));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${response.status}`);
-
-    return response.body; // { userId, accessToken }
+        callback(null, result.body); // { userId, accessToken }
+    });
 }
 
-async function registerUser(email, password) {
+function registerUser(email, password, callback) {
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const data = { email, password };
+    var data = {
+        email: email,
+        password: password,
+    };
 
     const url = settings.apiServerOrigin() + '/api/v1/register_user';
-    const [error, response] = await safe(superagent.post(url)
-        .send(data)
-        .timeout(30 * 1000)
-        .ok(() => true));
+    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${result.statusCode}`));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-
-    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, error.message);
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${response.status}`);
+        callback(null);
+    });
 }
 
-async function createUserToken() {
-    if (settings.isDemo()) throw new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode');
+function getUserToken(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    if (settings.isDemo()) return callback(new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode'));
 
-    const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const [error, response] = await safe(superagent.post(url)
-        .send({})
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
+        const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${response.status}`);
+        superagent.post(url).send({}).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${result.status}`));
 
-    return response.body.accessToken;
+            callback(null, result.body.accessToken);
+        });
+    });
 }
 
-async function getSubscription() {
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+function getSubscription(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    const url = settings.apiServerOrigin() + '/api/v1/subscription';
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const [error, response] = await safe(superagent.get(url)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
+        const url = settings.apiServerOrigin() + '/api/v1/subscription';
+        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR));
+            if (result.statusCode === 502) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`));
+            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR);
-    if (response.status === 502) throw new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`);
+            // update the features cache
+            gFeatures = result.body.features;
+            safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
 
-    // update the features cache
-    gFeatures = response.body.features;
-    safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
-
-    return response.body;
+            callback(null, result.body);
+        });
+    });
 }
 
 function isFreePlan(subscription) {
@@ -160,212 +174,225 @@ function isFreePlan(subscription) {
 }
 
 // See app.js install it will create a db record first but remove it again if appstore purchase fails
-async function purchaseApp(data) {
+function purchaseApp(data, callback) {
     assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId, appId }
     assert(data.appstoreId || data.manifestId);
     assert.strictEqual(typeof data.appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps`;
+        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps`;
 
-    const [error, response] = await safe(superagent.post(url)
-        .send(data)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
+        superagent.post(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); // appstoreId does not exist
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 402) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            // 200 if already purchased, 201 is newly purchased
+            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', result.status, result.body)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND); // appstoreId does not exist
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    // 200 if already purchased, 201 is newly purchased
-    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', response.status, response.body));
+            callback(null);
+        });
+    });
 }
 
-async function unpurchaseApp(appId, data) {
+function unpurchaseApp(appId, data, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId }
     assert(data.appstoreId || data.manifestId);
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
+        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
 
-    let [error, response] = await safe(superagent.get(url)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
+        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 404) return callback(null);   // was never purchased
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 404) return;   // was never purchased
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
+            superagent.del(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
+                if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
 
-    [error, response] = await safe(superagent.del(url)
-        .send(data)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
-
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
+                callback(null);
+            });
+        });
+    });
 }
 
-async function getBoxUpdate(options) {
+function getBoxUpdate(options, callback) {
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
+        const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
 
-    const query = {
-        accessToken: token,
-        boxVersion: constants.VERSION,
-        automatic: options.automatic
-    };
+        const query = {
+            accessToken: token,
+            boxVersion: constants.VERSION,
+            automatic: options.automatic
+        };
 
-    const [error, response] = await safe(superagent.get(url)
-        .query(query)
-        .timeout(30 * 1000)
-        .ok(() => true));
+        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode === 204) return callback(null, null); // no update
+            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status === 204) return; // no update
-    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+            var updateInfo = result.body;
 
-    const updateInfo = response.body;
+            if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
+                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', result.statusCode, result.text)));
+            }
 
-    if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
-        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', response.status, response.text));
-    }
+            // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
+            if (!updateInfo.version || typeof updateInfo.version !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', result.statusCode, result.text)));
 
-    // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
-    if (!updateInfo.version || typeof updateInfo.version !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
-    if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
-    if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', response.status, response.text));
-    if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', response.status, response.text));
-    if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', response.status, response.text));
-    if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', response.status, response.text));
-
-    return updateInfo;
+            callback(null, updateInfo);
+        });
+    });
 }
 
-async function getAppUpdate(app, options) {
+function getAppUpdate(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
-    const query = {
-        accessToken: token,
-        boxVersion: constants.VERSION,
-        appId: app.appStoreId,
-        appVersion: app.manifest.version,
-        automatic: options.automatic
-    };
+        const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
+        const query = {
+            accessToken: token,
+            boxVersion: constants.VERSION,
+            appId: app.appStoreId,
+            appVersion: app.manifest.version,
+            automatic: options.automatic
+        };
 
-    const [error, response] = await safe(superagent.get(url)
-        .query(query)
-        .timeout(30 * 1000)
-        .ok(() => true));
+        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode === 204) return callback(null); // no update
+            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status === 204) return; // no update
-    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+            const updateInfo = result.body;
 
-    const updateInfo = response.body;
+            // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
+            const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
 
-    // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
-    const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
+            // do some sanity checks
+            if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
+                debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
+                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', result.statusCode, result.text)));
+            }
 
-    // do some sanity checks
-    if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
-        debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
-        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', response.status, response.text));
-    }
+            updateInfo.unstable = !!updateInfo.unstable;
 
-    updateInfo.unstable = !!updateInfo.unstable;
-
-    // { id, creationDate, manifest, unstable }
-    return updateInfo;
+            // { id, creationDate, manifest, unstable }
+            callback(null, updateInfo);
+        });
+    });
 }
 
-async function registerCloudron(data) {
+function registerCloudron(data, callback) {
     assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     const url = `${settings.apiServerOrigin()}/api/v1/register_cloudron`;
 
-    const [error, response] = await safe(superagent.post(url)
-        .send(data)
-        .timeout(30 * 1000)
-        .ok(() => true));
+    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${result.statusCode} ${error.message}`));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${response.statusCode} ${error.message}`);
+        // cloudronId, token, licenseKey
+        if (!result.body.cloudronId) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id'));
+        if (!result.body.cloudronToken) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token'));
+        if (!result.body.licenseKey) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no license'));
 
-    // cloudronId, token, licenseKey
-    if (!response.body.cloudronId) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id');
-    if (!response.body.cloudronToken) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token');
-    if (!response.body.licenseKey) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no license');
+        async.series([
+            settings.setCloudronId.bind(null, result.body.cloudronId),
+            settings.setCloudronToken.bind(null, result.body.cloudronToken),
+            settings.setLicenseKey.bind(null, result.body.licenseKey),
+        ], function (error) {
+            if (error) return callback(error);
 
-    await settings.setCloudronId(response.body.cloudronId);
-    await settings.setCloudronToken(response.body.cloudronToken);
-    await settings.setLicenseKey(response.body.licenseKey);
+            debug(`registerCloudron: Cloudron registered with id ${result.body.cloudronId}`);
 
-    debug(`registerCloudron: Cloudron registered with id ${response.body.cloudronId}`);
+            callback();
+        });
+    });
 }
 
-async function updateCloudron(data) {
+function updateCloudron(data, callback) {
     assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    getCloudronToken(function (error, token) {
+        if (error && error.reason === BoxError.LICENSE_ERROR) return callback(null); // missing token. not registered yet
+        if (error) return callback(error);
 
-    const url = `${settings.apiServerOrigin()}/api/v1/update_cloudron`;
-    const query = {
-        accessToken: token
-    };
+        const url = `${settings.apiServerOrigin()}/api/v1/update_cloudron`;
+        const query = {
+            accessToken: token
+        };
 
-    const [error, response] = await safe(superagent.post(url)
-        .query(query)
-        .send(data)
-        .timeout(30 * 1000)
-        .ok(() => true));
+        superagent.post(url).query(query).send(data).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+            debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
 
-    debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
+            callback();
+        });
+    });
 }
 
-async function registerWithLoginCredentials(options) {
+function registerWithLoginCredentials(options, callback) {
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getCloudronToken();
-    if (token) throw new BoxError(BoxError.CONFLICT, 'Cloudron is already registered');
+    function maybeSignup(done) {
+        if (!options.signup) return done();
 
-    if (options.signup) await registerUser(options.email, options.password);
+        registerUser(options.email, options.password, done);
+    }
 
-    const result = await login(options.email, options.password, options.totpToken || '');
-    await registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION });
+    getCloudronToken(function (error, token) {
+        if (token) return callback(new BoxError(BoxError.CONFLICT, 'Cloudron is already registered'));
+
+        maybeSignup(function (error) {
+            if (error) return callback(error);
+
+            login(options.email, options.password, options.totpToken || '', function (error, result) {
+                if (error) return callback(error);
+
+                registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
+            });
+        });
+    });
 }
 
-async function createTicket(info, auditSource) {
+function createTicket(info, auditSource, callback) {
     assert.strictEqual(typeof info, 'object');
     assert.strictEqual(typeof info.email, 'string');
     assert.strictEqual(typeof info.displayName, 'string');
@@ -373,99 +400,128 @@ async function createTicket(info, auditSource) {
     assert.strictEqual(typeof info.subject, 'string');
     assert.strictEqual(typeof info.description, 'string');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
-
-    if (info.enableSshSupport) {
-        await safe(support.enableRemoteSupport(true, auditSource));
+    function collectAppInfoIfNeeded(callback) {
+        if (!info.appId) return callback();
+        apps.get(info.appId, callback);
     }
 
-    info.app = info.appId ? await apps.get(info.appId) : null;
-    info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
+    function enableSshIfNeeded(callback) {
+        if (!info.enableSshSupport) return callback();
 
-    const request = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true);
+        support.enableRemoteSupport(true, auditSource, function (error) {
+            // ensure we can at least get the ticket through
+            if (error) debug('Unable to enable SSH support.', error);
 
-    // either send as JSON through body or as multipart, depending on attachments
-    if (info.app) {
-        request.field('infoJSON', JSON.stringify(info));
-
-        apps.getLocalLogfilePaths(info.app).forEach(function (filePath) {
-            const logs = safe.child_process.execSync(`tail --lines=1000 ${filePath}`);
-            if (logs) request.attach(path.basename(filePath), logs, path.basename(filePath));
+            callback();
         });
-    } else {
-        request.send(info);
     }
 
-    const [error, response] = await safe(request);
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+        enableSshIfNeeded(function (error) {
+            if (error) return callback(error);
 
-    return { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` };
+            collectAppInfoIfNeeded(function (error, app) {
+                if (error) return callback(error);
+                if (app) info.app = app;
+
+                info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
+
+                var req = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
+                    .query({ accessToken: token })
+                    .timeout(30 * 1000);
+
+                // either send as JSON through body or as multipart, depending on attachments
+                if (info.app) {
+                    req.field('infoJSON', JSON.stringify(info));
+
+                    apps.getLocalLogfilePaths(info.app).forEach(function (filePath) {
+                        var logs = safe.child_process.execSync(`tail --lines=1000 ${filePath}`);
+                        if (logs) req.attach(path.basename(filePath), logs, path.basename(filePath));
+                    });
+                } else {
+                    req.send(info);
+                }
+
+                req.end(function (error, result) {
+                    if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                    if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                    if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                    if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+
+                    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+
+                    callback(null, { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
+                });
+            });
+        });
+    });
 }
 
-async function getApps() {
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+function getApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    const unstable = await settings.getUnstableAppsConfig();
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const url = `${settings.apiServerOrigin()}/api/v1/apps`;
+        settings.getUnstableAppsConfig(function (error, unstable) {
+            if (error) return callback(error);
 
-    const [error, response] = await safe(superagent.get(url)
-        .query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable })
-        .timeout(30 * 1000)
-        .ok(() => true));
+            const url = `${settings.apiServerOrigin()}/api/v1/apps`;
+            superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable }).timeout(30 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
+                if (!result.body.apps) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 403 || response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', response.status, response.body));
-    if (!response.body.apps) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+                settings.getAppstoreListingConfig(function (error, listingConfig) {
+                    if (error) return callback(error);
 
-    const listingConfig = await settings.getAppstoreListingConfig();
-    const filteredApps = response.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
-    return filteredApps;
+                    const filteredApps = result.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
+
+                    callback(null, filteredApps);
+                });
+            });
+        });
+    });
 }
 
-async function getAppVersion(appId, version) {
+function getAppVersion(appId, version, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof version, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const listingConfig = await settings.getAppstoreListingConfig();
+    settings.getAppstoreListingConfig(function (error, listingConfig) {
+        if (error) return callback(error);
 
-    if (!isAppAllowed(appId, listingConfig)) throw new BoxError(BoxError.FEATURE_DISABLED);
+        if (!isAppAllowed(appId, listingConfig)) return callback(new BoxError(BoxError.FEATURE_DISABLED));
 
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+        getCloudronToken(function (error, token) {
+            if (error) return callback(error);
 
-    let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
-    if (version !== 'latest') url += `/versions/${version}`;
+            let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
+            if (version !== 'latest') url += `/versions/${version}`;
 
-    const [error, response] = await safe(superagent.get(url)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
+            superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 403 || response.statusCode === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', response.status, response.body));
-
-    return response.body;
+                callback(null, result.body);
+            });
+        });
+    });
 }
 
-async function getApp(appId) {
+function getApp(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    return await getAppVersion(appId, 'latest');
+    getAppVersion(appId, 'latest', callback);
 }

src/apptaskmanager.js

@@ -20,11 +20,12 @@ let gPendingTasks = [ ];
 let gInitialized = false;
 
 const TASK_CONCURRENCY = 3;
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 function waitText(lockOperation) {
     if (lockOperation === locker.OP_BOX_UPDATE) return 'Waiting for Cloudron to finish updating. See the Settings view';
     if (lockOperation === locker.OP_PLATFORM_START) return 'Waiting for Cloudron to initialize';
-    if (lockOperation === locker.OP_FULL_BACKUP) return 'Waiting for Cloudron to finish backup. See the Backups view';
+    if (lockOperation === locker.OP_FULL_BACKUP) return 'Wait for Cloudron to finish backup. See the Backups view';
 
     return ''; // cannot happen
 }
@@ -35,31 +36,31 @@ function initializeSync() {
 }
 
 // callback is called when task is finished
-function scheduleTask(appId, taskId, options, onFinished) {
+function scheduleTask(appId, taskId, options, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof taskId, 'string');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof onFinished, 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     if (!gInitialized) initializeSync();
 
     if (appId in gActiveTasks) {
-        return onFinished(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
+        return callback(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
     }
 
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' });
-        gPendingTasks.push({ appId, taskId, options, onFinished });
+        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
+        gPendingTasks.push({ appId, taskId, options, callback });
         return;
     }
 
-    const lockError = locker.recursiveLock(locker.OP_APPTASK);
+    var lockError = locker.recursiveLock(locker.OP_APPTASK);
 
     if (lockError) {
         debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) });
-        gPendingTasks.push({ appId, taskId, options, onFinished });
+        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
+        gPendingTasks.push({ appId, taskId, options, callback });
         return;
     }
 
@@ -72,7 +73,7 @@ function scheduleTask(appId, taskId, options, onFinished) {
     scheduler.suspendJobs(appId);
 
     tasks.startTask(taskId, Object.assign(options, { logFile }), function (error, result) {
-        onFinished(error, result);
+        callback(error, result);
 
         delete gActiveTasks[appId];
         locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
@@ -87,5 +88,5 @@ function startNextTask() {
     assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
     const t = gPendingTasks.shift();
-    scheduleTask(t.appId, t.taskId, t.options, t.onFinished);
+    scheduleTask(t.appId, t.taskId, t.options, t.callback);
 }

src/auditsource.js

@@ -1,23 +1,15 @@
 'use strict';
 
-class AuditSource {
-    constructor(username, userId, ip) {
-        this.username = username;
-        this.userId = userId || null;
-        this.ip = ip || null;
-    }
+exports = module.exports = {
+    CRON:  { userId: null, username: 'cron' },
+    HEALTH_MONITOR: { userId: null, username: 'healthmonitor' },
+    EXTERNAL_LDAP_TASK: { userId: null, username: 'externalldap' },
+    EXTERNAL_LDAP_AUTO_CREATE: { userId: null, username: 'externalldap' },
 
-    static fromRequest(req) {
-        const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-        return new AuditSource(req.user?.username, req.user?.id, ip);
-    }
+    fromRequest
+};
+
+function fromRequest(req) {
+    const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
 }
-
-// these can be static variables but see https://stackoverflow.com/questions/60046847/eslint-does-not-allow-static-class-properties#comment122122927_60464446
-AuditSource.CRON =  new AuditSource('cron');
-AuditSource.HEALTH_MONITOR = new AuditSource('healthmonitor');
-AuditSource.EXTERNAL_LDAP_TASK = new AuditSource('externalldap');
-AuditSource.EXTERNAL_LDAP_AUTO_CREATE = new AuditSource('externalldap');
-AuditSource.APPTASK = new AuditSource('apptask');
-
-exports = module.exports = AuditSource;

src/autoconfig.xml.ejs

@@ -19,13 +19,7 @@
       <username>%EMAILADDRESS%</username>
       <addThisServer>true</addThisServer>
     </outgoingServer>
-    <incomingServer type="pop3">
-      <hostname><%= mailFqdn %></hostname>
-      <port>995</port>
-      <socketType>SSL</socketType>
-      <username>%EMAILADDRESS%</username>
-      <authentication>password-cleartext</authentication>
-    </incomingServer>
+
     <documentation url="http://cloudron.io/email/#autodiscover">
        <descr lang="en">Cloudron Email</descr>
     </documentation>

src/backupcleaner.js

@@ -1,280 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    run,
-
-    _applyBackupRetentionPolicy: applyBackupRetentionPolicy
-};
-
-const apps = require('./apps.js'),
-    assert = require('assert'),
-    backups = require('./backups.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:backupcleaner'),
-    moment = require('moment'),
-    path = require('path'),
-    paths = require('./paths.js'),
-    safe = require('safetydance'),
-    settings = require('./settings.js'),
-    storage = require('./storage.js'),
-    util = require('util'),
-    _ = require('underscore');
-
-function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
-    assert(Array.isArray(allBackups));
-    assert.strictEqual(typeof policy, 'object');
-    assert(Array.isArray(referencedBackupIds));
-
-    const now = new Date();
-
-    for (const backup of allBackups) {
-        if (backup.state === backups.BACKUP_STATE_ERROR) {
-            backup.discardReason = 'error';
-        } else if (backup.state === backups.BACKUP_STATE_CREATING) {
-            if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
-            else backup.discardReason = 'creating-too-long';
-        } else if (referencedBackupIds.includes(backup.id)) {
-            backup.keepReason = 'reference';
-        } else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) {
-            backup.keepReason = 'preserveSecs';
-        } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) {
-            backup.keepReason = 'keepWithinSecs';
-        }
-    }
-
-    const KEEP_FORMATS = {
-        keepDaily: 'Y-M-D',
-        keepWeekly: 'Y-W',
-        keepMonthly: 'Y-M',
-        keepYearly: 'Y'
-    };
-
-    for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) {
-        if (!(format in policy)) continue;
-
-        const n = policy[format]; // we want to keep "n" backups of format
-        if (!n) continue; // disabled rule
-
-        let lastPeriod = null, keptSoFar = 0;
-        for (const backup of allBackups) {
-            if (backup.discardReason) continue; // already discarded for some reason
-            if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason
-            const period = moment(backup.creationTime).format(KEEP_FORMATS[format]);
-            if (period === lastPeriod) continue; // already kept for this period
-
-            lastPeriod = period;
-            backup.keepReason = backup.keepReason ? `${backup.keepReason}+${format}` : format;
-            if (++keptSoFar === n) break;
-        }
-    }
-
-    if (policy.keepLatest) {
-        let latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL);
-        if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest';
-    }
-
-    for (const backup of allBackups) {
-        debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
-    }
-}
-
-async function cleanupBackup(backupConfig, backup, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof backup, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
-
-    return new Promise((resolve) => {
-        function done(error) {
-            if (error) {
-                debug('cleanupBackup: error removing backup %j : %s', backup, error.message);
-                return resolve();
-            }
-
-            // prune empty directory if possible
-            storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), async function (error) {
-                if (error) debug('cleanupBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message);
-
-                const [delError] = await safe(backups.del(backup.id));
-                if (delError) debug('cleanupBackup: error removing from database', delError);
-                else debug('cleanupBackup: removed %s', backup.id);
-
-                resolve();
-            });
-        }
-
-        if (backup.format ==='tgz') {
-            progressCallback({ message: `${backup.id}: Removing ${backupFilePath}`});
-            storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, done);
-        } else {
-            const events = storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath);
-            events.on('progress', (message) => progressCallback({ message: `${backup.id}: ${message}` }));
-            events.on('done', done);
-        }
-    });
-}
-
-async function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert(Array.isArray(referencedAppBackupIds));
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    let removedAppBackupIds = [];
-
-    const allApps = await apps.list();
-    const allAppIds = allApps.map(a => a.id);
-
-    const appBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000);
-
-    // collate the backups by app id. note that the app could already have been uninstalled
-    let appBackupsById = {};
-    for (const appBackup of appBackups) {
-        if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = [];
-        appBackupsById[appBackup.identifier].push(appBackup);
-    }
-
-    // apply backup policy per app. keep latest backup only for existing apps
-    let appBackupsToRemove = [];
-    for (const appId of Object.keys(appBackupsById)) {
-        applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds);
-        appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
-    }
-
-    for (const appBackup of appBackupsToRemove) {
-        await progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
-        removedAppBackupIds.push(appBackup.id);
-        await cleanupBackup(backupConfig, appBackup, progressCallback); // never errors
-    }
-
-    debug('cleanupAppBackups: done');
-
-    return removedAppBackupIds;
-}
-
-async function cleanupBoxBackups(backupConfig, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    let referencedAppBackupIds = [], removedBoxBackupIds = [];
-
-    const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
-
-    applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */);
-
-    for (const boxBackup of boxBackups) {
-        if (boxBackup.keepReason) {
-            referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn);
-            continue;
-        }
-
-        await progressCallback({ message: `Removing box backup ${boxBackup.id}`});
-
-        removedBoxBackupIds.push(boxBackup.id);
-        await cleanupBackup(backupConfig, boxBackup, progressCallback);
-    }
-
-    debug('cleanupBoxBackups: done');
-
-    return { removedBoxBackupIds, referencedAppBackupIds };
-}
-
-async function cleanupMissingBackups(backupConfig, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const perPage = 1000;
-    let missingBackupIds = [];
-    const backupExists = util.promisify(storage.api(backupConfig.provider).exists);
-
-    if (constants.TEST) return missingBackupIds;
-
-    let page = 1, result = [];
-    do {
-        result = await backups.list(page, perPage);
-
-        for (const backup of result) {
-            let backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
-            if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
-
-            const [existsError, exists] = await safe(backupExists(backupConfig, backupFilePath));
-            if (existsError || exists) continue;
-
-            await progressCallback({ message: `Removing missing backup ${backup.id}`});
-
-            const [delError] = await safe(backups.del(backup.id));
-            if (delError) debug(`cleanupBackup: error removing ${backup.id} from database`, delError);
-
-            missingBackupIds.push(backup.id);
-        }
-
-        ++ page;
-    } while (result.length === perPage);
-
-    debug('cleanupMissingBackups: done');
-
-    return missingBackupIds;
-}
-
-// removes the snapshots of apps that have been uninstalled
-async function cleanupSnapshots(backupConfig) {
-    assert.strictEqual(typeof backupConfig, 'object');
-
-    const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
-    const info = safe.JSON.parse(contents);
-    if (!info) return;
-
-    delete info.box;
-
-    for (const appId of Object.keys(info)) {
-        const app = await apps.get(appId);
-        if (app) continue; // app is still installed
-
-        await new Promise((resolve) => {
-            async function done(/* ignoredError */) {
-                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
-                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`));
-
-                await safe(backups.setSnapshotInfo(appId, null /* info */), { debug });
-                debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`);
-
-                resolve();
-            }
-
-            if (info[appId].format ==='tgz') {
-                storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), done);
-            } else {
-                const events = storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format));
-                events.on('progress', function (detail) { debug(`cleanupSnapshots: ${detail}`); });
-                events.on('done', done);
-            }
-        });
-    }
-
-    debug('cleanupSnapshots: done');
-}
-
-async function run(progressCallback) {
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const backupConfig = await settings.getBackupConfig();
-
-    if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
-        debug('cleanup: keeping all backups');
-        return {};
-    }
-
-    await progressCallback({ percent: 10, message: 'Cleaning box backups' });
-    const { removedBoxBackupIds, referencedAppBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback);
-
-    await progressCallback({ percent: 40, message: 'Cleaning app backups' });
-    const removedAppBackupIds = await cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback);
-
-    await progressCallback({ percent: 70, message: 'Cleaning missing backups' });
-    const missingBackupIds = await cleanupMissingBackups(backupConfig, progressCallback);
-
-    await progressCallback({ percent: 90, message: 'Cleaning snapshots' });
-    await cleanupSnapshots(backupConfig);
-
-    return { removedBoxBackupIds, removedAppBackupIds, missingBackupIds };
-}

src/backupdb.js

@@ -0,0 +1,176 @@
+'use strict';
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    safe = require('safetydance');
+
+const BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
+
+exports = module.exports = {
+    add,
+
+    getByTypePaged,
+    getByIdentifierPaged,
+    getByIdentifierAndStatePaged,
+
+    get,
+    del,
+    update,
+    list,
+
+    _clear: clear
+};
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
+
+    result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
+    delete result.manifestJson;
+}
+
+function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof state, 'string');
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ identifier, state, (page-1)*perPage, perPage ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
+function getByTypePaged(type, page, perPage, callback) {
+    assert.strictEqual(typeof type, 'string');
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ type, (page-1)*perPage, perPage ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
+function getByIdentifierPaged(identifier, page, perPage, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ identifier, (page-1)*perPage, perPage ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
+function list(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups ORDER BY creationTime DESC LIMIT ?,?',
+        [ (page-1)*perPage, perPage ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
+        [ id ], function (error, result) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+            if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
+
+            postProcess(result[0]);
+
+            callback(null, result[0]);
+        });
+}
+
+function add(id, data, callback) {
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof id, 'string');
+    assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number');
+    assert.strictEqual(typeof data.packageVersion, 'string');
+    assert.strictEqual(typeof data.type, 'string');
+    assert.strictEqual(typeof data.identifier, 'string');
+    assert.strictEqual(typeof data.state, 'string');
+    assert(Array.isArray(data.dependsOn));
+    assert.strictEqual(typeof data.manifest, 'object');
+    assert.strictEqual(typeof data.format, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var creationTime = data.creationTime || new Date(); // allow tests to set the time
+    var manifestJson = JSON.stringify(data.manifest);
+
+    database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ],
+        function (error) {
+            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            callback(null);
+        });
+}
+
+function update(id, backup, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof backup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var fields = [ ], values = [ ];
+    for (var p in backup) {
+        fields.push(p + ' = ?');
+        values.push(backup[p]);
+    }
+    values.push(id);
+
+    database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values, function (error) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE backups', [], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        callback(null);
+    });
+}

src/blobs.js

@@ -68,18 +68,17 @@ async function initSecrets() {
         await set(exports.ADDON_TURN_SECRET, Buffer.from(turnSecret));
     }
 
-    // TODO maybe skip this in tests if possible again
-    let dhparams = await get(exports.DHPARAMS);
-    if (!dhparams) {
-        debug('initSecrets: generating dhparams.pem. this takes forever');
-        if (constants.TEST) dhparams = safe.fs.readFileSync('/tmp/dhparams.pem');
-        if (!dhparams) dhparams = safe.child_process.execSync('openssl dhparam 2048');
-        if (!dhparams) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
-        if (constants.TEST) safe.fs.writeFileSync('/tmp/dhparams.pem', dhparams);
-        if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
-        await set(exports.DHPARAMS, dhparams);
-    } else if (!safe.fs.existsSync(paths.DHPARAMS_FILE)) {
-        if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
+    if (!constants.TEST) {
+        let dhparams = await get(exports.DHPARAMS);
+        if (!dhparams) {
+            debug('initSecrets: generating dhparams.pem. this takes forever');
+            dhparams = safe.child_process.execSync('openssl dhparam 2048');
+            if (!dhparams) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+            if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
+            await set(exports.DHPARAMS, dhparams);
+        } else if (!safe.fs.existsSync(paths.DHPARAMS_FILE)) {
+            if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
+        }
     }
 
     let sftpPrivateKey = await get(exports.SFTP_PRIVATE_KEY);

src/boxerror.js

@@ -60,7 +60,6 @@ BoxError.NGINX_ERROR = 'Nginx Error';
 BoxError.NOT_FOUND = 'Not found';
 BoxError.NOT_IMPLEMENTED = 'Not implemented';
 BoxError.NOT_SIGNED = 'Not Signed';
-BoxError.NOT_SUPPORTED = 'Not Supported';
 BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.PLAN_LIMIT = 'Plan Limit';
 BoxError.SPAWN_ERROR = 'Spawn Error';
@@ -86,7 +85,6 @@ BoxError.toHttpError = function (error) {
     case BoxError.ALREADY_EXISTS:
     case BoxError.BAD_STATE:
     case BoxError.CONFLICT:
-    case BoxError.NOT_SUPPORTED:
         return new HttpError(409, error);
     case BoxError.INVALID_CREDENTIALS:
         return new HttpError(412, error);

src/cloudron.js

@@ -10,7 +10,6 @@ exports = module.exports = {
     isRebootRequired,
 
     onActivated,
-    onRestored,
 
     setupDnsAndCert,
 
@@ -26,16 +25,14 @@ exports = module.exports = {
 const apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
-    AuditSource = require('./auditsource.js'),
+    async = require('async'),
+    auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
-    backuptask = require('./backuptask.js'),
     BoxError = require('./boxerror.js'),
     branding = require('./branding.js'),
     constants = require('./constants.js'),
     cron = require('./cron.js'),
     debug = require('debug')('box:cloudron'),
-    delay = require('delay'),
-    dns = require('./dns.js'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
@@ -57,175 +54,196 @@ const apps = require('./apps.js'),
 
 const REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');
 
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
 async function initialize() {
-    safe(runStartupTasks(), { debug });
+    runStartupTasks();
 
     await notifyUpdate();
 }
 
-async function uninitialize() {
-    await cron.stopJobs();
-    await platform.stopAllTasks();
+function uninitialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        cron.stopJobs,
+        platform.stopAllTasks
+    ], callback);
 }
 
-async function onActivated(options) {
+function onActivated(options, callback) {
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     debug('onActivated: running post activation tasks');
 
     // Starting the platform after a user is available means:
     // 1. mail bounces can now be sent to the cloudron owner
     // 2. the restore code path can run without sudo (since mail/ is non-root)
-    await platform.start(options);
-    await cron.startJobs();
-
-    // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
-    // the UI some time to query the dashboard domain in the restore code path
-    await delay(30000);
-    await reverseProxy.writeDefaultConfig({ activated :true });
-}
-
-async function onRestored(options) {
-    assert.strictEqual(typeof options, 'object');
-
-    debug('onRestored: downloading mail');
-
-    const [error, results] = await safe(backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_MAIL, backups.BACKUP_STATE_NORMAL, 1, 1));
-    if (error || results.length == 0) {
-        debug('startMail: could not find backup to restore', error, results);
-    } else {
-        debug(`startMail: downloading backup ${results[0].id}`);
-        const restoreConfig = { backupId: results[0].id, backupFormat: results[0].format };
-        // have to wait for download before starting mail container because we download as non-root user
-        await backuptask.downloadMail(restoreConfig, (progress) => debug(`startMail: ${progress.message}`));
-    }
-
-    await onActivated(options);
+    async.series([
+        platform.start.bind(null, options),
+        cron.startJobs,
+        // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
+        // the UI some time to query the dashboard domain in the restore code path
+        (done) => setTimeout(() => reverseProxy.writeDefaultConfig({ activated :true }, done), 30000)
+    ], callback);
 }
 
 async function notifyUpdate() {
     const version = safe.fs.readFileSync(paths.VERSION_FILE, 'utf8');
     if (version === constants.VERSION) return;
 
-    await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
+    await eventlog.add(eventlog.ACTION_UPDATE_FINISH, auditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
 
-    const [error] = await safe(tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }));
-    if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist
+    return new Promise((resolve, reject) => {
+        tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }, function (error) {
+            if (error && error.reason !== BoxError.NOT_FOUND) return reject(error); // when hotfixing, task may not exist
 
-    safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
+            safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
+
+            resolve();
+        });
+    });
 }
 
 // each of these tasks can fail. we will add some routes to fix/re-run them
-async function runStartupTasks() {
-    const tasks = [];
+function runStartupTasks() {
+    const tasks = [
+        // stop all the systemd tasks
+        platform.stopAllTasks,
 
-    // stop all the systemd tasks
-    tasks.push(platform.stopAllTasks);
+        // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
+        function (callback) {
+            settings.getBackupConfig(function (error, backupConfig) {
+                if (error) return callback(error);
 
-    // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
-    tasks.push(async function () {
-        const backupConfig = await settings.getBackupConfig();
-        await backups.configureCollectd(backupConfig);
-    });
+                backups.configureCollectd(backupConfig, callback);
+            });
+        },
 
-    // always generate webadmin config since we have no versioning mechanism for the ejs
-    tasks.push(async function () {
-        if (!settings.dashboardDomain()) return;
+        // always generate webadmin config since we have no versioning mechanism for the ejs
+        function (callback) {
+            if (!settings.dashboardDomain()) return callback();
 
-        await reverseProxy.writeDashboardConfig(settings.dashboardDomain());
-    });
+            reverseProxy.writeDashboardConfig(settings.dashboardDomain(), callback);
+        },
 
-    tasks.push(async function () {
         // check activation state and start the platform
-        const activated = await users.isActivated();
+        function (callback) {
+            users.isActivated(function (error, activated) {
+                if (error) return callback(error);
 
-        // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
-        // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
-        // we remove the config as a simple security measure to not expose IP <-> domain
-        if (!activated) {
-            debug('runStartupTasks: not activated. generating IP based redirection config');
-            return await reverseProxy.writeDefaultConfig({ activated: false });
+                // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
+                // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
+                // we remove the config as a simple security measure to not expose IP <-> domain
+                if (!activated) {
+                    debug('runStartupTasks: not activated. generating IP based redirection config');
+                    return reverseProxy.writeDefaultConfig({ activated: false }, callback);
+                }
+
+                onActivated({}, callback);
+            });
         }
-
-        await onActivated({});
-    });
-
-    // we used to run tasks in parallel but simultaneous nginx reloads was causing issues
-    for (let i = 0; i < tasks.length; i++) {
-        const [error] = await safe(tasks[i]());
-        if (error) debug(`Startup task at index ${i} failed: ${error.message}`);
-    }
-}
-
-async function getConfig() {
-    const release = safe.fs.readFileSync('/etc/lsb-release', 'utf-8');
-    if (release === null) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
-    const ubuntuVersion = release.match(/DISTRIB_DESCRIPTION="(.*)"/)[1];
-
-    const allSettings = await settings.list();
-
-    // be picky about what we send out here since this is sent for 'normal' users as well
-    return {
-        apiServerOrigin: settings.apiServerOrigin(),
-        webServerOrigin: settings.webServerOrigin(),
-        adminDomain: settings.dashboardDomain(),
-        adminFqdn: settings.dashboardFqdn(),
-        mailFqdn: settings.mailFqdn(),
-        version: constants.VERSION,
-        ubuntuVersion,
-        isDemo: settings.isDemo(),
-        cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
-        footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
-        features: appstore.getFeatures(),
-        profileLocked: allSettings[settings.DIRECTORY_CONFIG_KEY].lockUserProfiles,
-        mandatory2FA: allSettings[settings.DIRECTORY_CONFIG_KEY].mandatory2FA
-    };
-}
-
-async function reboot() {
-    await notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '');
-
-    const [error] = await safe(shell.promises.sudo('reboot', [ REBOOT_CMD ], {}));
-    if (error) debug('reboot: could not reboot', error);
-}
-
-async function isRebootRequired() {
-    // https://serverfault.com/questions/92932/how-does-ubuntu-keep-track-of-the-system-restart-required-flag-in-motd
-    return fs.existsSync('/var/run/reboot-required');
-}
-
-async function runSystemChecks() {
-    debug('runSystemChecks: checking status');
-
-    const checks = [
-        checkMailStatus(),
-        checkRebootRequired(),
-        checkUbuntuVersion()
     ];
 
-    await Promise.allSettled(checks);
+    // we used to run tasks in parallel but simultaneous nginx reloads was causing issues
+    async.series(async.reflectAll(tasks), function (error, results) {
+        results.forEach((result, idx) => {
+            if (result.error) debug(`Startup task at index ${idx} failed: ${result.error.message}`);
+        });
+    });
 }
 
-async function checkMailStatus() {
-    const message = await mail.checkConfiguration();
-    await notifications.alert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly', message);
+function getConfig(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    const release = safe.fs.readFileSync('/etc/lsb-release', 'utf-8');
+    if (release === null) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
+    const ubuntuVersion = release.match(/DISTRIB_DESCRIPTION="(.*)"/)[1];
+
+    settings.getAll(function (error, allSettings) {
+        if (error) return callback(error);
+
+        // be picky about what we send out here since this is sent for 'normal' users as well
+        callback(null, {
+            apiServerOrigin: settings.apiServerOrigin(),
+            webServerOrigin: settings.webServerOrigin(),
+            adminDomain: settings.dashboardDomain(),
+            adminFqdn: settings.dashboardFqdn(),
+            mailFqdn: settings.mailFqdn(),
+            version: constants.VERSION,
+            ubuntuVersion,
+            isDemo: settings.isDemo(),
+            cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
+            footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
+            features: appstore.getFeatures(),
+            profileLocked: allSettings[settings.DIRECTORY_CONFIG_KEY].lockUserProfiles,
+            mandatory2FA: allSettings[settings.DIRECTORY_CONFIG_KEY].mandatory2FA
+        });
+    });
 }
 
-async function checkRebootRequired() {
-    const rebootRequired = await isRebootRequired();
-    await notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '');
+function reboot(callback) {
+    notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '', function (error) {
+        if (error) debug('reboot: failed to clear reboot notification.', error);
+
+        shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
+    });
 }
 
-async function checkUbuntuVersion() {
+function isRebootRequired(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // https://serverfault.com/questions/92932/how-does-ubuntu-keep-track-of-the-system-restart-required-flag-in-motd
+    callback(null, fs.existsSync('/var/run/reboot-required'));
+}
+
+// called from cron.js
+function runSystemChecks(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('runSystemChecks: checking status');
+
+    async.parallel([
+        checkMailStatus,
+        checkRebootRequired,
+        checkUbuntuVersion
+    ], callback);
+}
+
+function checkMailStatus(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    mail.checkConfiguration(function (error, message) {
+        if (error) return callback(error);
+
+        notifications.alert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly', message, callback);
+    });
+}
+
+function checkRebootRequired(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    isRebootRequired(function (error, rebootRequired) {
+        if (error) return callback(error);
+
+        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '', callback);
+    });
+}
+
+function checkUbuntuVersion(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     const isXenial = fs.readFileSync('/etc/lsb-release', 'utf-8').includes('16.04');
-    if (!isXenial) return;
+    if (!isXenial) return callback();
 
-    await notifications.alert(notifications.ALERT_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 16.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-18/ to upgrade to Ubuntu 18 at the earliest.');
+    notifications.alert(notifications.ALERT_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 16.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-18/ to upgrade to Ubuntu 18 at the earliest.', callback);
 }
 
-async function getLogs(unit, options) {
+function getLogs(unit, options, callback) {
     assert.strictEqual(typeof unit, 'string');
     assert(options && typeof options === 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     assert.strictEqual(typeof options.lines, 'number');
     assert.strictEqual(typeof options.format, 'string');
@@ -243,15 +261,15 @@ async function getLogs(unit, options) {
     // need to handle box.log without subdir
     if (unit === 'box') args.push(path.join(paths.LOG_DIR, 'box.log'));
     else if (unit.startsWith('crash-')) args.push(path.join(paths.CRASH_LOG_DIR, unit.slice(6) + '.log'));
-    else throw new BoxError(BoxError.BAD_FIELD, 'No such unit', { field: 'unit' });
+    else return callback(new BoxError(BoxError.BAD_FIELD, 'No such unit', { field: 'unit' }));
 
-    const cp = spawn('/usr/bin/tail', args);
+    var cp = spawn('/usr/bin/tail', args);
 
-    const transformStream = split(function mapper(line) {
+    var transformStream = split(function mapper(line) {
         if (format !== 'json') return line + '\n';
 
-        const data = line.split(' '); // logs are <ISOtimestamp> <msg>
-        let timestamp = (new Date(data[0])).getTime();
+        var data = line.split(' '); // logs are <ISOtimestamp> <msg>
+        var timestamp = (new Date(data[0])).getTime();
         if (isNaN(timestamp)) timestamp = 0;
 
         return JSON.stringify({
@@ -265,98 +283,142 @@ async function getLogs(unit, options) {
 
     cp.stdout.pipe(transformStream);
 
-    return transformStream;
+    return callback(null, transformStream);
 }
 
-async function prepareDashboardDomain(domain, auditSource) {
+function prepareDashboardDomain(domain, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     debug(`prepareDashboardDomain: ${domain}`);
 
-    if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
+    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
 
-    const domainObject = await domains.get(domain);
-    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');
+    domains.get(domain, function (error, domainObject) {
+        if (error) return callback(error);
 
-    const fqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
+        const fqdn = domains.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
-    const result = await apps.list();
-    if (result.some(app => app.fqdn === fqdn)) throw new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app');
+        apps.getAll(function (error, result) {
+            if (error) return callback(error);
 
-    const taskId = await tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_LOCATION, domain, auditSource ]);
+            const conflict = result.filter(app => app.fqdn === fqdn);
+            if (conflict.length) return callback(new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app'));
 
-    tasks.startTask(taskId, {});
+            tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_LOCATION, domain, auditSource ], function (error, taskId) {
+                if (error) return callback(error);
 
-    return taskId;
+                tasks.startTask(taskId, {}, NOOP_CALLBACK);
+
+                callback(null, taskId);
+            });
+        });
+    });
 }
 
 // call this only pre activation since it won't start mail server
-async function setDashboardDomain(domain, auditSource) {
+function setDashboardDomain(domain, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     debug(`setDashboardDomain: ${domain}`);
 
-    const domainObject = await domains.get(domain);
-    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');
+    domains.get(domain, function (error, domainObject) {
+        if (error) return callback(error);
 
-    await reverseProxy.writeDashboardConfig(domain);
-    const fqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
+        reverseProxy.writeDashboardConfig(domain, function (error) {
+            if (error) return callback(error);
 
-    await settings.setDashboardLocation(domain, fqdn);
+            const fqdn = domains.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
-    await safe(appstore.updateCloudron({ domain }));
+            settings.setDashboardLocation(domain, fqdn, function (error) {
+                if (error) return callback(error);
 
-    await eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain, fqdn });
+                appstore.updateCloudron({ domain }, NOOP_CALLBACK);
+
+                eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain, fqdn });
+
+                callback(null);
+            });
+        });
+    });
 }
 
 // call this only post activation because it will restart mail server
-async function updateDashboardDomain(domain, auditSource) {
+function updateDashboardDomain(domain, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     debug(`updateDashboardDomain: ${domain}`);
 
-    if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
+    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
 
-    await setDashboardDomain(domain, auditSource);
+    setDashboardDomain(domain, auditSource, function (error) {
+        if (error) return callback(error);
 
-    safe(services.rebuildService('turn', auditSource), { debug }); // to update the realm variable
+        services.rebuildService('turn', NOOP_CALLBACK); // to update the realm variable
+
+        callback(null);
+    });
 }
 
-async function renewCerts(options, auditSource) {
+function renewCerts(options, auditSource, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const taskId = await tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ]);
-    tasks.startTask(taskId, {});
-    return taskId;
+    tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ], function (error, taskId) {
+        if (error) return callback(error);
+
+        tasks.startTask(taskId, {}, NOOP_CALLBACK);
+
+        callback(null, taskId);
+    });
 }
 
-async function setupDnsAndCert(subdomain, domain, auditSource, progressCallback) {
+function setupDnsAndCert(subdomain, domain, auditSource, progressCallback, callback) {
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
 
-    const domainObject = await domains.get(domain);
-    const dashboardFqdn = dns.fqdn(subdomain, domainObject);
+    domains.get(domain, function (error, domainObject) {
+        if (error) return callback(error);
 
-    const ip = await sysinfo.getServerIp();
+        const dashboardFqdn = domains.fqdn(subdomain, domainObject);
 
-    progressCallback({ message: `Updating DNS of ${dashboardFqdn}` });
-    await dns.upsertDnsRecords(subdomain, domain, 'A', [ ip ]);
-    progressCallback({ message: `Waiting for DNS of ${dashboardFqdn}` });
-    await dns.waitForDnsRecord(subdomain, domain, 'A', ip, { interval: 30000, times: 50000 });
-    progressCallback({ message: `Getting certificate of ${dashboardFqdn}` });
-    await reverseProxy.ensureCertificate(dns.fqdn(subdomain, domainObject), domain, auditSource);
+        sysinfo.getServerIp(function (error, ip) {
+            if (error) return callback(error);
+
+            async.series([
+                (done) => { progressCallback({ message: `Updating DNS of ${dashboardFqdn}` }); done(); },
+                domains.upsertDnsRecords.bind(null, subdomain, domain, 'A', [ ip ]),
+                (done) => { progressCallback({ message: `Waiting for DNS of ${dashboardFqdn}` }); done(); },
+                domains.waitForDnsRecord.bind(null, subdomain, domain, 'A', ip, { interval: 30000, times: 50000 }),
+                (done) => { progressCallback({ message: `Getting certificate of ${dashboardFqdn}` }); done(); },
+                reverseProxy.ensureCertificate.bind(null, domains.fqdn(subdomain, domainObject), domain, auditSource)
+            ], function (error) {
+                if (error) return callback(error);
+
+                callback(null);
+            });
+        });
+    });
 }
 
-async function syncDnsRecords(options) {
+function syncDnsRecords(options, callback) {
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const taskId = await tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ]);
-    tasks.startTask(taskId, {});
-    return taskId;
+    tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ], function (error, taskId) {
+        if (error) return callback(error);
+
+        tasks.startTask(taskId, {}, NOOP_CALLBACK);
+
+        callback(null, taskId);
+    });
 }

src/collectd.js

@@ -8,6 +8,7 @@ exports = module.exports = {
 const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('collectd'),
+    fs = require('fs'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -15,29 +16,40 @@ const assert = require('assert'),
 
 const CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh');
 
-async function addProfile(name, profile) {
+function addProfile(name, profile, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof profile, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     const configFilePath = path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`);
 
     // skip restarting collectd if the profile already exists with the same contents
     const currentProfile = safe.fs.readFileSync(configFilePath, 'utf8') || '';
-    if (currentProfile === profile) return;
+    if (currentProfile === profile) return callback(null);
 
-    if (!safe.fs.writeFileSync(configFilePath, profile)) throw new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${safe.error.message}`);
+    fs.writeFile(configFilePath, profile, function (error) {
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
+
+        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}, function (error) {
+            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
+
+            callback(null);
+        });
+    });
 
-    const [error] = await safe(shell.promises.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}));
-    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config');
 }
 
-async function removeProfile(name) {
+function removeProfile(name, callback) {
     assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (!safe.fs.unlinkSync(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`))) {
-        if (safe.error.code !== 'ENOENT') debug('Error removing collectd profile', safe.error);
-    }
+    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`), function (error) {
+        if (error && error.code !== 'ENOENT') debug('Error removing collectd profile', error);
 
-    const [error] = await safe(shell.promises.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}));
-    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config');
+        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}, function (error) {
+            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
+
+            callback(null);
+        });
+    });
 }

src/constants.js

@@ -49,11 +49,6 @@ exports = module.exports = {
 
     AUTOUPDATE_PATTERN_NEVER: 'never',
 
-    // the db field is a blob so we make this explicit
-    AVATAR_NONE: Buffer.from('', 'utf8'),
-    AVATAR_GRAVATAR: Buffer.from('gravatar', 'utf8'),
-    AVATAR_CUSTOM: Buffer.from('custom', 'utf8'),   // this is not used here just for reference. The field will contain a byte buffer instead of the type string
-
     SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8), // also used in dashboard client.js
 
     CLOUDRON: CLOUDRON,
@@ -63,6 +58,6 @@ exports = module.exports = {
 
     FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
 
-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '6.3.0-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '6.0.1-test'
 };
 

src/crashnotifier.js

@@ -5,8 +5,7 @@ exports = module.exports = {
 };
 
 const assert = require('assert'),
-    AuditSource = require('./auditsource.js'),
-    child_process = require('child_process'),
+    auditSource = require('./auditsource.js'),
     eventlog = require('./eventlog.js'),
     safe = require('safetydance'),
     path = require('path'),
@@ -18,36 +17,43 @@ const COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
 const CRASH_LOG_TIMESTAMP_OFFSET = 1000 * 60 * 60; // 60 min
 const CRASH_LOG_TIMESTAMP_FILE = '/tmp/crashlog.timestamp';
 
-async function collectLogs(unitName) {
+function collectLogs(unitName, callback) {
     assert.strictEqual(typeof unitName, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const logs = child_process.execSync(`sudo ${COLLECT_LOGS_CMD} ${unitName}`, { encoding: 'utf8' });
-    return logs;
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
+    if (!logs) return callback(safe.error);
+
+    callback(null, logs);
 }
 
-async function sendFailureLogs(unitName) {
+function sendFailureLogs(unitName, callback) {
     assert.strictEqual(typeof unitName, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     // check if we already sent a mail in the last CRASH_LOG_TIME_OFFSET window
     const timestamp = safe.fs.readFileSync(CRASH_LOG_TIMESTAMP_FILE, 'utf8');
     if (timestamp && (parseInt(timestamp) + CRASH_LOG_TIMESTAMP_OFFSET) > Date.now()) {
         console.log('Crash log already sent within window');
-        return;
+        return callback();
     }
 
-    let [error, logs] = await safe(collectLogs(unitName));
-    if (error) {
-        console.error('Failed to collect logs.', error);
-        logs = util.format('Failed to collect logs.', error);
-    }
+    collectLogs(unitName, async function (error, logs) {
+        if (error) {
+            console.error('Failed to collect logs.', error);
+            logs = util.format('Failed to collect logs.', error);
+        }
 
-    const crashId = `${new Date().toISOString()}`;
-    console.log(`Creating crash log for ${unitName} with id ${crashId}`);
+        const crashId = `${new Date().toISOString()}`;
+        console.log(`Creating crash log for ${unitName} with id ${crashId}`);
 
-    if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, `${crashId}.log`), logs)) console.log(`Failed to stash logs to ${crashId}.log:`, safe.error);
+        if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, `${crashId}.log`), logs)) console.log(`Failed to stash logs to ${crashId}.log:`, safe.error);
 
-    [error] = await safe(eventlog.add(eventlog.ACTION_PROCESS_CRASH, AuditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }));
-    if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);
+        [error] = await safe(eventlog.add(eventlog.ACTION_PROCESS_CRASH, auditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }));
+        if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);
 
-    safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
+        safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
+
+        callback();
+    });
 }

src/cron.js

@@ -19,7 +19,8 @@ exports = module.exports = {
 const appHealthMonitor = require('./apphealthmonitor.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
-    AuditSource = require('./auditsource.js'),
+    async = require('async'),
+    auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     cloudron = require('./cloudron.js'),
     constants = require('./constants.js'),
@@ -28,7 +29,6 @@ const appHealthMonitor = require('./apphealthmonitor.js'),
     dyndns = require('./dyndns.js'),
     eventlog = require('./eventlog.js'),
     janitor = require('./janitor.js'),
-    safe = require('safetydance'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
     system = require('./system.js'),
@@ -52,6 +52,8 @@ const gJobs = {
     appHealthMonitor: null
 };
 
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
 // cron format
 // Seconds: 0-59
 // Minutes: 0-59
@@ -60,81 +62,87 @@ const gJobs = {
 // Months: 0-11
 // Day of Week: 0-6
 
-async function startJobs() {
+function startJobs(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     debug('startJobs: starting cron jobs');
 
     const randomTick = Math.floor(60*Math.random());
     gJobs.systemChecks = new CronJob({
         cronTime: '00 30 2 * * *', // once a day. if you change this interval, change the notification messages with correct duration
-        onTick: async () => await safe(cloudron.runSystemChecks(), { debug }),
+        onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
         start: true
     });
 
     gJobs.diskSpaceChecker = new CronJob({
         cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: async () => await safe(system.checkDiskSpace(), { debug }),
+        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
         start: true
     });
 
     // this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
     gJobs.updateCheckerJob = new CronJob({
         cronTime: `${randomTick} ${randomTick} 1,5,9,13,17,21,23 * * *`,
-        onTick: async () => await safe(updateChecker.checkForUpdates({ automatic: true }), { debug }),
+        onTick: () => updateChecker.checkForUpdates({ automatic: true }, NOOP_CALLBACK),
         start: true
     });
 
     gJobs.cleanupTokens = new CronJob({
         cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: async () => await safe(janitor.cleanupTokens(), { debug }),
+        onTick: janitor.cleanupTokens,
         start: true
     });
 
     gJobs.cleanupBackups = new CronJob({
         cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN,
-        onTick: async () => await safe(backups.startCleanupTask(AuditSource.CRON), { debug }),
+        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
         start: true
     });
 
     gJobs.cleanupEventlog = new CronJob({
         cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 60 * 60 * 24 * 10 * 1000) }), { debug }), // 10 days ago
+        onTick: eventlog.cleanup.bind(null, { creationTime: new Date(Date.now() - 60 * 60 * 24 * 10 * 1000) }), // 10 days ago
         start: true
     });
 
     gJobs.dockerVolumeCleaner = new CronJob({
         cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug }),
+        onTick: janitor.cleanupDockerVolumes,
         start: true
     });
 
     gJobs.schedulerSync = new CronJob({
         cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
-        onTick: async () => await safe(scheduler.sync(), { debug }),
+        onTick: scheduler.sync,
         start: true
     });
 
     gJobs.certificateRenew = new CronJob({
         cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: async () => await safe(cloudron.renewCerts({}, AuditSource.CRON), { debug }),
+        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
         start: true
     });
 
     gJobs.appHealthMonitor = new CronJob({
         cronTime: '*/10 * * * * *', // every 10 seconds
-        onTick: async () => await safe(appHealthMonitor.run(10), { debug }), // 10 is the max run time
+        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
         start: true
     });
 
-    const allSettings = await settings.list();
+    settings.getAll(function (error, allSettings) {
+        if (error) return callback(error);
 
-    const tz = allSettings[settings.TIME_ZONE_KEY];
-    backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
-    autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz);
-    dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
+        const tz = allSettings[settings.TIME_ZONE_KEY];
+        backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
+        autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz);
+        dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
+
+        callback();
+    });
 }
 
 // eslint-disable-next-line no-unused-vars
-async function handleSettingsChanged(key, value) {
+function handleSettingsChanged(key, value) {
     assert.strictEqual(typeof key, 'string');
     // value is a variant
 
@@ -144,8 +152,10 @@ async function handleSettingsChanged(key, value) {
     case settings.AUTOUPDATE_PATTERN_KEY:
     case settings.DYNAMIC_DNS_KEY:
         debug('handleSettingsChanged: recreating all jobs');
-        await stopJobs();
-        await startJobs();
+        async.series([
+            stopJobs,
+            startJobs
+        ], NOOP_CALLBACK);
         break;
     default:
         break;
@@ -162,7 +172,7 @@ function backupConfigChanged(value, tz) {
 
     gJobs.backup = new CronJob({
         cronTime: value.schedulePattern,
-        onTick: async () => await safe(backups.startBackupTask(AuditSource.CRON), { debug }),
+        onTick: backups.startBackupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
         start: true,
         timeZone: tz
     });
@@ -180,21 +190,19 @@ function autoupdatePatternChanged(pattern, tz) {
 
     gJobs.autoUpdater = new CronJob({
         cronTime: pattern,
-        onTick: async function() {
+        onTick: function() {
             const updateInfo = updateChecker.getUpdateInfo();
             // do box before app updates. for the off chance that the box logic fixes some app update logic issue
             if (updateInfo.box && !updateInfo.box.unstable) {
                 debug('Starting box autoupdate to %j', updateInfo.box);
-                const [error] = await safe(updater.updateToLatest({ skipBackup: false }, AuditSource.CRON));
-                if (error) debug(`Failed to box autoupdate: ${error.message}`);
+                updater.updateToLatest({ skipBackup: false }, auditSource.CRON, NOOP_CALLBACK);
                 return;
             }
 
             const appUpdateInfo = _.omit(updateInfo, 'box');
             if (Object.keys(appUpdateInfo).length > 0) {
                 debug('Starting app update to %j', appUpdateInfo);
-                const [error] = await safe(apps.autoupdateApps(appUpdateInfo, AuditSource.CRON));
-                if (error) debug(`Failed to app autoupdate: ${error.message}`);
+                apps.autoupdateApps(appUpdateInfo, auditSource.CRON, NOOP_CALLBACK);
             } else {
                 debug('No app auto updates available');
             }
@@ -213,7 +221,7 @@ function dynamicDnsChanged(enabled) {
     if (enabled) {
         gJobs.dynamicDns = new CronJob({
             cronTime: '5 * * * * *',    // we only update the records if the ip has changed.
-            onTick: async () => await safe(dyndns.sync(AuditSource.CRON), { debug }),
+            onTick: dyndns.sync.bind(null, auditSource.CRON, NOOP_CALLBACK),
             start: true
         });
     } else {
@@ -222,10 +230,14 @@ function dynamicDnsChanged(enabled) {
     }
 }
 
-async function stopJobs() {
-    for (const job in gJobs) {
+function stopJobs(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    for (var job in gJobs) {
         if (!gJobs[job]) continue;
         gJobs[job].stop();
         gJobs[job] = null;
     }
+
+    callback();
 }

src/database.js

@@ -19,11 +19,10 @@ const assert = require('assert'),
     constants = require('./constants.js'),
     debug = require('debug')('box:database'),
     mysql = require('mysql'),
-    safe = require('safetydance'),
-    shell = require('./shell.js'),
+    once = require('once'),
     util = require('util');
 
-let gConnectionPool = null;
+var gConnectionPool = null;
 
 const gDatabase = {
     hostname: '127.0.0.1',
@@ -33,8 +32,10 @@ const gDatabase = {
     name: 'box'
 };
 
-async function initialize() {
-    if (gConnectionPool !== null) return;
+function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gConnectionPool !== null) return callback(null);
 
     if (constants.TEST) {
         // see setupTest script how the mysql-server is run
@@ -65,49 +66,57 @@ async function initialize() {
         connection.query('USE ' + gDatabase.name);
         connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
     });
+
+    callback(null);
 }
 
-async function uninitialize() {
-    if (!gConnectionPool) return;
+function uninitialize(callback) {
+    if (!gConnectionPool) return callback(null);
 
-    gConnectionPool.end();
+    gConnectionPool.end(callback);
     gConnectionPool = null;
 }
 
-async function clear() {
-    const cmd = util.format('mysql --host="%s" --user="%s" --password="%s" -Nse "SHOW TABLES" %s | grep -v "^migrations$" | while read table; do mysql --host="%s" --user="%s" --password="%s" -e "SET FOREIGN_KEY_CHECKS = 0; TRUNCATE TABLE $table" %s; done',
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var cmd = util.format('mysql --host="%s" --user="%s" --password="%s" -Nse "SHOW TABLES" %s | grep -v "^migrations$" | while read table; do mysql --host="%s" --user="%s" --password="%s" -e "SET FOREIGN_KEY_CHECKS = 0; TRUNCATE TABLE $table" %s; done',
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name,
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name);
 
-    await shell.promises.exec('clear_database', cmd);
+    child_process.exec(cmd, callback);
 }
 
-async function query() {
+function query() {
     assert.notStrictEqual(gConnectionPool, null);
 
     return new Promise((resolve, reject) => {
         let args = Array.prototype.slice.call(arguments);
+        const callback = typeof args[args.length - 1] === 'function' ? args.pop() : null;
 
         args.push(function queryCallback(error, result) {
-            if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+            if (error) return callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
 
-            resolve(result);
+            callback ? callback(null, result) : resolve(result);
         });
 
         gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
     });
 }
 
-async function transaction(queries) {
+function transaction(queries) {
     assert(Array.isArray(queries));
 
+    const args = Array.prototype.slice.call(arguments);
+    const callback = typeof args[args.length - 1] === 'function' ? once(args.pop()) : null;
+
     return new Promise((resolve, reject) => {
         gConnectionPool.getConnection(function (error, connection) {
-            if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+            if (error) return callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
 
             const releaseConnection = (error) => {
                 connection.release();
-                reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+                callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
             };
 
             connection.beginTransaction(function (error) {
@@ -123,7 +132,7 @@ async function transaction(queries) {
 
                         connection.release();
 
-                        resolve(results);
+                        callback ? callback(null, results) : resolve(results);
                     });
                 });
             });
@@ -131,25 +140,27 @@ async function transaction(queries) {
     });
 }
 
-async function importFromFile(file) {
+function importFromFile(file, callback) {
     assert.strictEqual(typeof file, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const cmd = `/usr/bin/mysql -h "${gDatabase.hostname}" -u ${gDatabase.username} -p${gDatabase.password} ${gDatabase.name} < ${file}`;
+    var cmd = `/usr/bin/mysql -h "${gDatabase.hostname}" -u ${gDatabase.username} -p${gDatabase.password} ${gDatabase.name} < ${file}`;
 
-    await query('CREATE DATABASE IF NOT EXISTS box');
-    const [error] = await safe(shell.promises.exec('importFromFile', cmd));
-    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
+    async.series([
+        query.bind(null, 'CREATE DATABASE IF NOT EXISTS box'),
+        child_process.exec.bind(null, cmd)
+    ], callback);
 }
 
-async function exportToFile(file) {
+function exportToFile(file, callback) {
     assert.strictEqual(typeof file, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
     // this option must not be set in production cloudrons which still use the old mysqldump
-    const colStats = (!constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
+    const disableColStats = (constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
 
-    const cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${colStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
+    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${disableColStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
 
-    const [error] = await safe(shell.promises.exec('exportToFile', cmd));
-    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
+    child_process.exec(cmd, callback);
 }

src/datalayout.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const assert = require('assert'),
+let assert = require('assert'),
     path = require('path');
 
 class DataLayout {

src/dns.js

@@ -1,335 +0,0 @@
-'use strict';
-
-module.exports = exports = {
-    fqdn,
-    getName,
-
-    getDnsRecords,
-    upsertDnsRecords,
-    removeDnsRecords,
-
-    waitForDnsRecord,
-
-    validateHostname,
-
-    makeWildcard,
-
-    registerLocations,
-    unregisterLocations,
-
-    checkDnsRecords,
-    syncDnsRecords,
-
-    resolve,
-
-    promises: {
-        resolve: require('util').promisify(resolve)
-    }
-};
-
-const apps = require('./apps.js'),
-    assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:dns'),
-    dns = require('dns'),
-    domains = require('./domains.js'),
-    mail = require('./mail.js'),
-    promiseRetry = require('./promise-retry.js'),
-    safe = require('safetydance'),
-    settings = require('./settings.js'),
-    sysinfo = require('./sysinfo.js'),
-    tld = require('tldjs'),
-    util = require('util'),
-    _ = require('underscore');
-
-// choose which subdomain backend we use for test purpose we use route53
-function api(provider) {
-    assert.strictEqual(typeof provider, 'string');
-
-    switch (provider) {
-    case 'cloudflare': return require('./dns/cloudflare.js');
-    case 'route53': return require('./dns/route53.js');
-    case 'gcdns': return require('./dns/gcdns.js');
-    case 'digitalocean': return require('./dns/digitalocean.js');
-    case 'gandi': return require('./dns/gandi.js');
-    case 'godaddy': return require('./dns/godaddy.js');
-    case 'linode': return require('./dns/linode.js');
-    case 'vultr': return require('./dns/vultr.js');
-    case 'namecom': return require('./dns/namecom.js');
-    case 'namecheap': return require('./dns/namecheap.js');
-    case 'netcup': return require('./dns/netcup.js');
-    case 'noop': return require('./dns/noop.js');
-    case 'manual': return require('./dns/manual.js');
-    case 'wildcard': return require('./dns/wildcard.js');
-    default: return null;
-    }
-}
-
-function fqdn(location, domainObject) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domainObject, 'object');
-
-    return location + (location ? '.' : '') + domainObject.domain;
-}
-
-// Hostname validation comes from RFC 1123 (section 2.1)
-// Domain name validation comes from RFC 2181 (Name syntax)
-// https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
-// We are validating the validity of the location-fqdn as host name (and not dns name)
-function validateHostname(location, domainObject) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domainObject, 'object');
-
-    const hostname = fqdn(location, domainObject);
-
-    const RESERVED_LOCATIONS = [
-        constants.SMTP_LOCATION,
-        constants.IMAP_LOCATION
-    ];
-    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
-
-    if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
-
-    // workaround https://github.com/oncletom/tld.js/issues/73
-    var tmp = hostname.replace('_', '-');
-    if (!tld.isValid(tmp)) return new BoxError(BoxError.BAD_FIELD, 'Hostname is not a valid domain name', { field: 'location' });
-
-    if (hostname.length > 253) return new BoxError(BoxError.BAD_FIELD, 'Hostname length exceeds 253 characters', { field: 'location' });
-
-    if (location) {
-        // label validation
-        if (location.split('.').some(function (p) { return p.length > 63 || p.length < 1; })) return new BoxError(BoxError.BAD_FIELD, 'Invalid subdomain length', { field: 'location' });
-        if (location.match(/^[A-Za-z0-9-.]+$/) === null) return new BoxError(BoxError.BAD_FIELD, 'Subdomain can only contain alphanumeric, hyphen and dot', { field: 'location' });
-        if (/^[-.]/.test(location)) return new BoxError(BoxError.BAD_FIELD, 'Subdomain cannot start or end with hyphen or dot', { field: 'location' });
-    }
-
-    return null;
-}
-
-// returns the 'name' that needs to be inserted into zone
-// eslint-disable-next-line no-unused-vars
-function getName(domain, location, type) {
-    const part = domain.domain.slice(0, -domain.zoneName.length - 1);
-
-    if (location === '') return part;
-
-    return part ? `${location}.${part}` : location;
-}
-
-function maybePromisify(func) {
-    if (util.types.isAsyncFunction(func)) return func;
-    return util.promisify(func);
-}
-
-async function getDnsRecords(location, domain, type) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof type, 'string');
-
-    const domainObject = await domains.get(domain);
-    return await maybePromisify(api(domainObject.provider).get)(domainObject, location, type);
-}
-
-async function checkDnsRecords(location, domain) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-
-    const values = await getDnsRecords(location, domain, 'A');
-
-    const ip = await sysinfo.getServerIp();
-
-    if (values.length === 0) return { needsOverwrite: false }; // does not exist
-    if (values[0] === ip) return { needsOverwrite: false }; // exists but in sync
-
-    return { needsOverwrite: true };
-}
-
-// note: for TXT records the values must be quoted
-async function upsertDnsRecords(location, domain, type, values) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert(Array.isArray(values));
-
-    debug(`upsertDNSRecord: location ${location} on domain ${domain} of type ${type} with values ${JSON.stringify(values)}`);
-
-    const domainObject = await domains.get(domain);
-    await maybePromisify(api(domainObject.provider).upsert)(domainObject, location, type, values);
-}
-
-async function removeDnsRecords(location, domain, type, values) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert(Array.isArray(values));
-
-    debug('removeDNSRecord: %s on %s type %s values', location, domain, type, values);
-
-    const domainObject = await domains.get(domain);
-    const [error] = await safe(maybePromisify(api(domainObject.provider).del)(domainObject, location, type, values));
-    if (error && error.reason !== BoxError.NOT_FOUND) throw error;
-}
-
-async function waitForDnsRecord(location, domain, type, value, options) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert(type === 'A' || type === 'TXT');
-    assert.strictEqual(typeof value, 'string');
-    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-
-    const domainObject = await domains.get(domain);
-
-    // linode DNS takes ~15mins
-    if (!options.interval) options.interval = domainObject.provider === 'linode' ? 20000 : 5000;
-
-    await maybePromisify(api(domainObject.provider).wait)(domainObject, location, type, value, options);
-}
-
-function makeWildcard(vhost) {
-    assert.strictEqual(typeof vhost, 'string');
-
-    // if the vhost is like *.example.com, this function will do nothing
-    let parts = vhost.split('.');
-    parts[0] = '*';
-    return parts.join('.');
-}
-
-async function registerLocations(locations, options, progressCallback) {
-    assert(Array.isArray(locations));
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
-
-    const overwriteDns = options.overwriteDns || false;
-
-    const ip = await sysinfo.getServerIp();
-
-    for (const location of locations) {
-        const error = await promiseRetry({ times: 200, interval: 5000 }, async function () {
-            progressCallback({ message: `Registering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
-
-            // get the current record before updating it
-            const [error, values] = await safe(getDnsRecords(location.subdomain, location.domain, 'A'));
-            if (error && error.reason === BoxError.EXTERNAL_ERROR) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }); // try again
-            // give up for other errors
-            if (error && error.reason === BoxError.ACCESS_DENIED) return new BoxError(BoxError.ACCESS_DENIED, error.message, { domain: location });
-            if (error && error.reason === BoxError.NOT_FOUND) return new BoxError(BoxError.NOT_FOUND, error.message, { domain: location });
-            if (error) return new BoxError(BoxError.EXTERNAL_ERROR, error.message, location);
-
-            if (values.length !== 0 && values[0] === ip) return null; // up-to-date
-
-            // refuse to update any existing DNS record for custom domains that we did not create
-            if (values.length !== 0 && !overwriteDns) return new BoxError(BoxError.ALREADY_EXISTS, 'DNS Record already exists', { domain: location });
-
-            const [upsertError] = await safe(upsertDnsRecords(location.subdomain, location.domain, 'A', [ ip ]));
-            if (upsertError && (upsertError.reason === BoxError.BUSY || upsertError.reason === BoxError.EXTERNAL_ERROR)) {
-                progressCallback({ message: `registerSubdomains: Upsert error. Will retry. ${upsertError.message}` });
-                throw new BoxError(BoxError.EXTERNAL_ERROR, upsertError.message, { domain: location }); // try again
-            }
-
-            return upsertError ? new BoxError(BoxError.EXTERNAL_ERROR, upsertError.message, location) : null;
-        });
-
-        if (error) throw error;
-    }
-}
-
-async function unregisterLocations(locations, progressCallback) {
-    assert(Array.isArray(locations));
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const ip = await sysinfo.getServerIp();
-
-    for (const location of locations) {
-        const error = await promiseRetry({ times: 30, interval: 5000 }, async function () {
-            progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
-
-            const [error] = await safe(removeDnsRecords(location.subdomain, location.domain, 'A', [ ip ]));
-            if (error && error.reason === BoxError.NOT_FOUND) return;
-            if (error && (error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                progressCallback({ message: `Error unregistering location. Will retry. ${error.message}`});
-                throw new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }); // try again
-            }
-
-            return error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }) : null; // give up for other errors
-        });
-
-        if (error) throw error;
-    }
-}
-
-async function syncDnsRecords(options, progressCallback) {
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    if (options.domain && options.type === 'mail') return await mail.setDnsRecords(options.domain);
-
-    let allDomains = await domains.list();
-
-    if (options.domain) allDomains = allDomains.filter(d => d.domain === options.domain);
-
-    const mailSubdomain = settings.mailFqdn().substr(0, settings.mailFqdn().length - settings.mailDomain().length - 1);
-
-    const allApps = await apps.list();
-
-    let progress = 1, errors = [];
-
-    // we sync by domain only to get some nice progress
-    for (const domain of allDomains) {
-        progressCallback({ percent: progress, message: `Updating DNS of ${domain.domain}`});
-        progress += Math.round(100/(1+allDomains.length));
-
-        let locations = [];
-        if (domain.domain === settings.dashboardDomain()) locations.push({ subdomain: constants.DASHBOARD_LOCATION, domain: settings.dashboardDomain() });
-        if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.dashboardFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
-
-        allApps.forEach(function (app) {
-            const appLocations = [{ subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains);
-            locations = locations.concat(appLocations.filter(al => al.domain === domain.domain));
-        });
-
-        try {
-            await registerLocations(locations, { overwriteDns: true }, progressCallback);
-            progressCallback({ message: `Updating mail DNS of ${domain.domain}`});
-            await mail.setDnsRecords(domain.domain);
-        } catch (error) {
-            errors.push({ domain: domain.domain, message: error.message });
-        }
-    }
-
-    return { errors };
-}
-
-// a note on TXT records. It doesn't have quotes ("") at the DNS level. Those quotes
-// are added for DNS server software to enclose spaces. Such quotes may also be returned
-// by the DNS REST API of some providers
-function resolve(hostname, rrtype, options, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof rrtype, 'string');
-    assert(options && typeof options === 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const defaultOptions = { server: '127.0.0.1', timeout: 5000 }; // unbound runs on 127.0.0.1
-    const resolver = new dns.Resolver();
-    options = _.extend({ }, defaultOptions, options);
-
-    // Only use unbound on a Cloudron
-    if (constants.CLOUDRON) resolver.setServers([ options.server ]);
-
-    // should callback with ECANCELLED but looks like we might hit https://github.com/nodejs/node/issues/14814
-    const timerId = setTimeout(resolver.cancel.bind(resolver), options.timeout || 5000);
-
-    resolver.resolve(hostname, rrtype, function (error, result) {
-        clearTimeout(timerId);
-
-        if (error && error.code === 'ECANCELLED') error.code = 'TIMEOUT';
-
-        // result is an empty array if there was no error but there is no record. when you query a random
-        // domain, it errors with ENOTFOUND. But if you query an existing domain (A record) but with different
-        // type (CNAME) it is not an error and empty array
-        // for TXT records, result is 2d array of strings
-        callback(error, result);
-    });
-}

src/dns/cloudflare.js

@@ -1,21 +1,22 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     async = require('async'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/cloudflare'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js'),
@@ -114,7 +115,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     debug('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
@@ -185,7 +186,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     getZoneByName(dnsConfig, zoneName, function(error, zone) {
         if (error) return callback(error);
@@ -210,7 +211,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     getZoneByName(dnsConfig, zoneName, function(error, zone) {
         if (error) return callback(error);
@@ -255,7 +256,7 @@ function wait(domainObject, location, type, value, options, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     debug('wait: %s for zone %s of type %s', fqdn, zoneName, type);
 

src/dns/digitalocean.js

@@ -15,7 +15,8 @@ const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/digitalocean'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
@@ -86,7 +87,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
@@ -166,7 +167,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     getInternal(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -189,7 +190,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     getInternal(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -229,7 +230,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/gandi.js

@@ -1,25 +1,26 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/gandi'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
 
-const GANDI_API = 'https://dns.api.gandi.net/api/v5';
+var GANDI_API = 'https://dns.api.gandi.net/api/v5';
 
 function formatError(response) {
     return util.format(`Gandi DNS error [${response.statusCode}] ${response.body.message}`);
@@ -43,7 +44,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -74,7 +75,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type}`);
 
@@ -102,7 +103,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -129,7 +130,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/gcdns.js

@@ -1,21 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/gcdns'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     GCDNS = require('@google-cloud/dns').DNS,
+    util = require('util'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
@@ -76,7 +78,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
@@ -118,7 +120,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
         if (error) return callback(error);
@@ -147,7 +149,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
         if (error) return callback(error);
@@ -181,7 +183,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/godaddy.js

@@ -1,20 +1,21 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/godaddy'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -49,7 +50,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -90,7 +91,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type}`);
 
@@ -122,7 +123,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -164,7 +165,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/interface.js

@@ -7,17 +7,18 @@
 // -------------------------------------------
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
-    BoxError = require('../boxerror.js');
+var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    util = require('util');
 
 function removePrivateFields(domainObject) {
     // in-place removal of tokens and api keys with constants.SECRET_PLACEHOLDER

src/dns/linode.js

@@ -10,12 +10,13 @@ exports = module.exports = {
     verifyDnsConfig
 };
 
-const async = require('async'),
+let async = require('async'),
     assert = require('assert'),
     constants = require('../constants.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/linode'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -116,7 +117,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '';
+        name = domains.getName(domainObject, location, type) || '';
 
     getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -139,7 +140,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '';
+        name = domains.getName(domainObject, location, type) || '';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
@@ -221,7 +222,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '';
+        name = domains.getName(domainObject, location, type) || '';
 
     getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -262,7 +263,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/manual.js

@@ -10,10 +10,12 @@ exports = module.exports = {
     verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/manual'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
+    util = require('util'),
     waitForDns = require('./waitfordns.js');
 
 function removePrivateFields(domainObject) {
@@ -64,7 +66,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/namecheap.js

@@ -1,25 +1,25 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    verifyDnsConfig,
-    wait
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    verifyDnsConfig: verifyDnsConfig,
+    wait: wait
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecheap'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     querystring = require('querystring'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     sysinfo = require('../sysinfo.js'),
-    util = require('util'),
     waitForDns = require('./waitfordns.js'),
     xml2js = require('xml2js');
 
@@ -34,17 +34,20 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-async function getQuery(dnsConfig) {
+function getQuery(dnsConfig, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const ip = await sysinfo.getServerIp();
+    sysinfo.getServerIp(function (error, ip) {
+        if (error) return callback(error);
 
-    return {
-        ApiUser: dnsConfig.username,
-        ApiKey: dnsConfig.token,
-        UserName: dnsConfig.username,
-        ClientIp: ip
-    };
+        callback(null, {
+            ApiUser: dnsConfig.username,
+            ApiKey: dnsConfig.token,
+            UserName: dnsConfig.username,
+            ClientIp: ip
+        });
+    });
 }
 
 function getZone(dnsConfig, zoneName, callback) {
@@ -52,7 +55,7 @@ function getZone(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    util.callbackify(getQuery)(dnsConfig, function (error, query) {
+    getQuery(dnsConfig, function (error, query) {
         if (error) return callback(error);
 
         query.Command = 'namecheap.domains.dns.getHosts';
@@ -90,7 +93,7 @@ function setZone(dnsConfig, zoneName, hosts, callback) {
     assert(Array.isArray(hosts));
     assert.strictEqual(typeof callback, 'function');
 
-    util.callbackify(getQuery)(dnsConfig, function (error, query) {
+    getQuery(dnsConfig, function (error, query) {
         if (error) return callback(error);
 
         query.Command = 'namecheap.domains.dns.setHosts';
@@ -148,7 +151,7 @@ function upsert(domainObject, subdomain, type, values, callback) {
     const dnsConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
-    subdomain = dns.getName(domainObject, subdomain, type) || '@';
+    subdomain = domains.getName(domainObject, subdomain, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
@@ -211,7 +214,7 @@ function get(domainObject, subdomain, type, callback) {
     const dnsConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
-    subdomain = dns.getName(domainObject, subdomain, type) || '@';
+    subdomain = domains.getName(domainObject, subdomain, type) || '@';
 
     getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
@@ -238,7 +241,7 @@ function del(domainObject, subdomain, type, values, callback) {
     const dnsConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
-    subdomain = dns.getName(domainObject, subdomain, type) || '@';
+    subdomain = domains.getName(domainObject, subdomain, type) || '@';
 
     debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
@@ -313,7 +316,7 @@ function wait(domainObject, subdomain, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(subdomain, domainObject);
+    const fqdn = domains.fqdn(subdomain, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/namecom.js

@@ -1,22 +1,24 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecom'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
+    util = require('util'),
     waitForDns = require('./waitfordns.js');
 
 const NAMECOM_API = 'https://api.name.com/v4';
@@ -160,7 +162,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '';
+        name = domains.getName(domainObject, location, type) || '';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -181,7 +183,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '';
+        name = domains.getName(domainObject, location, type) || '';
 
     getInternal(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -203,7 +205,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '';
+        name = domains.getName(domainObject, location, type) || '';
 
     debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -233,7 +235,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/netcup.js

@@ -1,25 +1,26 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/netcup'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
 
-const API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON';
+var API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON';
 
 function formatError(response) {
     if (response.body) return util.format('Netcup DNS error [%s] %s', response.body.statuscode, response.body.longmessage);
@@ -94,7 +95,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
@@ -162,7 +163,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug('get: %s for zone %s of type %s', name, zoneName, type);
 
@@ -187,7 +188,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '@';
+        name = domains.getName(domainObject, location, type) || '@';
 
     debug('del: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
@@ -248,7 +249,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/noop.js

@@ -1,17 +1,18 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
-    debug = require('debug')('box:dns/noop');
+var assert = require('assert'),
+    debug = require('debug')('box:dns/noop'),
+    util = require('util');
 
 function removePrivateFields(domainObject) {
     return domainObject;

src/dns/route53.js

@@ -1,21 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     AWS = require('aws-sdk'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/route53'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
+    util = require('util'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
@@ -100,7 +102,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
@@ -145,7 +147,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
@@ -181,7 +183,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = domains.fqdn(location, domainObject);
 
     getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
@@ -239,7 +241,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/vultr.js

@@ -15,7 +15,8 @@ const async = require('async'),
     constants = require('../constants.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/vultr'),
-    dns = require('../dns.js'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
@@ -70,7 +71,7 @@ function getZoneRecords(dnsConfig, zoneName, name, type, callback) {
                 iteratorDone();
             });
     }, function (testDone) { return testDone(null, !!cursor); }, function (error) {
-        debug('getZoneRecords: error:', error, JSON.stringify(records));
+        debug('getZoneRecords:', error, JSON.stringify(records));
 
         if (error) return callback(error);
 
@@ -86,7 +87,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '';
+        name = domains.getName(domainObject, location, type) || '';
 
     getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
         if (error) return callback(error);
@@ -108,7 +109,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '';
+        name = domains.getName(domainObject, location, type) || '';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
@@ -189,7 +190,7 @@ function del(domainObject, location, type, values, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = dns.getName(domainObject, location, type) || '';
+        name = domains.getName(domainObject, location, type) || '';
 
     getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
         if (error) return callback(error);
@@ -229,7 +230,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/waitfordns.js

@@ -2,11 +2,11 @@
 
 exports = module.exports = waitForDns;
 
-const assert = require('assert'),
+var assert = require('assert'),
     async = require('async'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/waitfordns'),
-    dns = require('../dns.js');
+    dns = require('../native-dns.js');
 
 function resolveIp(hostname, options, callback) {
     assert.strictEqual(typeof hostname, 'string');

src/dns/wildcard.js

@@ -1,21 +1,22 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields,
-    injectPrivateFields,
-    upsert,
-    get,
-    del,
-    wait,
-    verifyDnsConfig
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/manual'),
-    dns = require('../dns.js'),
-    safe = require('safetydance'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
     sysinfo = require('../sysinfo.js'),
+    util = require('util'),
     waitForDns = require('./waitfordns.js');
 
 function removePrivateFields(domainObject) {
@@ -65,32 +66,36 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = domains.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }
 
-async function verifyDnsConfig(domainObject) {
+function verifyDnsConfig(domainObject, callback) {
     assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     const zoneName = domainObject.zoneName;
 
     // Very basic check if the nameservers can be fetched
-    const [error, nameservers] = await safe(dns.promises.resolve(zoneName, 'NS', { timeout: 5000 }));
-    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' });
-    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' });
+    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
+        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
+        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
 
-    const location = 'cloudrontestdns';
-    const fqdn = dns.fqdn(location, domainObject);
+        const location = 'cloudrontestdns';
+        const fqdn = domains.fqdn(location, domainObject);
 
-    const [error2, result] = await safe(dns.promises.resolve(fqdn, 'A', { server: '127.0.0.1', timeout: 5000 }));
-    if (error2 && error2.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve ${fqdn}`, { field: 'nameservers' });
-    if (error2 || !result) throw new BoxError(BoxError.BAD_FIELD, error2 ? error2.message : `Unable to resolve ${fqdn}`, { field: 'nameservers' });
+        dns.resolve(fqdn, 'A', { server: '127.0.0.1', timeout: 5000 }, function (error, result) {
+            if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, `Unable to resolve ${fqdn}`, { field: 'nameservers' }));
+            if (error || !result) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : `Unable to resolve ${fqdn}`, { field: 'nameservers' }));
 
-    const [error3, ip] = await safe(sysinfo.getServerIp());
-    if (error3) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to detect IP of this server: ${error3.message}`);
+            sysinfo.getServerIp(function (error, ip) {
+                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to detect IP of this server: ${error.message}`));
 
-    if (result.length !== 1 || ip !== result[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(result)} instead of ${ip}`);
+                if (result.length !== 1 || ip !== result[0]) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(result)} instead of ${ip}`));
 
-    return {};
+                callback(null, {});
+            });
+        });
+    });
 }

src/dockerproxy.js

@@ -5,8 +5,9 @@ exports = module.exports = {
     stop
 };
 
-const apps = require('./apps.js'),
+var apps = require('./apps.js'),
     assert = require('assert'),
+    BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     express = require('express'),
     debug = require('debug')('box:dockerproxy'),
@@ -17,27 +18,27 @@ const apps = require('./apps.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
-    util = require('util'),
     _ = require('underscore');
 
-let gHttpServer = null;
+var gHttpServer = null;
 
-async function authorizeApp(req, res, next) {
+function authorizeApp(req, res, next) {
     // make the tests pass for now
     if (constants.TEST) {
         req.app = { id: 'testappid' };
         return next();
     }
 
-    const [error, app] = await safe(apps.getByIpAddress(req.connection.remoteAddress));
-    if (error) return next(new HttpError(500, error));
-    if (!app) return next(new HttpError(401, 'Unauthorized'));
+    apps.getByIpAddress(req.connection.remoteAddress, function (error, app) {
+        if (error && error.reason === BoxError.NOT_FOUND) return next(new HttpError(401, 'Unauthorized'));
+        if (error) return next(new HttpError(500, error));
 
-    if (!('docker' in app.manifest.addons)) return next(new HttpError(401, 'Unauthorized'));
+        if (!('docker' in app.manifest.addons)) return next(new HttpError(401, 'Unauthorized'));
 
-    req.app = app;
+        req.app = app;
 
-    next();
+        next();
+    });
 }
 
 function attachDockerRequest(req, res, next) {
@@ -107,17 +108,18 @@ function process(req, res, next) {
     }
 }
 
-async function start() {
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
     assert(gHttpServer === null, 'Already started');
 
-    const json = middleware.json({ strict: true });
+    let json = middleware.json({ strict: true });
 
     // we protect container create as the app/admin can otherwise mount random paths (like the ghost file)
     // protected other paths is done by preventing install/exec access of apps using docker addon
-    const router = new express.Router();
+    let router = new express.Router();
     router.post('/:version/containers/create', containersCreate);
 
-    const proxyServer = express();
+    let proxyServer = express();
 
     if (constants.TEST) {
         proxyServer.use(function (req, res, next) {
@@ -134,6 +136,7 @@ async function start() {
         .use(middleware.lastMile());
 
     gHttpServer = http.createServer(proxyServer);
+    gHttpServer.listen(constants.DOCKER_PROXY_PORT, '172.18.0.1', callback);
 
     // Overwrite the default 2min request timeout. This is required for large builds for example
     gHttpServer.setTimeout(60 * 60 * 1000);
@@ -167,12 +170,14 @@ async function start() {
             client.pipe(remote).pipe(client);
         });
     });
-
-    await util.promisify(gHttpServer.listen.bind(gHttpServer))(constants.DOCKER_PROXY_PORT, '172.18.0.1');
 }
 
-async function stop() {
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     if (gHttpServer) gHttpServer.close();
 
     gHttpServer = null;
+
+    callback();
 }

src/domaindb.js

@@ -0,0 +1,141 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    add,
+    get,
+    getAll,
+    update,
+    del,
+    clear
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    safe = require('safetydance');
+
+const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(',');
+
+function postProcess(data) {
+    data.config = safe.JSON.parse(data.configJson);
+    delete data.configJson;
+
+    data.tlsConfig = safe.JSON.parse(data.tlsConfigJson);
+    delete data.tlsConfigJson;
+
+    data.wellKnown = safe.JSON.parse(data.wellKnownJson);
+    delete data.wellKnownJson;
+
+    data.fallbackCertificate = safe.JSON.parse(data.fallbackCertificateJson);
+    delete data.fallbackCertificateJson;
+
+    return data;
+}
+
+function get(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`SELECT ${DOMAINS_FIELDS} FROM domains WHERE domain=?`, [ domain ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function getAll(callback) {
+    database.query(`SELECT ${DOMAINS_FIELDS} FROM domains ORDER BY domain`, function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function add(name, data, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof data.zoneName, 'string');
+    assert.strictEqual(typeof data.provider, 'string');
+    assert.strictEqual(typeof data.config, 'object');
+    assert.strictEqual(typeof data.tlsConfig, 'object');
+    assert.strictEqual(typeof data.fallbackCertificate, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    let queries = [
+        { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson, fallbackCertificateJson) VALUES (?, ?, ?, ?, ?, ?)',
+            args: [ name, data.zoneName, data.provider, JSON.stringify(data.config), JSON.stringify(data.tlsConfig), JSON.stringify(data.fallbackCertificate) ] },
+        { query: 'INSERT INTO mail (domain, dkimSelector) VALUES (?, ?)', args: [ name, data.dkimSelector || 'cloudron' ] },
+    ];
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'Domain already exists'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function update(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var args = [ ], fields = [ ];
+    for (var k in domain) {
+        if (k === 'config' || k === 'tlsConfig' || k === 'wellKnown' || k === 'fallbackCertificate') { // json fields
+            fields.push(`${k}Json = ?`);
+            args.push(JSON.stringify(domain[k]));
+        } else {
+            fields.push(k + ' = ?');
+            args.push(domain[k]);
+        }
+    }
+    args.push(name);
+
+    database.query('UPDATE domains SET ' + fields.join(', ') + ' WHERE domain=?', args, function (error) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function del(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    let queries = [
+        { query: 'DELETE FROM mail WHERE domain = ?', args: [ domain ] },
+        { query: 'DELETE FROM domains WHERE domain = ?', args: [ domain ] },
+    ];
+
+    database.transaction(queries, function (error, results) {
+        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') {
+            if (error.message.indexOf('apps_mailDomain_constraint') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by an app or the mailbox of an app. Check the domains of apps and the Email section of each app.'));
+            if (error.message.indexOf('subdomains') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more app(s).'));
+            if (error.message.indexOf('mail') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more mailboxes. Delete them first in the Email view.'));
+
+            return callback(new BoxError(BoxError.CONFLICT, error.message));
+        }
+
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results[1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM domains', function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(error);
+    });
+}

src/domains.js

@@ -3,45 +3,53 @@
 module.exports = exports = {
     add,
     get,
-    list,
+    getAll,
     update,
     del,
     clear,
 
+    fqdn,
+    getName,
+
+    getDnsRecords,
+    upsertDnsRecords,
+    removeDnsRecords,
+
+    waitForDnsRecord,
+
     removePrivateFields,
     removeRestrictedFields,
+
+    validateHostname,
+
+    makeWildcard,
+
+    parentDomain,
+
+    registerLocations,
+    unregisterLocations,
+
+    checkDnsRecords,
+    syncDnsRecords
 };
 
-const assert = require('assert'),
+const apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
     BoxError = require('./boxerror.js'),
+    constants = require('./constants.js'),
     crypto = require('crypto'),
-    database = require('./database.js'),
+    debug = require('debug')('box:domains'),
+    domaindb = require('./domaindb.js'),
     eventlog = require('./eventlog.js'),
     mail = require('./mail.js'),
     reverseProxy = require('./reverseproxy.js'),
-    safe = require('safetydance'),
     settings = require('./settings.js'),
+    sysinfo = require('./sysinfo.js'),
     tld = require('tldjs'),
-    util = require('util'),
     _ = require('underscore');
 
-const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(',');
-
-function postProcess(data) {
-    data.config = safe.JSON.parse(data.configJson);
-    delete data.configJson;
-
-    data.tlsConfig = safe.JSON.parse(data.tlsConfigJson);
-    delete data.tlsConfigJson;
-
-    data.wellKnown = safe.JSON.parse(data.wellKnownJson);
-    delete data.wellKnownJson;
-
-    data.fallbackCertificate = safe.JSON.parse(data.fallbackCertificateJson);
-    delete data.fallbackCertificateJson;
-
-    return data;
-}
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 // choose which subdomain backend we use for test purpose we use route53
 function api(provider) {
@@ -66,28 +74,68 @@ function api(provider) {
     }
 }
 
-function maybePromisify(func) {
-    if (util.types.isAsyncFunction(func)) return func;
-    return util.promisify(func);
+function parentDomain(domain) {
+    assert.strictEqual(typeof domain, 'string');
+    return domain.replace(/^\S+?\./, ''); // +? means non-greedy
 }
 
-async function verifyDnsConfig(dnsConfig, domain, zoneName, provider) {
+function verifyDnsConfig(dnsConfig, domain, zoneName, provider, callback) {
     assert(dnsConfig && typeof dnsConfig === 'object'); // the dns config to test with
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof provider, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const backend = api(provider);
-    if (!backend) throw new BoxError(BoxError.BAD_FIELD, 'Invalid provider', { field: 'provider' });
+    var backend = api(provider);
+    if (!backend) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid provider', { field: 'provider' }));
 
     const domainObject = { config: dnsConfig, domain: domain, zoneName: zoneName };
-    const [error, result] = await safe(maybePromisify(api(provider).verifyDnsConfig)(domainObject));
-    if (error && error.reason === BoxError.ACCESS_DENIED) return { error: new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`) };
-    if (error && error.reason === BoxError.NOT_FOUND) return { error: new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`) };
-    if (error && error.reason === BoxError.EXTERNAL_ERROR) return { error: new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`) };
-    if (error) return { error };
+    api(provider).verifyDnsConfig(domainObject, function (error, result) {
+        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`));
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`));
+        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`));
+        if (error) return callback(error);
 
-    return { error: null, sanitizedConfig: result };
+        callback(null, result);
+    });
+}
+
+function fqdn(location, domainObject) {
+    return location + (location ? '.' : '') + domainObject.domain;
+}
+
+// Hostname validation comes from RFC 1123 (section 2.1)
+// Domain name validation comes from RFC 2181 (Name syntax)
+// https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
+// We are validating the validity of the location-fqdn as host name (and not dns name)
+function validateHostname(location, domainObject) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domainObject, 'object');
+
+    const hostname = fqdn(location, domainObject);
+
+    const RESERVED_LOCATIONS = [
+        constants.SMTP_LOCATION,
+        constants.IMAP_LOCATION
+    ];
+    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
+
+    if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
+
+    // workaround https://github.com/oncletom/tld.js/issues/73
+    var tmp = hostname.replace('_', '-');
+    if (!tld.isValid(tmp)) return new BoxError(BoxError.BAD_FIELD, 'Hostname is not a valid domain name', { field: 'location' });
+
+    if (hostname.length > 253) return new BoxError(BoxError.BAD_FIELD, 'Hostname length exceeds 253 characters', { field: 'location' });
+
+    if (location) {
+        // label validation
+        if (location.split('.').some(function (p) { return p.length > 63 || p.length < 1; })) return new BoxError(BoxError.BAD_FIELD, 'Invalid subdomain length', { field: 'location' });
+        if (location.match(/^[A-Za-z0-9-.]+$/) === null) return new BoxError(BoxError.BAD_FIELD, 'Subdomain can only contain alphanumeric, hyphen and dot', { field: 'location' });
+        if (/^[-.]/.test(location)) return new BoxError(BoxError.BAD_FIELD, 'Subdomain cannot start or end with hyphen or dot', { field: 'location' });
+    }
+
+    return null;
 }
 
 function validateTlsConfig(tlsConfig, dnsProvider) {
@@ -117,37 +165,37 @@ function validateWellKnown(wellKnown) {
     return null;
 }
 
-async function add(domain, data, auditSource) {
+function add(domain, data, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof data.zoneName, 'string');
     assert.strictEqual(typeof data.provider, 'string');
     assert.strictEqual(typeof data.config, 'object');
     assert.strictEqual(typeof data.fallbackCertificate, 'object');
     assert.strictEqual(typeof data.tlsConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     let { zoneName, provider, config, fallbackCertificate, tlsConfig, dkimSelector } = data;
 
-    if (!tld.isValid(domain)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' });
-    if (domain.endsWith('.')) throw new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' });
+    if (!tld.isValid(domain)) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
+    if (domain.endsWith('.')) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
 
     if (zoneName) {
-        if (!tld.isValid(zoneName)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' });
-        if (zoneName.endsWith('.')) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' });
+        if (!tld.isValid(zoneName)) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' }));
+        if (zoneName.endsWith('.')) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' }));
     } else {
         zoneName = tld.getDomain(domain) || domain;
     }
 
     if (fallbackCertificate) {
         let error = reverseProxy.validateCertificate('test', { domain, config }, fallbackCertificate);
-        if (error) throw error;
+        if (error) return callback(error);
     } else {
-        fallbackCertificate = await reverseProxy.generateFallbackCertificate(domain);
+        fallbackCertificate = reverseProxy.generateFallbackCertificateSync(domain);
+        if (fallbackCertificate.error) return callback(fallbackCertificate.error);
     }
 
     let error = validateTlsConfig(tlsConfig, provider);
-    if (error) throw error;
-
-    const dkimKey = await mail.generateDkimKey();
+    if (error) return callback(error);
 
     if (!dkimSelector) {
         // create a unique suffix. this lets one add this domain can be added in another cloudron instance and not have their dkim selector conflict
@@ -155,41 +203,47 @@ async function add(domain, data, auditSource) {
         dkimSelector = `cloudron-${suffix}`;
     }
 
-    const result = await verifyDnsConfig(config, domain, zoneName, provider);
-    if (result.error) throw result.error;
+    verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
+        if (error) return callback(error);
 
-    let queries = [
-        { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson, fallbackCertificateJson) VALUES (?, ?, ?, ?, ?, ?)',
-            args: [ domain, zoneName, provider, JSON.stringify(result.sanitizedConfig), JSON.stringify(tlsConfig), JSON.stringify(fallbackCertificate) ] },
-        { query: 'INSERT INTO mail (domain, dkimKeyJson, dkimSelector) VALUES (?, ?, ?)', args: [ domain, JSON.stringify(dkimKey), dkimSelector || 'cloudron' ] },
-    ];
+        domaindb.add(domain, { zoneName, provider, config: sanitizedConfig, tlsConfig, dkimSelector, fallbackCertificate }, function (error) {
+            if (error) return callback(error);
 
-    [error] = await safe(database.transaction(queries));
-    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Domain already exists');
-    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
+            reverseProxy.setFallbackCertificate(domain, fallbackCertificate, function (error) {
+                if (error) return callback(error);
 
-    await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
+                eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
 
-    eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
+                mail.onDomainAdded(domain, NOOP_CALLBACK);
 
-    safe(mail.onDomainAdded(domain)); // background
+                callback();
+            });
+        });
+    });
 }
 
-async function get(domain) {
+function get(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const result = await database.query(`SELECT ${DOMAINS_FIELDS} FROM domains WHERE domain=?`, [ domain ]);
-    if (result.length === 0) return null;
-    return postProcess(result[0]);
+    domaindb.get(domain, function (error, result) {
+        if (error) return callback(error);
+
+        return callback(null, result);
+    });
 }
 
-async function list() {
-    const results = await database.query(`SELECT ${DOMAINS_FIELDS} FROM domains ORDER BY domain`);
-    results.forEach(postProcess);
-    return results;
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    domaindb.getAll(function (error, result) {
+        if (error) return callback(error);
+
+        return callback(null, result);
+    });
 }
 
-async function update(domain, data, auditSource) {
+function update(domain, data, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof data.zoneName, 'string');
     assert.strictEqual(typeof data.provider, 'string');
@@ -197,97 +251,196 @@ async function update(domain, data, auditSource) {
     assert.strictEqual(typeof data.fallbackCertificate, 'object');
     assert.strictEqual(typeof data.tlsConfig, 'object');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     let { zoneName, provider, config, fallbackCertificate, tlsConfig, wellKnown } = data;
-    let error;
 
-    if (settings.isDemo() && (domain === settings.dashboardDomain())) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
+    if (settings.isDemo() && (domain === settings.dashboardDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
 
-    const domainObject = await get(domain);
-    if (zoneName) {
-        if (!tld.isValid(zoneName)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' });
-    } else {
-        zoneName = domainObject.zoneName;
-    }
+    domaindb.get(domain, function (error, domainObject) {
+        if (error) return callback(error);
 
-    if (fallbackCertificate) {
-        let error = reverseProxy.validateCertificate('test', domainObject, fallbackCertificate);
-        if (error) throw error;
-    }
-
-    error = validateTlsConfig(tlsConfig, provider);
-    if (error) throw error;
-
-    error = validateWellKnown(wellKnown, provider);
-    if (error) throw error;
-
-    if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config);
-
-    const result = await verifyDnsConfig(config, domain, zoneName, provider);
-    if (result.error) throw result.error;
-
-    const newData = {
-        config: result.sanitizedConfig,
-        zoneName,
-        provider,
-        tlsConfig,
-        wellKnown,
-    };
-
-    if (fallbackCertificate) newData.fallbackCertificate = fallbackCertificate;
-
-    let args = [ ], fields = [ ];
-    for (const k in newData) {
-        if (k === 'config' || k === 'tlsConfig' || k === 'wellKnown' || k === 'fallbackCertificate') { // json fields
-            fields.push(`${k}Json = ?`);
-            args.push(JSON.stringify(newData[k]));
+        if (zoneName) {
+            if (!tld.isValid(zoneName)) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' }));
         } else {
-            fields.push(k + ' = ?');
-            args.push(newData[k]);
+            zoneName = domainObject.zoneName;
         }
-    }
-    args.push(domain);
 
-    [error] = await safe(database.query('UPDATE domains SET ' + fields.join(', ') + ' WHERE domain=?', args));
-    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
-    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
+        if (fallbackCertificate) {
+            let error = reverseProxy.validateCertificate('test', domainObject, fallbackCertificate);
+            if (error) return callback(error);
+        }
 
-    if (!fallbackCertificate) return;
+        error = validateTlsConfig(tlsConfig, provider);
+        if (error) return callback(error);
 
-    await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
+        error = validateWellKnown(wellKnown, provider);
+        if (error) return callback(error);
 
-    eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, zoneName, provider });
+        if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config);
+
+        verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
+            if (error) return callback(error);
+
+            let newData = {
+                config: sanitizedConfig,
+                zoneName,
+                provider,
+                tlsConfig,
+                wellKnown,
+            };
+
+            if (fallbackCertificate) newData.fallbackCertificate = fallbackCertificate;
+
+            domaindb.update(domain, newData, function (error) {
+                if (error) return callback(error);
+
+                if (!fallbackCertificate) return callback();
+
+                reverseProxy.setFallbackCertificate(domain, fallbackCertificate, function (error) {
+                    if (error) return callback(error);
+
+                    eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, zoneName, provider });
+
+                    callback();
+                });
+            });
+        });
+    });
 }
 
-async function del(domain, auditSource) {
+function del(domain, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (domain === settings.dashboardDomain()) throw new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain');
-    if (domain === settings.mailDomain()) throw new BoxError(BoxError.CONFLICT, 'Cannot remove mail domain. Change the mail server location first');
+    if (domain === settings.dashboardDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain'));
+    if (domain === settings.mailDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove mail domain. Change the mail server location first'));
 
-    let queries = [
-        { query: 'DELETE FROM mail WHERE domain = ?', args: [ domain ] },
-        { query: 'DELETE FROM domains WHERE domain = ?', args: [ domain ] },
-    ];
+    domaindb.del(domain, function (error) {
+        if (error) return callback(error);
 
-    const [error, results] = await safe(database.transaction(queries));
-    if (error && error.code === 'ER_ROW_IS_REFERENCED_2') {
-        if (error.message.indexOf('apps_mailDomain_constraint') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by an app or the mailbox of an app. Check the domains of apps and the Email section of each app.');
-        if (error.message.indexOf('subdomains') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more app(s).');
-        if (error.message.indexOf('mail') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more mailboxes. Delete them first in the Email view.');
-        throw new BoxError(BoxError.CONFLICT, error.message);
-    }
-    if (error) throw error;
-    if (results[1].affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
+        eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
 
-    eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
+        mail.onDomainRemoved(domain, NOOP_CALLBACK);
 
-    safe(mail.onDomainRemoved(domain));
+        return callback(null);
+    });
 }
 
-async function clear() {
-    await database.query('DELETE FROM domains');
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    domaindb.clear(function (error) {
+        if (error) return callback(error);
+
+        return callback(null);
+    });
+}
+
+// returns the 'name' that needs to be inserted into zone
+// eslint-disable-next-line no-unused-vars
+function getName(domain, location, type) {
+    const part = domain.domain.slice(0, -domain.zoneName.length - 1);
+
+    if (location === '') return part;
+
+    return part ? `${location}.${part}` : location;
+}
+
+function getDnsRecords(location, domain, type, callback) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(domain, function (error, domainObject) {
+        if (error) return callback(error);
+
+        api(domainObject.provider).get(domainObject, location, type, function (error, values) {
+            if (error) return callback(error);
+
+            callback(null, values);
+        });
+    });
+}
+
+function checkDnsRecords(location, domain, callback) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getDnsRecords(location, domain, 'A', function (error, values) {
+        if (error) return callback(error);
+
+        sysinfo.getServerIp(function (error, ip) {
+            if (error) return callback(error);
+
+            if (values.length === 0) return callback(null, { needsOverwrite: false }); // does not exist
+            if (values[0] === ip) return callback(null, { needsOverwrite: false }); // exists but in sync
+
+            callback(null, { needsOverwrite: true });
+        });
+    });
+}
+
+// note: for TXT records the values must be quoted
+function upsertDnsRecords(location, domain, type, values, callback) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('upsertDNSRecord: %s on %s type %s values', location, domain, type, values);
+
+    get(domain, function (error, domainObject) {
+        if (error) return callback(error);
+
+        api(domainObject.provider).upsert(domainObject, location, type, values, function (error) {
+            if (error) return callback(error);
+
+            callback(null);
+        });
+    });
+}
+
+function removeDnsRecords(location, domain, type, values, callback) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('removeDNSRecord: %s on %s type %s values', location, domain, type, values);
+
+    get(domain, function (error, domainObject) {
+        if (error) return callback(error);
+
+        api(domainObject.provider).del(domainObject, location, type, values, function (error) {
+            if (error && error.reason !== BoxError.NOT_FOUND) return callback(error);
+
+            callback(null);
+        });
+    });
+}
+
+function waitForDnsRecord(location, domain, type, value, options, callback) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert(type === 'A' || type === 'TXT');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+    assert.strictEqual(typeof callback, 'function');
+
+    get(domain, function (error, domainObject) {
+        if (error) return callback(error);
+
+        // linode DNS takes ~15mins
+        if (!options.interval) options.interval = domainObject.provider === 'linode' ? 20000 : 5000;
+
+        api(domainObject.provider).wait(domainObject, location, type, value, options, callback);
+    });
 }
 
 // removes all fields that are strictly private and should never be returned by API calls
@@ -304,3 +457,135 @@ function removeRestrictedFields(domain) {
 
     return result;
 }
+
+function makeWildcard(vhost) {
+    assert.strictEqual(typeof vhost, 'string');
+
+    // if the vhost is like *.example.com, this function will do nothing
+    let parts = vhost.split('.');
+    parts[0] = '*';
+    return parts.join('.');
+}
+
+function registerLocations(locations, options, progressCallback, callback) {
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
+
+    const overwriteDns = options.overwriteDns || false;
+
+    sysinfo.getServerIp(function (error, ip) {
+        if (error) return callback(error);
+
+        async.eachSeries(locations, function (location, iteratorDone) {
+            async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
+                progressCallback({ message: `Registering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+
+                // get the current record before updating it
+                getDnsRecords(location.subdomain, location.domain, 'A', function (error, values) {
+                    if (error && error.reason === BoxError.EXTERNAL_ERROR) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location })); // try again
+                    if (error && error.reason === BoxError.ACCESS_DENIED) return retryCallback(null, new BoxError(BoxError.ACCESS_DENIED, error.message, { domain: location }));
+                    if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, new BoxError(BoxError.NOT_FOUND, error.message, { domain: location }));
+                    if (error) return retryCallback(null, new BoxError(BoxError.EXTERNAL_ERROR, error.message, location)); // give up for other errors
+
+                    if (values.length !== 0 && values[0] === ip) return retryCallback(null); // up-to-date
+
+                    // refuse to update any existing DNS record for custom domains that we did not create
+                    if (values.length !== 0 && !overwriteDns) return retryCallback(null, new BoxError(BoxError.ALREADY_EXISTS, 'DNS Record already exists', { domain: location }));
+
+                    upsertDnsRecords(location.subdomain, location.domain, 'A', [ ip ], function (error) {
+                        if (error && (error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
+                            progressCallback({ message: `registerSubdomains: Upsert error. Will retry. ${error.message}` });
+                            return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location })); // try again
+                        }
+
+                        retryCallback(null, error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, location) : null);
+                    });
+                });
+            }, function (error, result) {
+                if (error || result) return iteratorDone(error || result);
+
+                iteratorDone(null);
+            });
+        }, callback);
+    });
+}
+
+function unregisterLocations(locations, progressCallback, callback) {
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    sysinfo.getServerIp(function (error, ip) {
+        if (error) return callback(error);
+
+        async.eachSeries(locations, function (location, iteratorDone) {
+            async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
+                progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+
+                removeDnsRecords(location.subdomain, location.domain, 'A', [ ip ], function (error) {
+                    if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, null);
+                    if (error && (error.reason === BoxError.SBUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
+                        progressCallback({ message: `Error unregistering location. Will retry. ${error.message}`});
+                        return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location })); // try again
+                    }
+
+                    retryCallback(null, error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }) : null);
+                });
+            }, function (error, result) {
+                if (error || result) return iteratorDone(error || result);
+
+                iteratorDone();
+            });
+        }, callback);
+    });
+}
+
+function syncDnsRecords(options, progressCallback, callback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (options.domain && options.type === 'mail') return mail.setDnsRecords(options.domain, callback);
+
+    getAll(function (error, domains) {
+        if (error) return callback(error);
+
+        if (options.domain) domains = domains.filter(d => d.domain === options.domain);
+
+        const mailSubdomain = settings.mailFqdn().substr(0, settings.mailFqdn().length - settings.mailDomain().length - 1);
+
+        apps.getAll(function (error, allApps) {
+            if (error) return callback(error);
+
+            let progress = 1, errors = [];
+
+            // we sync by domain only to get some nice progress
+            async.eachSeries(domains, function (domain, iteratorDone) {
+                progressCallback({ percent: progress, message: `Updating DNS of ${domain.domain}`});
+                progress += Math.round(100/(1+domains.length));
+
+                let locations = [];
+                if (domain.domain === settings.dashboardDomain()) locations.push({ subdomain: constants.DASHBOARD_LOCATION, domain: settings.dashboardDomain() });
+                if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.dashboardFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
+
+                allApps.forEach(function (app) {
+                    const appLocations = [{ subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains);
+                    locations = locations.concat(appLocations.filter(al => al.domain === domain.domain));
+                });
+
+                async.series([
+                    registerLocations.bind(null, locations, { overwriteDns: true }, progressCallback),
+                    progressCallback.bind(null, { message: `Updating mail DNS of ${domain.domain}`}),
+                    mail.setDnsRecords.bind(null, domain.domain)
+                ], function (error) {
+                    if (error) errors.push({ domain: domain.domain, message: error.message });
+                    iteratorDone();
+                });
+            }, () => callback(null, { errors }));
+        });
+    });
+}

src/dyndns.js

@@ -4,11 +4,12 @@ exports = module.exports = {
     sync
 };
 
-const apps = require('./apps.js'),
+let apps = require('./apps.js'),
     assert = require('assert'),
+    async = require('async'),
     constants = require('./constants.js'),
     debug = require('debug')('box:dyndns'),
-    dns = require('./dns.js'),
+    domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -16,33 +17,46 @@ const apps = require('./apps.js'),
     sysinfo = require('./sysinfo.js');
 
 // called for dynamic dns setups where we have to update the IP
-async function sync(auditSource) {
+function sync(auditSource, callback) {
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const ip = await sysinfo.getServerIp();
+    sysinfo.getServerIp(function (error, ip) {
+        if (error) return callback(error);
 
-    let info = safe.JSON.parse(safe.fs.readFileSync(paths.DYNDNS_INFO_FILE, 'utf8')) || { ip: null };
-    if (info.ip === ip) {
-        debug(`refreshDNS: no change in IP ${ip}`);
-        return;
-    }
+        let info = safe.JSON.parse(safe.fs.readFileSync(paths.DYNDNS_INFO_FILE, 'utf8')) || { ip: null };
+        if (info.ip === ip) {
+            debug(`refreshDNS: no change in IP ${ip}`);
+            return callback();
+        }
 
-    debug(`refreshDNS: updating ip from ${info.ip} to ${ip}`);
+        debug(`refreshDNS: updating ip from ${info.ip} to ${ip}`);
 
-    await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'A', [ ip ]);
-    debug('refreshDNS: updated admin location');
+        domains.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'A', [ ip ], function (error) {
+            if (error) return callback(error);
 
-    const result = await apps.list();
-    for (const app of result) {
-        // do not change state of installing apps since apptask will error if dns record already exists
-        if (app.installationState !== apps.ISTATE_INSTALLED) continue;
+            debug('refreshDNS: updated admin location');
 
-        await dns.upsertDnsRecords(app.location, app.domain, 'A', [ ip ]);
-    }
+            apps.getAll(function (error, result) {
+                if (error) return callback(error);
 
-    debug('refreshDNS: updated apps');
+                async.each(result, function (app, callback) {
+                    // do not change state of installing apps since apptask will error if dns record already exists
+                    if (app.installationState !== apps.ISTATE_INSTALLED) return callback();
 
-    await eventlog.add(eventlog.ACTION_DYNDNS_UPDATE, auditSource, { fromIp: info.ip, toIp: ip });
-    info.ip = ip;
-    safe.fs.writeFileSync(paths.DYNDNS_INFO_FILE, JSON.stringify(info), 'utf8');
+                    domains.upsertDnsRecords(app.location, app.domain, 'A', [ ip ], callback);
+                }, function (error) {
+                    if (error) return callback(error);
+
+                    debug('refreshDNS: updated apps');
+
+                    eventlog.add(eventlog.ACTION_DYNDNS_UPDATE, auditSource, { fromIp: info.ip, toIp: ip });
+                    info.ip = ip;
+                    safe.fs.writeFileSync(paths.DYNDNS_INFO_FILE, JSON.stringify(info), 'utf8');
+
+                    callback();
+                });
+            });
+        });
+    });
 }

src/eventlog.js

@@ -4,7 +4,7 @@ exports = module.exports = {
     add,
     upsertLoginEvent,
     get,
-    listPaged,
+    getAllPaged,
     cleanup,
     _clear: clear,
 
@@ -19,8 +19,6 @@ exports = module.exports = {
     ACTION_APP_UNINSTALL: 'app.uninstall',
     ACTION_APP_UPDATE: 'app.update',
     ACTION_APP_UPDATE_FINISH: 'app.update.finish',
-    ACTION_APP_BACKUP: 'app.backup',
-    ACTION_APP_BACKUP_FINISH: 'app.backup.finish',
     ACTION_APP_LOGIN: 'app.login',
     ACTION_APP_OOM: 'app.oom',
     ACTION_APP_UP: 'app.up',
@@ -56,11 +54,6 @@ exports = module.exports = {
     ACTION_PROVISION: 'cloudron.provision',
     ACTION_RESTORE: 'cloudron.restore', // unused
     ACTION_START: 'cloudron.start',
-
-    ACTION_SERVICE_CONFIGURE: 'service.configure',
-    ACTION_SERVICE_REBUILD: 'service.rebuild',
-    ACTION_SERVICE_RESTART: 'service.restart',
-
     ACTION_UPDATE: 'cloudron.update',
     ACTION_UPDATE_FINISH: 'cloudron.update.finish',
 
@@ -73,7 +66,6 @@ exports = module.exports = {
 
     ACTION_VOLUME_ADD: 'volume.add',
     ACTION_VOLUME_UPDATE: 'volume.update',
-    ACTION_VOLUME_REMOUNT: 'volume.remount',
     ACTION_VOLUME_REMOVE: 'volume.remove',
 
     ACTION_DYNDNS_UPDATE: 'dyndns.update',
@@ -86,6 +78,7 @@ exports = module.exports = {
 
 const assert = require('assert'),
     database = require('./database.js'),
+    debug = require('debug')('box:eventlog'),
     mysql = require('mysql'),
     notifications = require('./notifications.js'),
     safe = require('safetydance'),
@@ -108,9 +101,14 @@ async function add(action, source, data) {
     assert.strictEqual(typeof data, 'object');
 
     const id = uuid.v4();
-    await database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ]);
-    await notifications.onEvent(id, action, source, data);
-    return id;
+    try {
+        await database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ]);
+        await notifications.onEvent(id, action, source, data);
+        return id;
+    } catch (error) {
+        debug('add: error adding event', error);
+        return null;
+    }
 }
 
 // never throws, only logs because previously code did not take a callback
@@ -128,11 +126,16 @@ async function upsertLoginEvent(action, source, data) {
         args: [ action, JSON.stringify(source) ]
     }];
 
-    const result = await database.transaction(queries);
-    if (result[0].affectedRows >= 1) return result[1][0].id;
+    try {
+        const result = await database.transaction(queries);
+        if (result[0].affectedRows >= 1) return result[1][0].id;
 
-    // no existing eventlog found, create one
-    return await add(action, source, data);
+        // no existing eventlog found, create one
+        return await add(action, source, data);
+    } catch (error) {
+        debug('add: error adding event', error);
+        return null;
+    }
 }
 
 async function get(id) {
@@ -144,7 +147,7 @@ async function get(id) {
     return postProcess(result[0]);
 }
 
-async function listPaged(actions, search, page, perPage) {
+async function getAllPaged(actions, search, page, perPage) {
     assert(Array.isArray(actions));
     assert(typeof search === 'string' || search === null);
     assert.strictEqual(typeof page, 'number');

src/externalldap.js

@@ -1,8 +1,9 @@
 'use strict';
 
 exports = module.exports = {
+    search,
     verifyPassword,
-    maybeCreateUser,
+    createAndVerifyUserIfNotExist,
 
     testConfig,
     startSyncer,
@@ -13,19 +14,18 @@ exports = module.exports = {
     sync
 };
 
-const assert = require('assert'),
-    AuditSource = require('./auditsource.js'),
+var assert = require('assert'),
+    async = require('async'),
+    auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:externalldap'),
     groups = require('./groups.js'),
     ldap = require('ldapjs'),
     once = require('once'),
-    safe = require('safetydance'),
     settings = require('./settings.js'),
     tasks = require('./tasks.js'),
-    users = require('./users.js'),
-    util = require('util');
+    users = require('./users.js');
 
 function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.bindPassword === constants.SECRET_PLACEHOLDER) newConfig.bindPassword = currentConfig.bindPassword;
@@ -57,431 +57,532 @@ function validUserRequirements(user) {
 }
 
 // performs service bind if required
-async function getClient(externalLdapConfig, options) {
+function getClient(externalLdapConfig, doBindAuth, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof doBindAuth, 'boolean');
+    assert.strictEqual(typeof callback, 'function');
+
+    // ensure we only callback once since we also have to listen to client.error events
+    callback = once(callback);
 
     // basic validation to not crash
-    try { ldap.parseDN(externalLdapConfig.baseDn); } catch (e) { throw new BoxError(BoxError.BAD_FIELD, 'invalid baseDn'); }
-    try { ldap.parseFilter(externalLdapConfig.filter); } catch (e) { throw new BoxError(BoxError.BAD_FIELD, 'invalid filter'); }
+    try { ldap.parseDN(externalLdapConfig.baseDn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid baseDn')); }
+    try { ldap.parseFilter(externalLdapConfig.filter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid filter')); }
 
-    const config = {
+    var config = {
         url: externalLdapConfig.url,
         tlsOptions: {
             rejectUnauthorized: externalLdapConfig.acceptSelfSignedCerts ? false : true
         }
     };
 
-    let client;
+    var client;
     try {
         client = ldap.createClient(config);
     } catch (e) {
-        if (e instanceof ldap.ProtocolError) throw new BoxError(BoxError.BAD_FIELD, 'url protocol is invalid');
-        throw new BoxError(BoxError.INTERNAL_ERROR, e);
+        if (e instanceof ldap.ProtocolError) return callback(new BoxError(BoxError.BAD_FIELD, 'url protocol is invalid'));
+        return callback(new BoxError(BoxError.INTERNAL_ERROR, e));
     }
 
+    // ensure we don't just crash
+    client.on('error', function (error) {
+        callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+    });
+
     // skip bind auth if none exist or if not wanted
-    if (!externalLdapConfig.bindDn || !options.bind) return client;
+    if (!externalLdapConfig.bindDn || !doBindAuth) return callback(null, client);
 
-    return await new Promise((resolve, reject) => {
-        reject = once(reject);
+    client.bind(externalLdapConfig.bindDn, externalLdapConfig.bindPassword, function (error) {
+        if (error instanceof ldap.InvalidCredentialsError) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
-        // ensure we don't just crash
-        client.on('error', function (error) {
-            reject(new BoxError(BoxError.EXTERNAL_ERROR, error));
-        });
-
-        client.bind(externalLdapConfig.bindDn, externalLdapConfig.bindPassword, function (error) {
-            if (error instanceof ldap.InvalidCredentialsError) return reject(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (error) return reject(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-            resolve(client);
-        });
+        callback(null, client);
     });
 }
 
-async function clientSearch(client, dn, searchOptions) {
-    assert.strictEqual(typeof client, 'object');
+function ldapGetByDN(externalLdapConfig, dn, callback) {
+    assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof dn, 'string');
-    assert.strictEqual(typeof searchOptions, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    debug(`clientSearch: Get objects at ${dn} with options ${JSON.stringify(searchOptions)}`);
+    getClient(externalLdapConfig, true, function (error, client) {
+        if (error) return callback(error);
 
-    // basic validation to not crash
-    try { ldap.parseDN(dn); } catch (e) { throw new BoxError(BoxError.BAD_FIELD, 'invalid DN'); }
+        let searchOptions = {
+            paged: true,
+            scope: 'sub'    // We may have to make this configurable
+        };
+
+        debug(`Get object at ${dn}`);
+
+        // basic validation to not crash
+        try { ldap.parseDN(dn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid DN')); }
 
-    return await new Promise((resolve, reject) => {
         client.search(dn, searchOptions, function (error, result) {
-            if (error instanceof ldap.NoSuchObjectError) return reject(new BoxError(BoxError.NOT_FOUND));
-            if (error) return reject(new BoxError(BoxError.EXTERNAL_ERROR, error));
+            if (error instanceof ldap.NoSuchObjectError) return callback(new BoxError(BoxError.NOT_FOUND));
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
             let ldapObjects = [];
 
             result.on('searchEntry', entry => ldapObjects.push(entry.object));
-            result.on('error', error => reject(new BoxError(BoxError.EXTERNAL_ERROR, error)));
+            result.on('error', error => callback(new BoxError(BoxError.EXTERNAL_ERROR, error)));
 
             result.on('end', function (result) {
-                if (result.status !== 0) return reject(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
+                client.unbind();
 
-                resolve(ldapObjects);
+                if (result.status !== 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
+                if (ldapObjects.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
+
+                callback(null, ldapObjects[0]);
             });
         });
     });
 }
 
-async function ldapGetByDN(externalLdapConfig, dn) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof dn, 'string');
-
-    const searchOptions = {
-        paged: true,
-        scope: 'sub'    // We may have to make this configurable
-    };
-
-    debug(`ldapGetByDN: Get object at ${dn}`);
-
-    const client = await getClient(externalLdapConfig, { bind: true });
-    const result = await clientSearch(client, dn, searchOptions);
-    client.unbind();
-    if (result.length === 0) throw new BoxError(BoxError.NOT_FOUND);
-    return result[0];
-}
-
 // TODO support search by email
-async function ldapUserSearch(externalLdapConfig, options) {
+function ldapUserSearch(externalLdapConfig, options, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const searchOptions = {
-        paged: true,
-        filter: ldap.parseFilter(externalLdapConfig.filter),
-        scope: 'sub'    // We may have to make this configurable
-    };
+    getClient(externalLdapConfig, true, function (error, client) {
+        if (error) return callback(error);
 
-    if (options.filter) { // https://github.com/ldapjs/node-ldapjs/blob/master/docs/filters.md
-        const extraFilter = ldap.parseFilter(options.filter);
-        searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
-    }
+        let searchOptions = {
+            paged: true,
+            filter: ldap.parseFilter(externalLdapConfig.filter),
+            scope: 'sub'    // We may have to make this configurable
+        };
 
-    debug(`Listing users at ${externalLdapConfig.baseDn} with filter ${searchOptions.filter.toString()}`);
+        if (options.filter) { // https://github.com/ldapjs/node-ldapjs/blob/master/docs/filters.md
+            let extraFilter = ldap.parseFilter(options.filter);
+            searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
+        }
 
-    const client = await getClient(externalLdapConfig, { bind: true });
-    const result = await clientSearch(client, externalLdapConfig.baseDn, searchOptions);
-    client.unbind();
-    return result;
+        debug(`Listing users at ${externalLdapConfig.baseDn} with filter ${searchOptions.filter.toString()}`);
+
+        client.search(externalLdapConfig.baseDn, searchOptions, function (error, result) {
+            if (error instanceof ldap.NoSuchObjectError) return callback(new BoxError(BoxError.NOT_FOUND));
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+
+            let ldapUsers = [];
+
+            result.on('searchEntry', entry => ldapUsers.push(entry.object));
+            result.on('error', error => callback(new BoxError(BoxError.EXTERNAL_ERROR, error)));
+
+            result.on('end', function (result) {
+                client.unbind();
+
+                if (result.status !== 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
+
+                callback(null, ldapUsers);
+            });
+        });
+    });
 }
 
-async function ldapGroupSearch(externalLdapConfig, options) {
+function ldapGroupSearch(externalLdapConfig, options, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const searchOptions = {
-        paged: true,
-        scope: 'sub'    // We may have to make this configurable
-    };
+    getClient(externalLdapConfig, true, function (error, client) {
+        if (error) return callback(error);
 
-    if (externalLdapConfig.groupFilter) searchOptions.filter = ldap.parseFilter(externalLdapConfig.groupFilter);
+        let searchOptions = {
+            paged: true,
+            scope: 'sub'    // We may have to make this configurable
+        };
 
-    if (options.filter) { // https://github.com/ldapjs/node-ldapjs/blob/master/docs/filters.md
-        const extraFilter = ldap.parseFilter(options.filter);
-        searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
-    }
+        if (externalLdapConfig.groupFilter) searchOptions.filter = ldap.parseFilter(externalLdapConfig.groupFilter);
 
-    debug(`Listing groups at ${externalLdapConfig.groupBaseDn} with filter ${searchOptions.filter.toString()}`);
+        if (options.filter) { // https://github.com/ldapjs/node-ldapjs/blob/master/docs/filters.md
+            let extraFilter = ldap.parseFilter(options.filter);
+            searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
+        }
 
-    const client = await getClient(externalLdapConfig, { bind: true });
-    const result = await clientSearch(client, externalLdapConfig.groupBaseDn, searchOptions);
-    client.unbind();
-    return result;
+        debug(`Listing groups at ${externalLdapConfig.groupBaseDn} with filter ${searchOptions.filter.toString()}`);
+
+        client.search(externalLdapConfig.groupBaseDn, searchOptions, function (error, result) {
+            if (error instanceof ldap.NoSuchObjectError) return callback(new BoxError(BoxError.NOT_FOUND));
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+
+            let ldapGroups = [];
+
+            result.on('searchEntry', entry => ldapGroups.push(entry.object));
+            result.on('error', error => callback(new BoxError(BoxError.EXTERNAL_ERROR, error)));
+
+            result.on('end', function (result) {
+                client.unbind();
+
+                if (result.status !== 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
+
+                callback(null, ldapGroups);
+            });
+        });
+    });
 }
 
-async function testConfig(config) {
+function testConfig(config, callback) {
     assert.strictEqual(typeof config, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (config.provider === 'noop') return null;
+    if (config.provider === 'noop') return callback();
 
-    if (!config.url) return new BoxError(BoxError.BAD_FIELD, 'url must not be empty');
-    if (!config.url.startsWith('ldap://') && !config.url.startsWith('ldaps://')) return new BoxError(BoxError.BAD_FIELD, 'url is missing ldap:// or ldaps:// prefix');
+    if (!config.url) return callback(new BoxError(BoxError.BAD_FIELD, 'url must not be empty'));
+    if (!config.url.startsWith('ldap://') && !config.url.startsWith('ldaps://')) return callback(new BoxError(BoxError.BAD_FIELD, 'url is missing ldap:// or ldaps:// prefix'));
     if (!config.usernameField) config.usernameField = 'uid';
 
     // bindDn may not be a dn!
-    if (!config.baseDn) return new BoxError(BoxError.BAD_FIELD, 'basedn must not be empty');
-    try { ldap.parseDN(config.baseDn); } catch (e) { return new BoxError(BoxError.BAD_FIELD, 'invalid baseDn'); }
+    if (!config.baseDn) return callback(new BoxError(BoxError.BAD_FIELD, 'basedn must not be empty'));
+    try { ldap.parseDN(config.baseDn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid baseDn')); }
 
-    if (!config.filter) return new BoxError(BoxError.BAD_FIELD, 'filter must not be empty');
-    try { ldap.parseFilter(config.filter); } catch (e) { return new BoxError(BoxError.BAD_FIELD, 'invalid filter'); }
+    if (!config.filter) return callback(new BoxError(BoxError.BAD_FIELD, 'filter must not be empty'));
+    try { ldap.parseFilter(config.filter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid filter')); }
 
-    if ('syncGroups' in config && typeof config.syncGroups !== 'boolean') return new BoxError(BoxError.BAD_FIELD, 'syncGroups must be a boolean');
-    if ('acceptSelfSignedCerts' in config && typeof config.acceptSelfSignedCerts !== 'boolean') return new BoxError(BoxError.BAD_FIELD, 'acceptSelfSignedCerts must be a boolean');
+    if ('syncGroups' in config && typeof config.syncGroups !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'syncGroups must be a boolean'));
+    if ('acceptSelfSignedCerts' in config && typeof config.acceptSelfSignedCerts !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'acceptSelfSignedCerts must be a boolean'));
 
     if (config.syncGroups) {
-        if (!config.groupBaseDn) return new BoxError(BoxError.BAD_FIELD, 'groupBaseDn must not be empty');
-        try { ldap.parseDN(config.groupBaseDn); } catch (e) { return new BoxError(BoxError.BAD_FIELD, 'invalid groupBaseDn'); }
+        if (!config.groupBaseDn) return callback(new BoxError(BoxError.BAD_FIELD, 'groupBaseDn must not be empty'));
+        try { ldap.parseDN(config.groupBaseDn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid groupBaseDn')); }
 
-        if (!config.groupFilter) return new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty');
-        try { ldap.parseFilter(config.groupFilter); } catch (e) { return new BoxError(BoxError.BAD_FIELD, 'invalid groupFilter'); }
+        if (!config.groupFilter) return callback(new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty'));
+        try { ldap.parseFilter(config.groupFilter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid groupFilter')); }
 
-        if (!config.groupnameField || typeof config.groupnameField !== 'string') return new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty');
+        if (!config.groupnameField || typeof config.groupnameField !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty'));
     }
 
-    const [error, client] = await safe(getClient(config, { bind: true }));
-    if (error) return error;
+    getClient(config, true, function (error, client) {
+        if (error) return callback(error);
 
-    const opts = {
-        filter: config.filter,
-        scope: 'sub'
-    };
+        var opts = {
+            filter: config.filter,
+            scope: 'sub'
+        };
 
-    const [searchError, ] = await safe(clientSearch(client, config.baseDn, opts));
-    client.unbind();
-    if (searchError) return searchError;
+        client.search(config.baseDn, opts, function (error, result) {
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
-    return null;
+            result.on('searchEntry', function (/* entry */) {});
+            result.on('error', function (error) { client.unbind(); callback(new BoxError(BoxError.BAD_FIELD, `Unable to search directory: ${error.message}`)); });
+            result.on('end', function (/* result */) { client.unbind(); callback(); });
+        });
+    });
 }
 
-// eslint-disable-next-line no-unused-vars
-async function search(identifier) {
+function search(identifier, callback) {
     assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const externalLdapConfig = await settings.getExternalLdapConfig();
-    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
+    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
+        if (error) return callback(error);
+        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
 
-    const ldapUsers = await ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` });
+        ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
+            if (error) return callback(error);
 
-    // translate ldap properties to ours
-    let users = ldapUsers.map(function (u) { return translateUser(externalLdapConfig, u); });
+            // translate ldap properties to ours
+            let users = ldapUsers.map(function (u) { return translateUser(externalLdapConfig, u); });
 
-    return users;
+            callback(null, users);
+        });
+    });
 }
 
-async function maybeCreateUser(identifier, password) {
+function createAndVerifyUserIfNotExist(identifier, password, callback) {
     assert.strictEqual(typeof identifier, 'string');
     assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const externalLdapConfig = await settings.getExternalLdapConfig();
-    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
-    if (!externalLdapConfig.autoCreate) throw new BoxError(BoxError.BAD_STATE, 'auto create not enabled');
+    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
+        if (error) return callback(error);
+        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+        if (!externalLdapConfig.autoCreate) return callback(new BoxError(BoxError.BAD_STATE, 'auto create not enabled'));
 
-    const ldapUsers = await ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` });
-    if (ldapUsers.length === 0) throw new BoxError(BoxError.NOT_FOUND);
-    if (ldapUsers.length > 1) throw new BoxError(BoxError.CONFLICT);
+        ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
+            if (error) return callback(error);
+            if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
+            if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
 
-    const user = translateUser(externalLdapConfig, ldapUsers[0]);
-    if (!validUserRequirements(user)) throw new BoxError(BoxError.BAD_FIELD);
+            let user = translateUser(externalLdapConfig, ldapUsers[0]);
+            if (!validUserRequirements(user)) return callback(new BoxError(BoxError.BAD_FIELD));
 
-    const [error] = await safe(users.add(user.email, { username: user.username, password: null, displayName: user.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP_AUTO_CREATE));
-    if (error) {
-        debug(`maybeCreateUser: failed to auto create user ${user.username}`, error);
-        throw error;
-    }
+            users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_AUTO_CREATE, function (error, user) {
+                if (error) {
+                    debug(`createAndVerifyUserIfNotExist: Failed to auto create user ${user.username}`, error);
+                    return callback(new BoxError(BoxError.INTERNAL_ERROR));
+                }
 
-    return user;
+                verifyPassword(user, password, function (error) {
+                    if (error) return callback(error);
+                    callback(null, user);
+                });
+            });
+        });
+    });
 }
 
-async function verifyPassword(user, password) {
+function verifyPassword(user, password, callback) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const externalLdapConfig = await settings.getExternalLdapConfig();
-    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
+    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
+        if (error) return callback(error);
+        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
 
-    const ldapUsers = await ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${user.username}` });
-    if (ldapUsers.length === 0) throw new BoxError(BoxError.NOT_FOUND);
-    if (ldapUsers.length > 1) throw new BoxError(BoxError.CONFLICT);
+        ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${user.username}` }, function (error, ldapUsers) {
+            if (error) return callback(error);
+            if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
+            if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
 
-    const client = await getClient(externalLdapConfig, { bind: false });
+            getClient(externalLdapConfig, false, function (error, client) {
+                if (error) return callback(error);
 
-    const [error] = await safe(util.promisify(client.bind)(ldapUsers[0].dn, password));
-    client.unbind();
-    if (error instanceof ldap.InvalidCredentialsError) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error);
+                client.bind(ldapUsers[0].dn, password, function (error) {
+                    if (error instanceof ldap.InvalidCredentialsError) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                    if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
-    return translateUser(externalLdapConfig, ldapUsers[0]);
-}
-
-async function startSyncer() {
-    const externalLdapConfig = await settings.getExternalLdapConfig();
-    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
-
-    const taskId = await tasks.add(tasks.TASK_SYNC_EXTERNAL_LDAP, []);
-
-    tasks.startTask(taskId, {}, function (error, result) {
-        debug('sync: done', error, result);
-    });
-
-    return taskId;
-}
-
-async function syncUsers(externalLdapConfig, progressCallback) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const ldapUsers = await ldapUserSearch(externalLdapConfig, {});
-
-    debug(`syncUsers: Found ${ldapUsers.length} users`);
-
-    let percent = 10;
-    let step = 30/(ldapUsers.length+1); // ensure no divide by 0
-
-    // we ignore all errors here and just log them for now
-    for (let i = 0; i < ldapUsers.length; i++) {
-        let ldapUser = translateUser(externalLdapConfig, ldapUsers[i]);
-        if (!validUserRequirements(ldapUser)) continue;
-
-        percent += step;
-        progressCallback({ percent, message: `Syncing... ${ldapUser.username}` });
-
-        const user = await users.getByUsername(ldapUser.username);
-
-        if (!user) {
-            debug(`syncUsers: [adding user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
-
-            const [userAddError] = await safe(users.add(ldapUser.email, { username: ldapUser.username, password: null, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP_TASK));
-            if (userAddError) debug('syncUsers: Failed to create user', ldapUser, userAddError.message);
-        } else if (user.source !== 'ldap') {
-            debug(`syncUsers: [mapping user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
-
-            const [userMappingError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP_TASK));
-            if (userMappingError) debug('Failed to map user', ldapUser, userMappingError);
-        } else if (user.email !== ldapUser.email || user.displayName !== ldapUser.displayName) {
-            debug(`syncUsers: [updating user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
-
-            const [userUpdateError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName }, AuditSource.EXTERNAL_LDAP_TASK));
-            if (userUpdateError) debug('Failed to update user', ldapUser, userUpdateError);
-        } else {
-            // user known and up-to-date
-            debug(`syncUsers: [up-to-date user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
-        }
-    }
-
-    debug('syncUsers: done');
-}
-
-async function syncGroups(externalLdapConfig, progressCallback) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    if (!externalLdapConfig.syncGroups) {
-        debug('syncGroups: Group sync is disabled');
-        progressCallback({ percent: 70, message: 'Skipping group sync...' });
-        return [];
-    }
-
-    const ldapGroups = await ldapGroupSearch(externalLdapConfig, {});
-
-    debug(`syncGroups: Found ${ldapGroups.length} groups`);
-
-    let percent = 40;
-    let step = 30/(ldapGroups.length+1); // ensure no divide by 0
-
-    // we ignore all non internal errors here and just log them for now
-    for (const ldapGroup of ldapGroups) {
-        let groupName = ldapGroup[externalLdapConfig.groupnameField];
-        if (!groupName) return;
-        // some servers return empty array for unknown properties :-/
-        if (typeof groupName !== 'string') return;
-
-        // groups are lowercase
-        groupName = groupName.toLowerCase();
-
-        percent += step;
-        progressCallback({ percent, message: `Syncing... ${groupName}` });
-
-        const result = await groups.getByName(groupName);
-
-        if (!result) {
-            debug(`syncGroups: [adding group] groupname=${groupName}`);
-
-            const [error] = await safe(groups.add({ name: groupName, source: 'ldap' }));
-            if (error) debug('syncGroups: Failed to create group', groupName, error);
-        } else {
-            debug(`syncGroups: [up-to-date group] groupname=${groupName}`);
-        }
-    }
-
-    debug('syncGroups: sync done');
-}
-
-async function syncGroupUsers(externalLdapConfig, progressCallback) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    if (!externalLdapConfig.syncGroups) {
-        debug('syncGroupUsers: Group users sync is disabled');
-        progressCallback({ percent: 99, message: 'Skipping group users sync...' });
-        return [];
-    }
-
-    const allGroups = await groups.list();
-    const ldapGroups = allGroups.filter(function (g) { return g.source === 'ldap'; });
-    debug(`syncGroupUsers: Found ${ldapGroups.length} groups to sync users`);
-
-    for (const group of ldapGroups) {
-        debug(`syncGroupUsers: Sync users for group ${group.name}`);
-
-        const result = await ldapGroupSearch(externalLdapConfig, {});
-        if (!result || result.length === 0) {
-            debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
-            continue;
-        }
-
-        // since our group names are lowercase we cannot use potentially case matching ldap filters
-        let found = result.find(function (r) {
-            if (!r[externalLdapConfig.groupnameField]) return false;
-            return r[externalLdapConfig.groupnameField].toLowerCase() === group.name;
+                    callback(null, translateUser(externalLdapConfig, ldapUsers[0]));
+                });
+            });
         });
-
-        if (!found) {
-            debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
-            continue;
-        }
-
-        let ldapGroupMembers = found.member || found.uniqueMember || [];
-
-        // if only one entry is in the group ldap returns a string, not an array!
-        if (typeof ldapGroupMembers === 'string') ldapGroupMembers = [ ldapGroupMembers ];
-
-        debug(`syncGroupUsers: Group ${group.name} has ${ldapGroupMembers.length} members.`);
-
-        for (const memberDn of ldapGroupMembers) {
-            const [ldapError, result] = await safe(ldapGetByDN(externalLdapConfig, memberDn));
-            if (ldapError) {
-                debug(`syncGroupUsers: Failed to get ${memberDn}:`, ldapError);
-                continue;
-            }
-
-            debug(`syncGroupUsers: Found member object at ${memberDn} adding to group ${group.name}`);
-
-            const username = result[externalLdapConfig.usernameField];
-            if (!username) continue;
-
-            const [getError, userObject] = await safe(users.getByUsername(username));
-            if (getError || !userObject) {
-                debug(`syncGroupUsers: Failed to get user by username ${username}`, getError ? getError : 'User not found');
-                continue;
-            }
-
-            const [addError] = await safe(groups.addMember(group.id, userObject.id));
-            if (addError && addError.reason !== BoxError.ALREADY_EXISTS) debug('syncGroupUsers: Failed to add member', addError);
-        }
-    }
-
-    debug('syncGroupUsers: done');
+    });
 }
 
-async function sync(progressCallback) {
+function startSyncer(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
+        if (error) return callback(error);
+        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+
+        tasks.add(tasks.TASK_SYNC_EXTERNAL_LDAP, [], function (error, taskId) {
+            if (error) return callback(error);
+
+            tasks.startTask(taskId, {}, function (error, result) {
+                debug('sync: done', error, result);
+            });
+
+            callback(null, taskId);
+        });
+    });
+}
+
+function syncUsers(externalLdapConfig, progressCallback, callback) {
+    assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    ldapUserSearch(externalLdapConfig, {}, function (error, ldapUsers) {
+        if (error) return callback(error);
+
+        debug(`Found ${ldapUsers.length} users`);
+
+        let percent = 10;
+        let step = 30/(ldapUsers.length+1); // ensure no divide by 0
+
+        // we ignore all errors here and just log them for now
+        async.eachSeries(ldapUsers, function (user, iteratorCallback) {
+            user = translateUser(externalLdapConfig, user);
+
+            if (!validUserRequirements(user)) return iteratorCallback();
+
+            percent += step;
+            progressCallback({ percent, message: `Syncing... ${user.username}` });
+
+            users.getByUsername(user.username, function (error, result) {
+                if (error && error.reason !== BoxError.NOT_FOUND) return iteratorCallback(error);
+
+                if (!result) {
+                    debug(`[adding user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+
+                    users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
+                        if (error) debug('syncUsers: Failed to create user', user, error.message);
+                        iteratorCallback();
+                    });
+                } else if (result.source !== 'ldap') {
+                    debug(`[conflicting user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+
+                    iteratorCallback();
+                } else if (result.email !== user.email || result.displayName !== user.displayName) {
+                    debug(`[updating user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+
+                    users.update(result, { email: user.email, fallbackEmail: user.email, displayName: user.displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
+                        if (error) debug('Failed to update user', user, error);
+
+                        iteratorCallback();
+                    });
+                } else {
+                    // user known and up-to-date
+                    debug(`[up-to-date user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+
+                    iteratorCallback();
+                }
+            });
+        }, callback);
+    });
+}
+
+function syncGroups(externalLdapConfig, progressCallback, callback) {
+    assert.strictEqual(typeof externalLdapConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!externalLdapConfig.syncGroups) {
+        debug('Group sync is disabled');
+        progressCallback({ percent: 70, message: 'Skipping group sync...' });
+        return callback(null, []);
+    }
+
+    ldapGroupSearch(externalLdapConfig, {}, function (error, ldapGroups) {
+        if (error) return callback(error);
+
+        debug(`Found ${ldapGroups.length} groups`);
+
+        let percent = 40;
+        let step = 30/(ldapGroups.length+1); // ensure no divide by 0
+
+        // we ignore all non internal errors here and just log them for now
+        async.eachSeries(ldapGroups, function (ldapGroup, iteratorCallback) {
+            var groupName = ldapGroup[externalLdapConfig.groupnameField];
+            if (!groupName) return iteratorCallback();
+            // some servers return empty array for unknown properties :-/
+            if (typeof groupName !== 'string') return iteratorCallback();
+
+            // groups are lowercase
+            groupName = groupName.toLowerCase();
+
+            percent += step;
+            progressCallback({ percent, message: `Syncing... ${groupName}` });
+
+            groups.getByName(groupName, function (error, result) {
+                if (error && error.reason !== BoxError.NOT_FOUND) return iteratorCallback(error);
+
+                if (!result) {
+                    debug(`[adding group] groupname=${groupName}`);
+
+                    groups.create(groupName, 'ldap', function (error) {
+                        if (error) debug('syncGroups: Failed to create group', groupName, error);
+                        iteratorCallback();
+                    });
+                } else {
+                    debug(`[up-to-date group] groupname=${groupName}`);
+
+                    iteratorCallback();
+                }
+            });
+        }, function (error) {
+            if (error) return callback(error);
+
+            debug('sync: ldap sync is done', error);
+
+            callback(error);
+        });
+    });
+}
+
+function syncGroupUsers(externalLdapConfig, progressCallback, callback) {
+    assert.strictEqual(typeof externalLdapConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!externalLdapConfig.syncGroups) {
+        debug('Group users sync is disabled');
+        progressCallback({ percent: 99, message: 'Skipping group users sync...' });
+        return callback(null, []);
+    }
+
+    groups.getAll(function (error, result) {
+        if (error) return callback(error);
+
+        var ldapGroups = result.filter(function (g) { return g.source === 'ldap'; });
+        debug(`Found ${ldapGroups.length} groups to sync users`);
+
+        async.eachSeries(ldapGroups, function (group, iteratorCallback) {
+            debug(`Sync users for group ${group.name}`);
+
+            ldapGroupSearch(externalLdapConfig, {}, function (error, result) {
+                if (error) return callback(error);
+                if (!result || result.length === 0) {
+                    debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
+                    return callback();
+                }
+
+                // since our group names are lowercase we cannot use potentially case matching ldap filters
+                let found = result.find(function (r) {
+                    if (!r[externalLdapConfig.groupnameField]) return false;
+                    return r[externalLdapConfig.groupnameField].toLowerCase() === group.name;
+                });
+
+                if (!found) {
+                    debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
+                    return callback();
+                }
+
+                var ldapGroupMembers = found.member || found.uniqueMember || [];
+
+                // if only one entry is in the group ldap returns a string, not an array!
+                if (typeof ldapGroupMembers === 'string') ldapGroupMembers = [ ldapGroupMembers ];
+
+                debug(`Group ${group.name} has ${ldapGroupMembers.length} members.`);
+
+                async.eachSeries(ldapGroupMembers, function (memberDn, iteratorCallback) {
+                    ldapGetByDN(externalLdapConfig, memberDn, function (error, result) {
+                        if (error) {
+                            debug(`Failed to get ${memberDn}:`, error);
+                            return iteratorCallback();
+                        }
+
+                        debug(`Found member object at ${memberDn} adding to group ${group.name}`);
+
+                        const username = result[externalLdapConfig.usernameField];
+                        if (!username) return iteratorCallback();
+
+                        users.getByUsername(username, function (error, result) {
+                            if (error) {
+                                debug(`syncGroupUsers: Failed to get user by username ${username}`, error);
+                                return iteratorCallback();
+                            }
+
+                            groups.addMember(group.id, result.id, function (error) {
+                                if (error && error.reason !== BoxError.ALREADY_EXISTS) debug('syncGroupUsers: Failed to add member', error);
+                                iteratorCallback();
+                            });
+                        });
+                    });
+                }, function (error) {
+                    if (error) debug('syncGroupUsers: ', error);
+                    iteratorCallback();
+                });
+            });
+        }, callback);
+    });
+}
+
+function sync(progressCallback, callback) {
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     progressCallback({ percent: 10, message: 'Starting ldap user sync' });
 
-    const externalLdapConfig = await settings.getExternalLdapConfig();
-    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
+    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
+        if (error) return callback(error);
+        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
 
-    await syncUsers(externalLdapConfig, progressCallback);
-    await syncGroups(externalLdapConfig, progressCallback);
-    await syncGroupUsers(externalLdapConfig, progressCallback);
+        async.series([
+            syncUsers.bind(null, externalLdapConfig, progressCallback),
+            syncGroups.bind(null, externalLdapConfig, progressCallback),
+            syncGroupUsers.bind(null, externalLdapConfig, progressCallback)
+        ], function (error) {
+            if (error) return callback(error);
 
-    progressCallback({ percent: 100, message: 'Done' });
+            progressCallback({ percent: 100, message: 'Done' });
 
-    debug('sync: done');
+            debug('sync: ldap sync is done', error);
+
+            callback(error);
+        });
+    });
 }

src/groupdb.js

@@ -0,0 +1,275 @@
+'use strict';
+
+exports = module.exports = {
+    get,
+    getByName,
+    getWithMembers,
+    getAll,
+    getAllWithMembers,
+    add,
+    update,
+    del,
+    count,
+
+    getMembers,
+    addMember,
+    removeMember,
+    setMembers,
+    isMember,
+
+    getMembership,
+    setMembership,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js');
+
+var GROUPS_FIELDS = [ 'id', 'name', 'source' ].join(',');
+
+function get(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups WHERE id = ? ORDER BY name', [ groupId ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+
+        callback(null, result[0]);
+    });
+}
+
+function getByName(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups WHERE name = ?', [ name ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+
+        callback(null, result[0]);
+    });
+}
+
+function getWithMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM userGroups LEFT OUTER JOIN groupMembers ON userGroups.id = groupMembers.groupId ' +
+                    ' WHERE userGroups.id = ? ' +
+                    ' GROUP BY userGroups.id', [ groupId ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+
+        var result = results[0];
+        result.userIds = result.userIds ? result.userIds.split(',') : [ ];
+
+        callback(null, result);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups ORDER BY name', function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getAllWithMembers(callback) {
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM userGroups LEFT OUTER JOIN groupMembers ON userGroups.id = groupMembers.groupId ' +
+                    ' GROUP BY userGroups.id ORDER BY name', function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
+
+        callback(null, results);
+    });
+}
+
+function add(id, name, source, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof source, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO userGroups (id, name, source) VALUES (?, ?, ?)', [ id, name, source ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function update(id, data, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var args = [ ];
+    var fields = [ ];
+    for (var k in data) {
+        if (k === 'name') {
+            assert.strictEqual(typeof data.name, 'string');
+            fields.push(k + ' = ?');
+            args.push(data.name);
+        }
+    }
+    args.push(id);
+
+    database.query('UPDATE userGroups SET ' + fields.join(', ') + ' WHERE id = ?', args, function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('userGroups_name') !== -1) return callback(new BoxError(BoxError.ALREADY_EXISTS, 'name already exists'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+
+        return callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ id ] });
+    queries.push({ query: 'DELETE FROM userGroups WHERE id = ?', args: [ id ] });
+
+    database.transaction(queries, function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result[1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+
+        callback(error);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM userGroups', function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM groupMembers', function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        database.query('DELETE FROM userGroups', function (error) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            callback(error);
+        });
+    });
+}
+
+function getMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT userId FROM groupMembers WHERE groupId=?', [ groupId ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        // if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found')); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.userId; }));
+    });
+}
+
+function setMembers(groupId, userIds, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert(Array.isArray(userIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [];
+    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ groupId ] });
+    for (var i = 0; i < userIds.length; i++) {
+        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', args: [ groupId, userIds[i] ] });
+    }
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.CONFLICT, 'Duplicate member in list'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(error);
+    });
+}
+
+function getMembership(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT groupId FROM groupMembers WHERE userId=? ORDER BY groupId', [ userId ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        // if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found')); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.groupId; }));
+    });
+}
+
+function setMembership(userId, groupIds, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [ ];
+    queries.push({ query: 'DELETE from groupMembers WHERE userId = ?', args: [ userId ] });
+    groupIds.forEach(function (gid) {
+        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (? , ?)', args: [ gid, userId ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, error.message));
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.CONFLICT, 'Already member'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function addMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ groupId, userId ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+        if (error || result.affectedRows !== 1) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function removeMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM groupMembers WHERE groupId = ? AND userId = ?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+
+        callback(null);
+    });
+}
+
+function isMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT 1 FROM groupMembers WHERE groupId=? AND userId=?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, result.length !== 0);
+    });
+}

src/groups.js

@@ -1,14 +1,14 @@
 'use strict';
 
 exports = module.exports = {
-    add,
+    create,
     remove,
     get,
     getByName,
     update,
     getWithMembers,
-    list,
-    listWithMembers,
+    getAll,
+    getAllWithMembers,
 
     getMembers,
     addMember,
@@ -18,16 +18,16 @@ exports = module.exports = {
 
     setMembership,
     getMembership,
+
+    count
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    database = require('./database.js'),
-    safe = require('safetydance'),
-    uuid = require('uuid');
-
-const GROUPS_FIELDS = [ 'id', 'name', 'source' ].join(',');
+    groupdb = require('./groupdb.js'),
+    uuid = require('uuid'),
+    _ = require('underscore');
 
 // keep this in sync with validateUsername
 function validateGroupname(name) {
@@ -52,187 +52,199 @@ function validateGroupSource(source) {
     return null;
 }
 
-async function add(group) {
-    assert.strictEqual(typeof group, 'object');
+function create(name, source, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof source, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    let { name, source } = group;
+    // we store names in lowercase
+    name = name.toLowerCase();
 
-    name = name.toLowerCase(); // we store names in lowercase
-    source = source || '';
-
-    let error = validateGroupname(name);
-    if (error) throw error;
+    var error = validateGroupname(name);
+    if (error) return callback(error);
 
     error = validateGroupSource(source);
-    if (error) throw error;
+    if (error) return callback(error);
 
-    const id = `gid-${uuid.v4()}`;
+    var id = 'gid-' + uuid.v4();
+    groupdb.add(id, name, source, function (error) {
+        if (error) return callback(error);
 
-    [error] = await safe(database.query('INSERT INTO userGroups (id, name, source) VALUES (?, ?, ?)', [ id, name, source ]));
-    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, error);
-    if (error) throw error;
-
-    return { id, name };
+        callback(null, { id: id, name: name });
+    });
 }
 
-async function remove(id) {
+function remove(id, callback) {
     assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    // also cleanup the groupMembers table
-    let queries = [];
-    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ id ] });
-    queries.push({ query: 'DELETE FROM userGroups WHERE id = ?', args: [ id ] });
+    groupdb.del(id, function (error) {
+        if (error) return callback(error);
 
-    const result = await database.transaction(queries);
-    if (result[1].affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
+        callback(null);
+    });
 }
 
-async function get(id) {
+function get(id, callback) {
     assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const result = await database.query(`SELECT ${GROUPS_FIELDS} FROM userGroups WHERE id = ? ORDER BY name`, [ id ]);
-    if (result.length === 0) return null;
+    groupdb.get(id, function (error, result) {
+        if (error) return callback(error);
 
-    return result[0];
+        return callback(null, result);
+    });
 }
 
-async function getByName(name) {
+function getByName(name, callback) {
     assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const result = await database.query(`SELECT ${GROUPS_FIELDS} FROM userGroups WHERE name = ?`, [ name ]);
-    if (result.length === 0) return null;
+    groupdb.getByName(name, function (error, result) {
+        if (error) return callback(error);
 
-    return result[0];
+        return callback(null, result);
+    });
 }
 
-async function getWithMembers(id) {
+function getWithMembers(id, callback) {
     assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const results = await database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
-        ' FROM userGroups LEFT OUTER JOIN groupMembers ON userGroups.id = groupMembers.groupId ' +
-        ' WHERE userGroups.id = ? ' +
-        ' GROUP BY userGroups.id', [ id ]);
+    groupdb.getWithMembers(id, function (error, result) {
+        if (error) return callback(error);
 
-    if (results.length === 0) return null;
-
-    const result = results[0];
-    result.userIds = result.userIds ? result.userIds.split(',') : [ ];
-
-    return result;
+        return callback(null, result);
+    });
 }
 
-async function list() {
-    const results = await database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups ORDER BY name');
-    return results;
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAll(function (error, result) {
+        if (error) return callback(error);
+
+        return callback(null, result);
+    });
 }
 
-async function listWithMembers() {
-    const results = await database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
-        ' FROM userGroups LEFT OUTER JOIN groupMembers ON userGroups.id = groupMembers.groupId ' +
-        ' GROUP BY userGroups.id ORDER BY name');
+function getAllWithMembers(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
-    return results;
+    groupdb.getAllWithMembers(function (error, result) {
+        if (error) return callback(error);
+
+        return callback(null, result);
+    });
 }
 
-async function getMembers(groupId) {
+function getMembers(groupId, callback) {
     assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const result = await database.query('SELECT userId FROM groupMembers WHERE groupId=?', [ groupId ]);
+    groupdb.getMembers(groupId, function (error, result) {
+        if (error) return callback(error);
 
-    return result.map(function (r) { return r.userId; });
+        return callback(null, result);
+    });
 }
 
-async function getMembership(userId) {
+function getMembership(userId, callback) {
     assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const result = await database.query('SELECT groupId FROM groupMembers WHERE userId=? ORDER BY groupId', [ userId ]);
+    groupdb.getMembership(userId, function (error, result) {
+        if (error) return callback(error);
 
-    return result.map(function (r) { return r.groupId; });
+        return callback(null, result);
+    });
 }
 
-async function setMembership(userId, groupIds) {
+function setMembership(userId, groupIds, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
 
-    let queries = [ ];
-    queries.push({ query: 'DELETE from groupMembers WHERE userId = ?', args: [ userId ] });
-    groupIds.forEach(function (gid) {
-        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (? , ?)', args: [ gid, userId ] });
+    groupdb.setMembership(userId, groupIds, function (error) {
+        if (error) return callback(error);
+
+        return callback(null);
     });
-
-    const [error] = await safe(database.transaction(queries));
-    if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, error.message);
-    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.CONFLICT, 'Already member');
-    if (error) throw error;
 }
 
-async function addMember(groupId, userId) {
+function addMember(groupId, userId, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const [error] = await safe(database.query('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ groupId, userId ]));
-    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, error);
-    if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.includes('userId')) throw new BoxError(BoxError.NOT_FOUND, 'User not found');
-    if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
-    if (error) throw error;
+    groupdb.addMember(groupId, userId, function (error) {
+        if (error) return callback(error);
+
+        return callback(null);
+    });
 }
 
-async function setMembers(groupId, userIds) {
+function setMembers(groupId, userIds, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert(Array.isArray(userIds));
+    assert.strictEqual(typeof callback, 'function');
 
-    let queries = [];
-    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ groupId ] });
-    for (let i = 0; i < userIds.length; i++) {
-        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', args: [ groupId, userIds[i] ] });
-    }
+    groupdb.setMembers(groupId, userIds, function (error) {
+        if (error) return callback(error);
 
-    const [error] = await safe(database.transaction(queries));
-    if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
-    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.CONFLICT, 'Duplicate member in list');
-    if (error) throw error;
+        return callback(null);
+    });
 }
 
-async function removeMember(groupId, userId) {
+function removeMember(groupId, userId, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const result = await database.query('DELETE FROM groupMembers WHERE groupId = ? AND userId = ?', [ groupId, userId ]);
-    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
+    groupdb.removeMember(groupId, userId, function (error) {
+        if (error) return callback(error);
+
+        return callback(null);
+    });
 }
 
-async function isMember(groupId, userId) {
+function isMember(groupId, userId, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const result = await database.query('SELECT 1 FROM groupMembers WHERE groupId=? AND userId=?', [ groupId, userId ]);
-    return result.length !== 0;
+    groupdb.isMember(groupId, userId, function (error, result) {
+        if (error) return callback(error);
+
+        return callback(null, result);
+    });
 }
 
-async function update(id, data) {
-    assert.strictEqual(typeof id, 'string');
+function update(groupId, data, callback) {
+    assert.strictEqual(typeof groupId, 'string');
     assert(data && typeof data === 'object');
+    assert.strictEqual(typeof callback, 'function');
 
+    let error;
     if ('name' in data) {
         assert.strictEqual(typeof data.name, 'string');
-        const error = validateGroupname(data.name);
-        if (error) throw error;
+        error = validateGroupname(data.name);
+        if (error) return callback(error);
     }
 
-    const args = [];
-    const fields = [];
-    for (const k in data) {
-        if (k === 'name') {
-            assert.strictEqual(typeof data.name, 'string');
-            fields.push(k + ' = ?');
-            args.push(data.name);
-        }
-    }
-    args.push(id);
+    groupdb.update(groupId, _.pick(data, 'name'), function (error) {
+        if (error) return callback(error);
 
-    const [updateError, result] = await safe(database.query('UPDATE userGroups SET ' + fields.join(', ') + ' WHERE id = ?', args));
-    if (updateError && updateError.code === 'ER_DUP_ENTRY' && updateError.sqlMessage.indexOf('userGroups_name') !== -1) throw new BoxError(BoxError.ALREADY_EXISTS, 'name already exists');
-    if (updateError) throw updateError;
-    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
+        callback(null);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.count(function (error, count) {
+        if (error) return callback(error);
+
+        callback(null, count);
+    });
 }

src/infra_version.js

@@ -17,11 +17,11 @@ exports = module.exports = {
     'images': {
         'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.3.1@sha256:759cafab7625ff538418a1f2ed5558b1d5bff08c576bba577d865d6d02b49091' },
         'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.0.7@sha256:6679c2fb96f8d6d62349b607748570640a90fc46b50aad80ca2c0161655d07f4' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.1.1@sha256:86e4e2f4fd43809efca7c9cb1def4d7608cf36cb9ea27052f9b64da4481db43a' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.0.6@sha256:e583082e15e8e41b0e3b80c3efc917ec429f19fa08a19e14fc27144a8bfe446a' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.0.2@sha256:9df297ccc3370f38c54f8d614e214e082b363777cd1c6c9522e29663cc8f5362' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.0.4@sha256:5c60de75d078ae609da5565f32dcd91030f45907e945756cc976ff207b8c6199' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.4.0@sha256:5f9795cad3634c177f789019c12f8d53a6481de3cc627fb5a4866ce085006507' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.0.3@sha256:37e5222e01ae89bc5a742ce12030631de25a127b5deec8a0e992c68df0fdec10' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.3.3@sha256:b1093e6f38bebf4a9ae903ca385aea3a32e7cccae5ede7f2e01a34681e361a5f' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.0.1@sha256:bed9f6b5d06fe2c5289e895e806cfa5b74ad62993d705be55d4554a67d128029' },
-        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.4.1@sha256:13e066fcd52230f23244c16fdd2f7aa447a91e98ff703269f48b1afe3b393e31' }
+        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.3.0@sha256:183c11150d5a681cb02f7d2bd542ddb8a8f097422feafb7fac8fdbca0ca55d47' }
     }
 };

src/iputils.js

@@ -7,7 +7,6 @@ exports = module.exports = {
 
 const assert = require('assert');
 
-// this code is used in migrations - 20201120212726-apps-add-containerIp.js
 function intFromIp(address) {
     assert.strictEqual(typeof address, 'string');
 
@@ -21,7 +20,6 @@ function intFromIp(address) {
            (parseInt(parts[3], 10) << (8*0)) & 0x000000FF;
 }
 
-// this code is used in migrations - 20201120212726-apps-add-containerIp.js
 function ipFromInt(input) {
     assert.strictEqual(typeof input, 'number');
 

src/janitor.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const assert = require('assert'),
+    async = require('async'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:janitor'),
     Docker = require('dockerode'),
@@ -12,6 +13,8 @@ exports = module.exports = {
     cleanupDockerVolumes
 };
 
+const NOOP_CALLBACK = function () { };
+
 const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });
 
 async function cleanupTokens() {
@@ -23,34 +26,44 @@ async function cleanupTokens() {
     debug(`Cleaned up ${result} expired tokens`,);
 }
 
-async function cleanupTmpVolume(containerInfo) {
+function cleanupTmpVolume(containerInfo, callback) {
     assert.strictEqual(typeof containerInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const cmd = 'find /tmp -type f -mtime +10 -exec rm -rf {} +'.split(' '); // 10 day old files
+    var cmd = 'find /tmp -type f -mtime +10 -exec rm -rf {} +'.split(' '); // 10 day old files
 
     debug('cleanupTmpVolume %j', containerInfo.Names);
 
-    const [error, execContainer] = await safe(gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }));
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`);
+    gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
+        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`));
 
-    const [startError, stream] = await safe(execContainer.start({ hijack: true }));
-    if (startError) throw new BoxError(BoxError.DOCKER_ERROR, `Failed to start exec container: ${startError.message}`);
+        execContainer.start({ hijack: true }, function (error, stream) {
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to start exec container: ${error.message}`));
 
-    gConnection.modem.demuxStream(stream, process.stdout, process.stderr);
+            stream.on('error', callback);
+            stream.on('end', callback);
 
-    return new Promise((resolve, reject) => {
-        stream.on('error', (error) => reject(new BoxError(BoxError.DOCKER_ERROR, `Failed to cleanup in exec container: ${error.message}`)));
-        stream.on('end', resolve);
+            gConnection.modem.demuxStream(stream, process.stdout, process.stderr);
+        });
     });
 }
 
-async function cleanupDockerVolumes() {
+function cleanupDockerVolumes(callback) {
+    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+
+    callback = callback || NOOP_CALLBACK;
+
     debug('Cleaning up docker volumes');
 
-    const [error, containers] = await safe(gConnection.listContainers({ all: 0 }));
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
+    gConnection.listContainers({ all: 0 }, function (error, containers) {
+        if (error) return callback(error);
 
-    for (const container of containers) {
-        await safe(cleanupTmpVolume(container), { debug }); // intentionally ignore error
-    }
+        async.eachSeries(containers, function (container, iteratorDone) {
+            cleanupTmpVolume(container, function (error) {
+                if (error) debug('Error cleaning tmp: %s', error);
+
+                iteratorDone(); // intentionally ignore error
+            });
+        }, callback);
+    });
 }

src/locker.js

@@ -55,7 +55,7 @@ Locker.prototype.recursiveLock = function (operation) {
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
-    if (this._operation !== operation) throw new BoxError(BoxError.BAD_STATE, 'Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
+    if (this._operation !== operation) throw BoxError(BoxError.BAD_STATE, 'Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
     if (--this._lockDepth === 0) {
         debug('Released : %s', this._operation);

src/mail_templates/new_login_location-html.ejs

@@ -2,15 +2,15 @@
 
 <img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
 
-<h3>{{ newLoginEmail.topic }}</h3>
+<h3>We've noticed a new login on your Cloudron account.</h3>
 
-<p>{{ newLoginEmail.salutation }}</p>
+<p>Hi <%= user %>,</p>
 
-<p>{{ newLoginEmail.notice }}</p>
+<p>We noticed a login on your Cloudron account from a new device.</p>
 <p>IP: <%= ip %> (<%= city %>, <%= country %>)</p>
 <p>Browser: <%= userAgent %></p>
 
-<p>{{ newLoginEmail.action }}</p>
+<p>If this was you, you can safely disregard this email. If this wasn't you, you should change your password immediately.</p>
 
 <br/>
 <br/>

src/mail_templates/new_login_location-text.ejs

@@ -1,14 +1,14 @@
-{{ newLoginEmail.topic }}
+We've noticed a new login on your Cloudron account.
 
-{{ newLoginEmail.salutation }}
+Hi <%= user %>,
 
-{{ newLoginEmail.notice }}
+We noticed a login on your Cloudron account from a new device.
 
 IP: <%= ip %> (<%= city %>, <%= country %>)
 
 Browser: <%= userAgent %> 
 
-{{ newLoginEmail.action }}
+If this was you, you can safely disregard this email. If this wasn't you, you should change your password immediately.
 
 Powered by https://cloudron.io
 

src/mailboxdb.js

@@ -0,0 +1,408 @@
+'use strict';
+
+exports = module.exports = {
+    addMailbox,
+    addList,
+
+    updateMailbox,
+    updateList,
+    del,
+
+    getMailboxCount,
+    listMailboxes,
+    getLists,
+
+    listAllMailboxes,
+
+    get,
+    getMailbox,
+    getList,
+    getAlias,
+
+    getAliasesForName,
+    setAliasesForName,
+
+    getByOwnerId,
+    delByOwnerId,
+    delByDomain,
+
+    updateName,
+
+    _clear: clear,
+
+    TYPE_MAILBOX: 'mailbox',
+    TYPE_LIST: 'list',
+    TYPE_ALIAS: 'alias'
+};
+
+var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    mysql = require('mysql'),
+    safe = require('safetydance'),
+    util = require('util');
+
+var MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'ownerType', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain', 'active' ].join(',');
+
+function postProcess(data) {
+    data.members = safe.JSON.parse(data.membersJson) || [ ];
+    delete data.membersJson;
+
+    data.membersOnly = !!data.membersOnly;
+    data.active = !!data.active;
+
+    return data;
+}
+
+function postProcessAliases(data) {
+    const aliasNames = JSON.parse(data.aliasNames), aliasDomains = JSON.parse(data.aliasDomains);
+    delete data.aliasNames;
+    delete data.aliasDomains;
+    data.aliases = [];
+    for (let i = 0; i < aliasNames.length; i++) { // NOTE: aliasNames is [ null ] when no aliases
+        if (aliasNames[i]) data.aliases[i] = { name: aliasNames[i], domain: aliasDomains[i] };
+    }
+}
+
+function addMailbox(name, domain, data, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const { ownerId, ownerType, active } = data;
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
+    assert.strictEqual(typeof active, 'boolean');
+
+    database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, active) VALUES (?, ?, ?, ?, ?, ?)', [ name, exports.TYPE_MAILBOX, domain, ownerId, ownerType, active ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function updateMailbox(name, domain, data, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const { ownerId, ownerType, active } = data;
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
+    assert.strictEqual(typeof active, 'boolean');
+
+    database.query('UPDATE mailboxes SET ownerId = ?, ownerType = ?, active = ? WHERE name = ? AND domain = ?', [ ownerId, ownerType, active, name, domain ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+        callback(null);
+    });
+}
+
+function addList(name, domain, data, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const { members, membersOnly, active } = data;
+    assert(Array.isArray(members));
+    assert.strictEqual(typeof membersOnly, 'boolean');
+    assert.strictEqual(typeof active, 'boolean');
+
+    database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, membersJson, membersOnly, active) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+        [ name, exports.TYPE_LIST, domain, 'admin', 'user', JSON.stringify(members), membersOnly, active ], function (error) {
+            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists'));
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            callback(null);
+        });
+}
+
+function updateList(name, domain, data, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const { members, membersOnly, active } = data;
+    assert(Array.isArray(members));
+    assert.strictEqual(typeof membersOnly, 'boolean');
+    assert.strictEqual(typeof active, 'boolean');
+
+    database.query('UPDATE mailboxes SET membersJson = ?, membersOnly = ?, active = ? WHERE name = ? AND domain = ?',
+        [ JSON.stringify(members), membersOnly, active, name, domain ], function (error, result) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+            if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+            callback(null);
+        });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE mailboxes', [], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // deletes aliases as well
+    database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+        callback(null);
+    });
+}
+
+function delByDomain(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM mailboxes WHERE domain = ?', [ domain ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function delByOwnerId(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM mailboxes WHERE ownerId=?', [ id ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function updateName(oldName, oldDomain, newName, newDomain, callback) {
+    assert.strictEqual(typeof oldName, 'string');
+    assert.strictEqual(typeof oldDomain, 'string');
+    assert.strictEqual(typeof newName, 'string');
+    assert.strictEqual(typeof newDomain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // skip if no changes
+    if (oldName === newName && oldDomain === newDomain) return callback(null);
+
+    database.query('UPDATE mailboxes SET name=?, domain=? WHERE name=? AND domain = ?', [ newName, newDomain, oldName, oldDomain ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+        callback(null);
+    });
+}
+
+function get(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ?',
+        [ name, domain ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+            if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+            callback(null, postProcess(results[0]));
+        });
+}
+
+function getMailbox(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND type = ? AND domain = ?',
+        [ name, exports.TYPE_MAILBOX, domain ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+            if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+            callback(null, postProcess(results[0]));
+        });
+}
+
+function getMailboxCount(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM mailboxes WHERE type = ? AND domain = ?', [ exports.TYPE_MAILBOX, domain ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, results[0].total);
+    });
+}
+
+function listMailboxes(domain, search, page, perPage, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    const escapedSearch = mysql.escape('%' + search + '%'); // this also quotes the string
+    const searchQuery = search ? ` HAVING (name LIKE ${escapedSearch} OR aliasNames LIKE ${escapedSearch} OR aliasDomains LIKE ${escapedSearch})` : ''; // having instead of where because of aggregated columns use
+
+    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains '
+        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
+        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
+        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
+        + ' WHERE m1.domain = ?'
+        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
+        + searchQuery
+        + ' ORDER BY name LIMIT ?,?';
+
+    database.query(query, [ domain, (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        results.forEach(postProcess);
+        results.forEach(postProcessAliases);
+
+        callback(null, results);
+    });
+}
+
+function listAllMailboxes(page, perPage, callback) {
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains '
+        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
+        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
+        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
+        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
+        + ' ORDER BY name LIMIT ?,?';
+
+    database.query(query, [ (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        results.forEach(postProcess);
+        results.forEach(postProcessAliases);
+
+        callback(null, results);
+    });
+}
+
+function getLists(domain, search, page, perPage, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    let query = `SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? AND domain = ?`;
+    if (search) query += ' AND (name LIKE ' + mysql.escape('%' + search + '%') + ' OR membersJson LIKE ' + mysql.escape('%' + search + '%') + ')';
+
+    query += 'ORDER BY name LIMIT ?,?';
+
+    database.query(query, [ exports.TYPE_LIST, domain, (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function getList(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE type = ? AND name = ? AND domain = ?',
+        [ exports.TYPE_LIST, name, domain ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+            if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+            callback(null, postProcess(results[0]));
+        });
+}
+
+function getByOwnerId(ownerId, callback) {
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE ownerId = ? ORDER BY name', [ ownerId ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function setAliasesForName(name, domain, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert(Array.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ?', [ name, domain ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+        var queries = [];
+        // clear existing aliases
+        queries.push({ query: 'DELETE FROM mailboxes WHERE aliasName = ? AND aliasDomain = ? AND type = ?', args: [ name, domain, exports.TYPE_ALIAS ] });
+        aliases.forEach(function (alias) {
+            queries.push({ query: 'INSERT INTO mailboxes (name, domain, type, aliasName, aliasDomain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?, ?, ?)',
+                args: [ alias.name, alias.domain, exports.TYPE_ALIAS, name, domain, results[0].ownerId, results[0].ownerType ] });
+        });
+
+        database.transaction(queries, function (error) {
+            if (error && error.code === 'ER_DUP_ENTRY' && error.message.indexOf('mailboxes_name_domain_unique_index') !== -1) {
+                var aliasMatch = error.message.match(new RegExp(`^ER_DUP_ENTRY: Duplicate entry '(.*)-${domain}' for key 'mailboxes_name_domain_unique_index'$`));
+                if (!aliasMatch) return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
+
+                return callback(new BoxError(BoxError.ALREADY_EXISTS, `Mailbox, mailinglist or alias for ${aliasMatch[1]} already exists`));
+            }
+
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            callback(null);
+        });
+    });
+}
+
+function getAliasesForName(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name, domain FROM mailboxes WHERE type = ? AND aliasName = ? AND aliasDomain = ? ORDER BY name',
+        [ exports.TYPE_ALIAS, name, domain ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            callback(null, results);
+        });
+}
+
+function getAlias(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND type = ? AND domain = ?',
+        [ name, exports.TYPE_ALIAS, domain ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+            if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results[0]);
+        });
+}

src/maildb.js

@@ -0,0 +1,99 @@
+'use strict';
+
+exports = module.exports = {
+    get,
+    list,
+    update,
+
+    clear,
+
+    TYPE_USER: 'user',
+    TYPE_APP: 'app',
+    TYPE_GROUP: 'group'
+};
+
+var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    safe = require('safetydance');
+
+var MAILDB_FIELDS = [ 'domain', 'enabled', 'mailFromValidation', 'catchAllJson', 'relayJson', 'dkimSelector', 'bannerJson' ].join(',');
+
+function postProcess(data) {
+    data.enabled = !!data.enabled; // int to boolean
+    data.mailFromValidation = !!data.mailFromValidation; // int to boolean
+
+    data.catchAll = safe.JSON.parse(data.catchAllJson) || [ ];
+    delete data.catchAllJson;
+
+    data.relay = safe.JSON.parse(data.relayJson) || { provider: 'cloudron-smtp' };
+    delete data.relayJson;
+
+    data.banner = safe.JSON.parse(data.bannerJson) || { text: null, html: null };
+    delete data.bannerJson;
+
+    return data;
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // using TRUNCATE makes it fail foreign key check
+    database.query('DELETE FROM mail', [], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        callback(null);
+    });
+}
+
+function get(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + MAILDB_FIELDS + ' FROM mail WHERE domain = ?', [ domain ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mail domain not found'));
+
+        callback(null, postProcess(results[0]));
+    });
+}
+
+function list(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + MAILDB_FIELDS + ' FROM mail ORDER BY domain', function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function update(domain, data, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var args = [ ];
+    var fields = [ ];
+    for (var k in data) {
+        if (k === 'catchAll' || k === 'banner') {
+            fields.push(`${k}Json = ?`);
+            args.push(JSON.stringify(data[k]));
+        } else if (k === 'relay') {
+            fields.push('relayJson = ?');
+            args.push(JSON.stringify(data[k]));
+        } else {
+            fields.push(k + ' = ?');
+            args.push(data[k]);
+        }
+    }
+    args.push(domain);
+
+    database.query('UPDATE mail SET ' + fields.join(', ') + ' WHERE domain=?', args, function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Mail domain not found'));
+
+        callback(null);
+    });
+}

src/mailer.js

@@ -25,54 +25,68 @@ const assert = require('assert'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     translation = require('./translation.js'),
-    smtpTransport = require('nodemailer-smtp-transport'),
-    util = require('util');
+    smtpTransport = require('nodemailer-smtp-transport');
+
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 const MAIL_TEMPLATES_DIR = path.join(__dirname, 'mail_templates');
 
 // This will collect the most common details required for notification emails
-async function getMailConfig() {
-    const cloudronName = await settings.getCloudronName();
-    const supportConfig = await settings.getSupportConfig();
+function getMailConfig(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    return {
-        cloudronName,
-        notificationFrom: `"${cloudronName}" <no-reply@${settings.dashboardDomain()}>`,
-        supportEmail: supportConfig.email
-    };
+    settings.getCloudronName(function (error, cloudronName) {
+        if (error) debug('Error getting cloudron name: ', error);
+
+        settings.getSupportConfig(function (error, supportConfig) {
+            if (error) debug('Error getting support config: ', error);
+
+            callback(null, {
+                cloudronName: cloudronName || '',
+                notificationFrom: `"${cloudronName}" <no-reply@${settings.dashboardDomain()}>`,
+                supportEmail: supportConfig.email
+            });
+        });
+    });
 }
 
-async function sendMail(mailOptions) {
+function sendMail(mailOptions, callback) {
     assert.strictEqual(typeof mailOptions, 'object');
+    callback = callback || NOOP_CALLBACK;
 
     if (process.env.BOX_ENV === 'test') {
         exports._mailQueue.push(mailOptions);
-        return;
+        return callback();
     }
 
-    const data = await mail.getMailAuth();
+    mail.getMailAuth(function (error, data) {
+        if (error) return callback(error);
 
-    const transport = nodemailer.createTransport(smtpTransport({
-        host: data.ip,
-        port: data.port,
-        auth: {
-            user: mailOptions.authUser || `no-reply@${settings.dashboardDomain()}`,
-            pass: data.relayToken
-        }
-    }));
+        var transport = nodemailer.createTransport(smtpTransport({
+            host: data.ip,
+            port: data.port,
+            auth: {
+                user: mailOptions.authUser || `no-reply@${settings.dashboardDomain()}`,
+                pass: data.relayToken
+            }
+        }));
 
-    const transportSendMail = util.promisify(transport.sendMail).bind(transport);
-    const [error] = await safe(transportSendMail(mailOptions));
-    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error);
-    debug(`Email "${mailOptions.subject}" sent to ${mailOptions.to}`);
+        transport.sendMail(mailOptions, function (error) {
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+
+            debug(`Email "${mailOptions.subject}" sent to ${mailOptions.to}`);
+
+            callback(null);
+        });
+    });
 }
 
 function render(templateFile, params, translationAssets) {
     assert.strictEqual(typeof templateFile, 'string');
     assert.strictEqual(typeof params, 'object');
 
-    let content = null;
-    let raw = safe.fs.readFileSync(path.join(MAIL_TEMPLATES_DIR, templateFile), 'utf8');
+    var content = null;
+    var raw = safe.fs.readFileSync(path.join(MAIL_TEMPLATES_DIR, templateFile), 'utf8');
     if (raw === null) {
         debug(`Error loading ${templateFile}`);
         return '';
@@ -89,35 +103,42 @@ function render(templateFile, params, translationAssets) {
     return content;
 }
 
-async function sendInvite(user, invitor, inviteLink) {
+function sendInvite(user, invitor, inviteLink) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof invitor, 'object');
     assert.strictEqual(typeof inviteLink, 'string');
 
-    const mailConfig = await getMailConfig();
-    const translationAssets = await translation.getTranslations();
+    debug('Sending invite mail');
 
-    const templateData = {
-        user: user.displayName || user.username || user.email,
-        webadminUrl: settings.dashboardOrigin(),
-        inviteLink: inviteLink,
-        invitor: invitor ? invitor.email : null,
-        cloudronName: mailConfig.cloudronName,
-        cloudronAvatarUrl: settings.dashboardOrigin() + '/api/v1/cloudron/avatar'
-    };
+    getMailConfig(function (error, mailConfig) {
+        if (error) return debug('Error getting mail details:', error);
 
-    const mailOptions = {
-        from: mailConfig.notificationFrom,
-        to: user.fallbackEmail,
-        subject: ejs.render(translation.translate('{{ welcomeEmail.subject }}', translationAssets.translations || {}, translationAssets.fallback || {}), { cloudron: mailConfig.cloudronName }),
-        text: render('welcome_user-text.ejs', templateData, translationAssets),
-        html: render('welcome_user-html.ejs', templateData, translationAssets)
-    };
+        translation.getTranslations(function (error, translationAssets) {
+            if (error) return debug('Error getting translations:', error);
 
-    await sendMail(mailOptions);
+            var templateData = {
+                user: user.displayName || user.username || user.email,
+                webadminUrl: settings.dashboardOrigin(),
+                inviteLink: inviteLink,
+                invitor: invitor ? invitor.email : null,
+                cloudronName: mailConfig.cloudronName,
+                cloudronAvatarUrl: settings.dashboardOrigin() + '/api/v1/cloudron/avatar'
+            };
+
+            var mailOptions = {
+                from: mailConfig.notificationFrom,
+                to: user.fallbackEmail,
+                subject: ejs.render(translation.translate('{{ welcomeEmail.subject }}', translationAssets.translations || {}, translationAssets.fallback || {}), { cloudron: mailConfig.cloudronName }),
+                text: render('welcome_user-text.ejs', templateData, translationAssets),
+                html: render('welcome_user-html.ejs', templateData, translationAssets)
+            };
+
+            sendMail(mailOptions);
+        });
+    });
 }
 
-async function sendNewLoginLocation(user, loginLocation) {
+function sendNewLoginLocation(user, loginLocation) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof loginLocation, 'object');
 
@@ -128,101 +149,124 @@ async function sendNewLoginLocation(user, loginLocation) {
     assert.strictEqual(typeof country, 'string');
     assert.strictEqual(typeof city, 'string');
 
-    const mailConfig = await getMailConfig();
-    const translationAssets = await translation.getTranslations();
+    debug('Sending new login location mail');
 
-    const templateData = {
-        user: user.displayName || user.username || user.email,
-        ip,
-        userAgent: userAgent || 'unknown',
-        country,
-        city,
-        cloudronName: mailConfig.cloudronName,
-        cloudronAvatarUrl: settings.dashboardOrigin() + '/api/v1/cloudron/avatar'
-    };
+    getMailConfig(function (error, mailConfig) {
+        if (error) return debug('Error getting mail details:', error);
 
-    const mailOptions = {
-        from: mailConfig.notificationFrom,
-        to: user.fallbackEmail,
-        subject: ejs.render(translation.translate('{{ newLoginEmail.subject }}', translationAssets.translations || {}, translationAssets.fallback || {}), { cloudron: mailConfig.cloudronName }),
-        text: render('new_login_location-text.ejs', templateData, translationAssets),
-        html: render('new_login_location-html.ejs', templateData, translationAssets)
-    };
+        translation.getTranslations(function (error, translationAssets) {
+            if (error) return debug('Error getting translations:', error);
 
-    await sendMail(mailOptions);
+            const templateData = {
+                user: user.displayName || user.username || user.email,
+                ip,
+                userAgent: userAgent || 'unknown',
+                country,
+                city,
+                cloudronName: mailConfig.cloudronName,
+                cloudronAvatarUrl: settings.dashboardOrigin() + '/api/v1/cloudron/avatar'
+            };
+
+            const mailOptions = {
+                from: mailConfig.notificationFrom,
+                to: user.fallbackEmail,
+                subject: `[${mailConfig.cloudronName}] New login on your account`,
+                text: render('new_login_location-text.ejs', templateData, translationAssets),
+                html: render('new_login_location-html.ejs', templateData, translationAssets)
+            };
+
+            sendMail(mailOptions);
+        });
+    });
 }
 
-async function passwordReset(user, resetLink) {
+function passwordReset(user) {
     assert.strictEqual(typeof user, 'object');
 
-    const mailConfig = await getMailConfig();
-    const translationAssets = await translation.getTranslations();
+    debug('Sending mail for password reset for user %s.', user.email, user.id);
 
-    const templateData = {
-        user: user.displayName || user.username || user.email,
-        resetLink: resetLink,
-        cloudronName: mailConfig.cloudronName,
-        cloudronAvatarUrl: settings.dashboardOrigin() + '/api/v1/cloudron/avatar'
-    };
+    getMailConfig(function (error, mailConfig) {
+        if (error) return debug('Error getting mail details:', error);
 
-    const mailOptions = {
-        from: mailConfig.notificationFrom,
-        to: user.fallbackEmail,
-        subject: ejs.render(translation.translate('{{ passwordResetEmail.subject }}', translationAssets.translations || {}, translationAssets.fallback || {}), { cloudron: mailConfig.cloudronName }),
-        text: render('password_reset-text.ejs', templateData, translationAssets),
-        html: render('password_reset-html.ejs', templateData, translationAssets)
-    };
+        translation.getTranslations(function (error, translationAssets) {
+            if (error) return debug('Error getting translations:', error);
 
-    await sendMail(mailOptions);
+            var templateData = {
+                user: user.displayName || user.username || user.email,
+                resetLink: `${settings.dashboardOrigin()}/login.html?resetToken=${user.resetToken}`,
+                cloudronName: mailConfig.cloudronName,
+                cloudronAvatarUrl: settings.dashboardOrigin() + '/api/v1/cloudron/avatar'
+            };
+
+            var mailOptions = {
+                from: mailConfig.notificationFrom,
+                to: user.fallbackEmail,
+                subject: ejs.render(translation.translate('{{ passwordResetEmail.subject }}', translationAssets.translations || {}, translationAssets.fallback || {}), { cloudron: mailConfig.cloudronName }),
+                text: render('password_reset-text.ejs', templateData, translationAssets),
+                html: render('password_reset-html.ejs', templateData, translationAssets)
+            };
+
+            sendMail(mailOptions);
+        });
+    });
 }
 
-async function backupFailed(mailTo, errorMessage, logUrl) {
+function backupFailed(mailTo, errorMessage, logUrl, callback) {
     assert.strictEqual(typeof mailTo, 'string');
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof logUrl, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const mailConfig = await getMailConfig();
+    getMailConfig(function (error, mailConfig) {
+        if (error) return debug('Error getting mail details:', error);
 
-    const mailOptions = {
-        from: mailConfig.notificationFrom,
-        to: mailTo,
-        subject: `[${mailConfig.cloudronName}] Failed to backup`,
-        text: render('backup_failed.ejs', { cloudronName: mailConfig.cloudronName, message: errorMessage, logUrl, format: 'text' })
-    };
+        var mailOptions = {
+            from: mailConfig.notificationFrom,
+            to: mailTo,
+            subject: `[${mailConfig.cloudronName}] Failed to backup`,
+            text: render('backup_failed.ejs', { cloudronName: mailConfig.cloudronName, message: errorMessage, logUrl, format: 'text' })
+        };
 
-    await sendMail(mailOptions);
+        sendMail(mailOptions, callback);
+    });
 }
 
-async function certificateRenewalError(mailTo, domain, message) {
+function certificateRenewalError(mailTo, domain, message, callback) {
     assert.strictEqual(typeof mailTo, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof message, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const mailConfig = await getMailConfig();
+    getMailConfig(function (error, mailConfig) {
+        if (error) return debug('Error getting mail details:', error);
 
-    const mailOptions = {
-        from: mailConfig.notificationFrom,
-        to: mailTo,
-        subject: `[${mailConfig.cloudronName}] Certificate renewal error`,
-        text: render('certificate_renewal_error.ejs', { domain: domain, message: message, format: 'text' })
-    };
+        const mailOptions = {
+            from: mailConfig.notificationFrom,
+            to: mailTo,
+            subject: `[${mailConfig.cloudronName}] Certificate renewal error`,
+            text: render('certificate_renewal_error.ejs', { domain: domain, message: message, format: 'text' })
+        };
 
-    await sendMail(mailOptions);
+        sendMail(mailOptions, callback);
+    });
 }
 
-async function sendTestMail(domain, email) {
+function sendTestMail(domain, email, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof email, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const mailConfig = await getMailConfig();
+    getMailConfig(function (error, mailConfig) {
+        if (error) return debug('Error getting mail details:', error);
 
-    const mailOptions = {
-        authUser: `no-reply@${domain}`,
-        from: `"${mailConfig.cloudronName}" <no-reply@${domain}>`,
-        to: email,
-        subject: `[${mailConfig.cloudronName}] Test Email`,
-        text: render('test.ejs', { cloudronName: mailConfig.cloudronName, format: 'text'})
-    };
+        var mailOptions = {
+            authUser: `no-reply@${domain}`,
+            from: `"${mailConfig.cloudronName}" <no-reply@${domain}>`,
+            to: email,
+            subject: `[${mailConfig.cloudronName}] Test Email`,
+            text: render('test.ejs', { cloudronName: mailConfig.cloudronName, format: 'text'})
+        };
 
-    await sendMail(mailOptions);
+        sendMail(mailOptions, callback);
+    });
 }

src/mounts.js

@@ -1,19 +1,15 @@
 'use strict';
 
 exports = module.exports = {
-    isMountProvider,
-    mountObjectFromBackupConfig,
     tryAddMount,
     removeMount,
     validateMountOptions,
     getStatus,
-    remount
 };
 
 const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    debug = require('debug')('box:mounts'),
     ejs = require('ejs'),
     fs = require('fs'),
     path = require('path'),
@@ -23,7 +19,6 @@ const assert = require('assert'),
 
 const ADD_MOUNT_CMD = path.join(__dirname, 'scripts/addmount.sh');
 const RM_MOUNT_CMD = path.join(__dirname, 'scripts/rmmount.sh');
-const REMOUNT_MOUNT_CMD = path.join(__dirname, 'scripts/remountmount.sh');
 const SYSTEMD_MOUNT_EJS = fs.readFileSync(path.join(__dirname, 'systemd-mount.ejs'), { encoding: 'utf8' });
 
 // https://man7.org/linux/man-pages/man8/mount.8.html
@@ -56,31 +51,18 @@ function validateMountOptions(type, options) {
         if (typeof options.diskPath !== 'string') return new BoxError(BoxError.BAD_FIELD, 'diskPath is not a string');
         return null;
     default:
-        return new BoxError(BoxError.BAD_FIELD, 'Bad mount type');
+        return new BoxError(BoxError.BAD_FIELD, 'Bad volume mount type');
     }
 }
 
-function isMountProvider(provider) {
-    return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4';
-}
-
-function mountObjectFromBackupConfig(backupConfig) {
-    return {
-        name: 'backup',
-        hostPath: backupConfig.mountPoint,
-        mountType: backupConfig.provider,
-        mountOptions: backupConfig.mountOptions
-    };
-}
-
 // https://www.man7.org/linux/man-pages/man8/mount.8.html for various mount option flags
 // nfs - no_root_squash is mode on server to map all root to 'nobody' user. all_squash does this for all users (making it like ftp)
 // sshfs - supports users/permissions
 // cifs - does not support permissions
-function renderMountFile(mount) {
-    assert.strictEqual(typeof mount, 'object');
+function renderMountFile(volume) {
+    assert.strictEqual(typeof volume, 'object');
 
-    const {name, hostPath, mountType, mountOptions} = mount;
+    const {name, hostPath, mountType, mountOptions} = volume;
 
     let options, what, type;
     switch (mountType) {
@@ -114,14 +96,14 @@ function renderMountFile(mount) {
     return ejs.render(SYSTEMD_MOUNT_EJS, { name, what, where: hostPath, options, type });
 }
 
-async function removeMount(mount) {
-    assert.strictEqual(typeof mount, 'object');
+async function removeMount(volume) {
+    assert.strictEqual(typeof volume, 'object');
 
-    const { hostPath, mountType, mountOptions } = mount;
+    const { hostPath, mountType, mountOptions } = volume;
 
     if (constants.TEST) return;
 
-    await safe(shell.promises.sudo('removeMount', [ RM_MOUNT_CMD, hostPath ], {}), { debug }); // ignore any error
+    await safe(shell.promises.sudo('removeMount', [ RM_MOUNT_CMD, hostPath ], {})); // ignore any error
 
     if (mountType === 'sshfs') {
         const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${mountOptions.host}`);
@@ -135,32 +117,25 @@ async function getStatus(mountType, hostPath) {
 
     if (mountType === 'filesystem') return { state: 'active', message: 'Mounted' };
 
-    const state = safe.child_process.execSync(`mountpoint -q -- ${hostPath}`) ? 'active' : 'inactive';
+    if (mountType === 'mountpoint') {
+        if (safe.child_process.execSync(`mountpoint -q -- ${hostPath}`)) {
+            return { state: 'active', message: 'Mounted' };
+        } else {
+            return { state: 'inactive', message: 'Not mounted' };
+        }
+    }
 
-    if (mountType === 'mountpoint') return { state, message: state === 'active' ? 'Mounted' : 'Not mounted' };
-
-    // we used to rely on "systemctl show -p ActiveState" output before but some mounts like sshfs.fuse show the status as "active" event though the mount commant failed (on ubuntu 18)
+    let output = safe.child_process.execSync(`systemctl show -p ActiveState $(systemd-escape -p --suffix=mount ${hostPath})`, { encoding: 'utf8' }); // --value does not work in ubuntu 16
+    const state = output ? output.trim().split('=')[1] : '';
     let message;
 
     if (state !== 'active') { // find why it failed
-        const logsJson = safe.child_process.execSync(`journalctl -u $(systemd-escape -p --suffix=mount ${hostPath}) -n 10 --no-pager -o json`, { encoding: 'utf8' });
+        output = safe.child_process.execSync(`journalctl -u $(systemd-escape -p --suffix=mount ${hostPath}) -n 10 --no-pager -o cat`, { encoding: 'utf8' });
 
-        if (logsJson) {
-            const lines = logsJson.trim().split('\n').map(l => JSON.parse(l)); // array of json
-            let start = -1, end = -1; // start and end of error message block
-            for (let idx = lines.length - 1; idx >= 0; idx--) { // reverse
-                const line = lines[idx];
-                const match = line['SYSLOG_IDENTIFIER'] === 'mount' || line['_EXE'].includes('mount') || line['_COMM'].includes('mount');
-                if (match) {
-                    if (end === -1) end = idx;
-                    start = idx;
-                    continue;
-                }
-
-                if (end !== -1) break; // no match and we already found a block
-            }
-
-            if (end !== -1) message = lines.slice(start, end+1).map(line => line['MESSAGE']).join('\n');
+        if (output) {
+            const rlines = output.split('\n').reverse();
+            const idx = rlines.findIndex(l => /^mount./.test(l) || l.includes('failed') || l.includes('error') || l.includes('reset'));
+            if (idx !== -1) message = rlines[idx];
         }
         if (!message) message = `Could not determine failure reason. ${safe.error ? safe.error.message : ''}`;
     } else {
@@ -170,40 +145,27 @@ async function getStatus(mountType, hostPath) {
     return { state, message };
 }
 
-async function tryAddMount(mount, options) {
-    assert.strictEqual(typeof mount, 'object'); // { name, hostPath, mountType, mountOptions }
-    assert.strictEqual(typeof options, 'object'); // { timeout, skipCleanup }
+async function tryAddMount(volume, options) {
+    assert.strictEqual(typeof volume, 'object');
+    assert.strictEqual(typeof options, 'object');
 
-    if (mount.mountType === 'mountpoint') return;
+    if (volume.mountType === 'mountpoint') return;
 
     if (constants.TEST) return;
 
-    if (mount.mountType === 'sshfs') {
-        const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${mount.mountOptions.host}`);
+    if (volume.mountType === 'sshfs') {
+        const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${volume.mountOptions.host}`);
 
         safe.fs.mkdirSync(paths.SSHFS_KEYS_DIR);
-        if (!safe.fs.writeFileSync(keyFilePath, `${mount.mountOptions.privateKey}\n`, { mode: 0o600 })) throw new BoxError(BoxError.FS_ERROR, safe.error);
+        if (!safe.fs.writeFileSync(keyFilePath, `${volume.mountOptions.privateKey}\n`, { mode: 0o600 })) throw new BoxError(BoxError.FS_ERROR, safe.error);
     }
 
-    const [error] = await safe(shell.promises.sudo('addMount', [ ADD_MOUNT_CMD, renderMountFile(mount), options.timeout ], {}));
+    const [error] = await safe(shell.promises.sudo('addMount', [ ADD_MOUNT_CMD, renderMountFile(volume), options.timeout ], {}));
     if (error && error.code === 2) throw new BoxError(BoxError.MOUNT_ERROR, 'Failed to unmount existing mount'); // at this point, the old mount config is still there
 
-    if (options.skipCleanup) return;
-
-    const status = await getStatus(mount.mountType, mount.hostPath);
+    const status = await getStatus(volume.mountType, volume.hostPath);
     if (status.state !== 'active') {  // cleanup
-        await removeMount(mount);
+        await removeMount(volume);
         throw new BoxError(BoxError.MOUNT_ERROR, `Failed to mount (${status.state}): ${status.message}`);
     }
 }
-
-async function remount(mount) {
-    assert.strictEqual(typeof mount, 'object'); // { name, hostPath, mountType, mountOptions }
-
-    if (mount.mountType === 'mountpoint') return;
-
-    if (constants.TEST) return;
-
-    const [error] = await safe(shell.promises.sudo('remountMount', [ REMOUNT_MOUNT_CMD, mount.hostPath ], {}));
-    if (error && error.code === 2) throw new BoxError(BoxError.MOUNT_ERROR, 'Failed to remount existing mount'); // at this point, the old mount config is still there
-}

src/native-dns.js

@@ -0,0 +1,43 @@
+'use strict';
+
+exports = module.exports = {
+    resolve
+};
+
+var assert = require('assert'),
+    constants = require('./constants.js'),
+    dns = require('dns'),
+    _ = require('underscore');
+
+const DEFAULT_OPTIONS = { server: '127.0.0.1', timeout: 5000 }; // unbound runs on 127.0.0.1
+
+// a note on TXT records. It doesn't have quotes ("") at the DNS level. Those quotes
+// are added for DNS server software to enclose spaces. Such quotes may also be returned
+// by the DNS REST API of some providers
+function resolve(hostname, rrtype, options, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof rrtype, 'string');
+    assert(options && typeof options === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const resolver = new dns.Resolver();
+    options = _.extend({ }, DEFAULT_OPTIONS, options);
+
+    // Only use unbound on a Cloudron
+    if (constants.CLOUDRON) resolver.setServers([ options.server ]);
+
+    // should callback with ECANCELLED but looks like we might hit https://github.com/nodejs/node/issues/14814
+    const timerId = setTimeout(resolver.cancel.bind(resolver), options.timeout || 5000);
+
+    resolver.resolve(hostname, rrtype, function (error, result) {
+        clearTimeout(timerId);
+
+        if (error && error.code === 'ECANCELLED') error.code = 'TIMEOUT';
+
+        // result is an empty array if there was no error but there is no record. when you query a random
+        // domain, it errors with ENOTFOUND. But if you query an existing domain (A record) but with different
+        // type (CNAME) it is not an error and empty array
+        // for TXT records, result is 2d array of strings
+        callback(error, result);
+    });
+}

src/network.js

@@ -17,36 +17,48 @@ const assert = require('assert'),
 
 const SET_BLOCKLIST_CMD = path.join(__dirname, 'scripts/setblocklist.sh');
 
-async function getBlocklist() {
-    return await settings.getFirewallBlocklist();
+function getBlocklist(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getFirewallBlocklist(function (error, blocklist) {
+        if (error) return callback(error);
+
+        callback(null, blocklist);
+    });
 }
 
-async function setBlocklist(blocklist, auditSource) {
+function setBlocklist(blocklist, auditSource, callback) {
     assert.strictEqual(typeof blocklist, 'string');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     const parsedIp = ipaddr.process(auditSource.ip);
 
     for (const line of blocklist.split('\n')) {
         if (!line || line.startsWith('#')) continue;
         const rangeOrIP = line.trim();
-        if (!validator.isIP(rangeOrIP) && !validator.isIPRange(rangeOrIP)) throw new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} is not a valid IP or range`);
+        if (!validator.isIP(rangeOrIP) && !validator.isIPRange(rangeOrIP)) return callback(new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} is not a valid IP or range`));
 
         if (rangeOrIP.indexOf('/') === -1) {
-            if (auditSource.ip === rangeOrIP) throw new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} includes client IP. Cannot block yourself`);
+            if (auditSource.ip === rangeOrIP) return callback(new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} includes client IP. Cannot block yourself`));
         } else {
             const parsedRange = ipaddr.parseCIDR(rangeOrIP); // returns [addr, range]
-            if (parsedRange[0].kind() === parsedIp.kind() && parsedIp.match(parsedRange)) throw new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} includes client IP. Cannot block yourself`);
+            if (parsedRange[0].kind() === parsedIp.kind() && parsedIp.match(parsedRange)) return callback(new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} includes client IP. Cannot block yourself`));
         }
     }
 
-    if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
+    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
 
-    await settings.setFirewallBlocklist(blocklist);
+    settings.setFirewallBlocklist(blocklist, function (error) {
+        if (error) return callback(error);
 
-    // this is done only because it's easier for the shell script and the firewall service to get the value
-    if (!safe.fs.writeFileSync(paths.FIREWALL_BLOCKLIST_FILE, blocklist + '\n', 'utf8')) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
+        // this is done only because it's easier for the shell script and the firewall service to get the value
+        if (!safe.fs.writeFileSync(paths.FIREWALL_BLOCKLIST_FILE, blocklist + '\n', 'utf8')) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
 
-    const [error] = await safe(shell.promises.sudo('setBlocklist', [ SET_BLOCKLIST_CMD ], {}));
-    if (error) throw new BoxError(BoxError.IPTABLES_ERROR, `Error setting blocklist: ${error.message}`);
+        shell.sudo('setBlocklist', [ SET_BLOCKLIST_CMD ], {}, function (error) {
+            if (error) return callback(new BoxError(BoxError.IPTABLES_ERROR, `Error setting blocklist: ${error.message}`));
+
+            callback(null);
+        });
+    });
 }

src/nginxconfig.ejs

@@ -275,9 +275,6 @@ server {
         if ($http_user_agent ~* "docker") {
             return 401;
         }
-        if ($http_user_agent ~* "container") {
-            return 401;
-        }
         return 302 /login?redirect=$request_uri;
     }
 

src/notifications.js

@@ -22,14 +22,16 @@ exports = module.exports = {
 };
 
 const assert = require('assert'),
-    AuditSource = require('./auditsource.js'),
+    async = require('async'),
+    auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     changelog = require('./changelog.js'),
     database = require('./database.js'),
     eventlog = require('./eventlog.js'),
     mailer = require('./mailer.js'),
     settings = require('./settings.js'),
-    users = require('./users.js');
+    users = require('./users.js'),
+    util = require('util');
 
 const NOTIFICATION_FIELDS = [ 'id', 'eventId', 'title', 'message', 'creationTime', 'acknowledged' ];
 
@@ -120,44 +122,38 @@ async function list(filters, page, perPage) {
     return results;
 }
 
-async function oomEvent(eventId, containerId, app, addonName /*, event*/) {
+async function oomEvent(eventId, app, addon, containerId /*, event*/) {
     assert.strictEqual(typeof eventId, 'string');
-    assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof app, 'object');
-    assert(addonName === null || typeof addonName === 'string');
+    assert.strictEqual(typeof addon, 'object');
+    assert.strictEqual(typeof containerId, 'string');
 
-    assert(app || addonName);
+    assert(app || addon);
 
     let title, message;
-    if (addonName) {
-        if (app) {
-            title = `The ${addonName} service of the app at ${app.fqdn} ran out of memory`;
-        } else {
-            title = `The ${addonName} service ran out of memory`;
-        }
+    if (app) {
+        title = `The application at ${app.fqdn} ran out of memory.`;
+        message = `The application has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.dashboardOrigin()}/#/app/${app.id}/resources)`;
+    } else if (addon) {
+        title = `The ${addon.name} service ran out of memory`;
         message = `The service has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.dashboardOrigin()}/#/services)`;
-    } else if (app) {
-        title = `The app at ${app.fqdn} ran out of memory.`;
-        message = `The app has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.dashboardOrigin()}/#/app/${app.id}/resources)`;
     }
 
     await add(eventId, title, message);
 }
 
-async function appUpdated(eventId, app, fromManifest, toManifest) {
+async function appUpdated(eventId, app) {
     assert.strictEqual(typeof eventId, 'string');
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof fromManifest, 'object');
-    assert.strictEqual(typeof toManifest, 'object');
 
     if (!app.appStoreId) return; // skip notification of dev apps
 
-    const tmp = toManifest.description.match(/<upstream>(.*)<\/upstream>/i);
+    const tmp = app.manifest.description.match(/<upstream>(.*)<\/upstream>/i);
     const upstreamVersion = (tmp && tmp[1]) ? tmp[1] : '';
-    const title = upstreamVersion ? `${toManifest.title} at ${app.fqdn} updated to ${upstreamVersion} (package version ${toManifest.version})`
-        : `${toManifest.title} at ${app.fqdn} updated to package version ${toManifest.version}`;
+    const title = upstreamVersion ? `${app.manifest.title} at ${app.fqdn} updated to ${upstreamVersion} (package version ${app.manifest.version})`
+        : `${app.manifest.title} at ${app.fqdn} updated to package version ${app.manifest.version}`;
 
-    await add(eventId, title, `The application installed at https://${app.fqdn} was updated.\n\nChangelog:\n${toManifest.changelog}\n`);
+    await add(eventId, title, `The application installed at https://${app.fqdn} was updated.\n\nChangelog:\n${app.manifest.changelog}\n`);
 }
 
 async function boxUpdated(eventId, oldVersion, newVersion) {
@@ -183,12 +179,21 @@ async function certificateRenewalError(eventId, vhost, errorMessage) {
     assert.strictEqual(typeof vhost, 'string');
     assert.strictEqual(typeof errorMessage, 'string');
 
-    await add(eventId, `Certificate renewal of ${vhost} failed`, `Failed to renew certs of ${vhost}: ${errorMessage}. Renewal will be retried in 12 hours.`);
+    return new Promise((resolve, reject) => {
+        users.getAdmins(function (error, admins) {
+            if (error) return reject(error);
 
-    const admins = await users.getAdmins();
-    for (const admin of admins) {
-        await mailer.certificateRenewalError(admin.email, vhost, errorMessage);
-    }
+            async.eachSeries(admins, function (admin, iteratorDone) {
+                mailer.certificateRenewalError(admin.email, vhost, errorMessage, iteratorDone);
+            }, async function (error) {
+                if (error) return reject(error);
+
+                await add(eventId, `Certificate renewal of ${vhost} failed`, `Failed to renew certs of ${vhost}: ${errorMessage}. Renewal will be retried in 12 hours.`);
+
+                resolve();
+            });
+        });
+    });
 }
 
 async function backupFailed(eventId, taskId, errorMessage) {
@@ -199,18 +204,27 @@ async function backupFailed(eventId, taskId, errorMessage) {
     await add(eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}).`);
 
     // only send mail if the past 3 automated backups failed
-    const backupEvents = await eventlog.listPaged([eventlog.ACTION_BACKUP_FINISH], null /* search */, 1, 20);
+    const backupEvents = await eventlog.getAllPaged([eventlog.ACTION_BACKUP_FINISH], null /* search */, 1, 20);
     let count = 0;
     for (const event of backupEvents) {
         if (!event.data.errorMessage) return; // successful backup (manual or cron)
-        if (event.source.username === AuditSource.CRON.username && ++count === 3) break; // last 3 consecutive crons have failed
+        if (event.source.username === auditSource.CRON.username && ++count === 3) break; // last 3 consecutive crons have failed
     }
     if (count !== 3) return; // less than 3 failures
 
-    const superadmins = await users.getSuperadmins();
-    for (const superadmin of superadmins) {
-        await mailer.backupFailed(superadmin.email, errorMessage, `${settings.dashboardOrigin()}/logs.html?taskId=${taskId}`);
-    }
+    return new Promise((resolve, reject) => {
+        users.getSuperadmins(function (error, superadmins) {
+            if (error) return reject(error);
+
+            async.eachSeries(superadmins, function (superadmin, iteratorDone) {
+                mailer.backupFailed(superadmin.email, errorMessage, `${settings.dashboardOrigin()}/logs.html?taskId=${taskId}`, iteratorDone);
+            }, function (error) {
+                if (error) return reject(error);
+
+                resolve();
+            });
+        });
+    });
 }
 
 // id is unused but nice to search code
@@ -247,16 +261,14 @@ async function onEvent(id, action, source, data) {
     assert.strictEqual(typeof data, 'object');
 
     // external ldap syncer does not generate notifications - FIXME username might be an issue here
-    if (source.username === AuditSource.EXTERNAL_LDAP_TASK.username) return;
+    if (source.username === auditSource.EXTERNAL_LDAP_TASK.username) return;
 
     switch (action) {
     case eventlog.ACTION_APP_OOM:
-        return await oomEvent(id, data.containerId, data.app, data.addonName, data.event);
+        return await oomEvent(id, data.app, data.addon, data.containerId, data.event);
 
     case eventlog.ACTION_APP_UPDATE_FINISH:
-        if (source.username !== AuditSource.CRON.username) return; // updated by user
-        if (data.errorMessage) return; // the update indicator will still appear, so no need to notify user
-        return await appUpdated(id, data.app, data.fromManifest, data.toManifest);
+        return await appUpdated(id, data.app);
 
     case eventlog.ACTION_CERTIFICATE_RENEWAL:
     case eventlog.ACTION_CERTIFICATE_NEW:
@@ -266,7 +278,7 @@ async function onEvent(id, action, source, data) {
 
     case eventlog.ACTION_BACKUP_FINISH:
         if (!data.errorMessage) return;
-        if (source.username !== AuditSource.CRON.username && !data.timedOut) return; // manual stop by user
+        if (source.username !== auditSource.CRON.username && !data.timedOut) return; // manual stop by user
 
         return await backupFailed(id, data.taskId, data.errorMessage); // only notify for automated backups or timedout
 

src/paths.js

@@ -4,7 +4,7 @@ var constants = require('./constants.js'),
     path = require('path');
 
 function baseDir() {
-    const homeDir = process.env.HOME;
+    const homeDir = process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
     if (constants.CLOUDRON) return homeDir;
     if (constants.TEST) return path.join(homeDir, '.cloudron_test');
     // cannot reach
@@ -25,10 +25,10 @@ exports = module.exports = {
 
     PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'),
     APPS_DATA_DIR: path.join(baseDir(), 'appsdata'),
+    BOX_DATA_DIR: path.join(baseDir(), 'boxdata'),  // box data dir is part of box backup
 
     ACME_CHALLENGES_DIR: path.join(baseDir(), 'platformdata/acme'),
     ADDON_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons'),
-    MAIL_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons/mail'),
     COLLECTD_APPCONFIG_DIR: path.join(baseDir(), 'platformdata/collectd/collectd.conf.d'),
     LOGROTATE_CONFIG_DIR: path.join(baseDir(), 'platformdata/logrotate.d'),
     NGINX_CONFIG_DIR: path.join(baseDir(), 'platformdata/nginx'),
@@ -49,7 +49,7 @@ exports = module.exports = {
     SFTP_PRIVATE_KEY_FILE: path.join(baseDir(), 'platformdata/sftp/ssh/ssh_host_rsa_key'),
     FIREWALL_BLOCKLIST_FILE: path.join(baseDir(), 'platformdata/firewall/blocklist.txt'),
 
-    BOX_DATA_DIR: path.join(baseDir(), 'boxdata/box'),
+    // this is not part of appdata because an icon may be set before install
     MAIL_DATA_DIR: path.join(baseDir(), 'boxdata/mail'),
 
     LOG_DIR: path.join(baseDir(), 'platformdata/logs'),

src/platform.js

@@ -10,6 +10,7 @@ exports = module.exports = {
 
 const apps = require('./apps.js'),
     assert = require('assert'),
+    async = require('async'),
     debug = require('debug')('box:platform'),
     fs = require('fs'),
     infra = require('./infra_version.js'),
@@ -20,15 +21,17 @@ const apps = require('./apps.js'),
     services = require('./services.js'),
     shell = require('./shell.js'),
     tasks = require('./tasks.js'),
-    volumes = require('./volumes.js'),
     _ = require('underscore');
 
-async function start(options) {
-    if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return;
+function start(options, callback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return callback();
 
     debug('initializing addon infrastructure');
 
-    let existingInfra = { version: 'none' };
+    var existingInfra = { version: 'none' };
     if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
         existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8'));
         if (!existingInfra) existingInfra = { version: 'corrupt' };
@@ -40,50 +43,57 @@ async function start(options) {
 
         onPlatformReady(false /* !infraChanged */);
 
-        return;
+        return callback();
     }
 
     debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version);
 
-    const error = locker.lock(locker.OP_PLATFORM_START);
-    if (error) throw error;
+    var error = locker.lock(locker.OP_PLATFORM_START);
+    if (error) return callback(error);
 
-    if (existingInfra.version !== infra.version) await removeAllContainers();
-    if (existingInfra.version === 'none') await volumes.mountAll(); // when restoring, mount all volumes
-    await markApps(existingInfra, options); // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
-    await services.startServices(existingInfra);
-    await fs.promises.writeFile(paths.INFRA_VERSION_FILE, JSON.stringify(infra, null, 4));
+    async.series([
+        (next) => { if (existingInfra.version !== infra.version) removeAllContainers(next); else next(); },
+        markApps.bind(null, existingInfra, options), // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
+        services.startServices.bind(null, existingInfra),
+        fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra, null, 4))
+    ], function (error) {
+        if (error) return callback(error);
 
-    locker.unlock(locker.OP_PLATFORM_START);
+        locker.unlock(locker.OP_PLATFORM_START);
 
-    onPlatformReady(true /* infraChanged */); // background
+        onPlatformReady(true /* infraChanged */);
+
+        callback();
+    });
 }
 
-async function stopAllTasks() {
-    await tasks.stopAllTasks();
+function stopAllTasks(callback) {
+    tasks.stopAllTasks(callback);
 }
 
-async function onPlatformReady(infraChanged) {
+function onPlatformReady(infraChanged) {
     debug(`onPlatformReady: platform is ready. infra changed: ${infraChanged}`);
     exports._isReady = true;
 
-    if (infraChanged) await pruneInfraImages();
+    let tasks = [ apps.schedulePendingTasks ];
+    if (infraChanged) tasks.push(pruneInfraImages);
 
-    await apps.schedulePendingTasks();
+    async.series(async.reflectAll(tasks), function (error, results) {
+        results.forEach((result, idx) => {
+            if (result.error) debug(`Startup task at index ${idx} failed: ${result.error.message}`);
+        });
+    });
 }
 
-async function pruneInfraImages() {
+function pruneInfraImages(callback) {
     debug('pruneInfraImages: checking existing images');
 
     // cannot blindly remove all unused images since redis image may not be used
     const images = infra.baseImages.concat(Object.keys(infra.images).map(function (addon) { return infra.images[addon]; }));
 
-    for (const image of images) {
+    async.eachSeries(images, function (image, iteratorCallback) {
         let output = safe.child_process.execSync(`docker images --digests ${image.repo} --format "{{.ID}} {{.Repository}}:{{.Tag}}@{{.Digest}}"`, { encoding: 'utf8' });
-        if (output === null) {
-            debug(`Failed to list images of ${image}`, safe.error);
-            throw safe.error;
-        }
+        if (output === null) return iteratorCallback(safe.error);
 
         let lines = output.trim().split('\n');
         for (let line of lines) {
@@ -95,27 +105,32 @@ async function pruneInfraImages() {
             let result = safe.child_process.execSync(`docker rmi ${parts[0]}`, { encoding: 'utf8' });
             if (result === null) debug(`Error removing image ${parts[0]}: ${safe.error.mesage}`);
         }
-    }
+
+        iteratorCallback();
+    }, callback);
 }
 
-async function removeAllContainers() {
+function removeAllContainers(callback) {
     debug('removeAllContainers: removing all containers for infra upgrade');
 
-    await shell.promises.exec('removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker stop');
-    await shell.promises.exec('removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker rm -f');
+    async.series([
+        shell.exec.bind(null, 'removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker stop'),
+        shell.exec.bind(null, 'removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker rm -f')
+    ], callback);
 }
 
-async function markApps(existingInfra, options) {
+function markApps(existingInfra, options, callback) {
     assert.strictEqual(typeof existingInfra, 'object');
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     if (existingInfra.version === 'none') { // cloudron is being restored from backup
         debug('markApps: restoring installed apps');
-        await apps.restoreInstalledApps(options);
+        apps.restoreInstalledApps(options, callback);
     } else if (existingInfra.version !== infra.version) {
         debug('markApps: reconfiguring installed apps');
         reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start
-        await apps.configureInstalledApps();
+        apps.configureInstalledApps(callback);
     } else {
         let changedAddons = [];
         if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) changedAddons.push('mysql');
@@ -126,9 +141,10 @@ async function markApps(existingInfra, options) {
         if (changedAddons.length) {
             // restart apps if docker image changes since the IP changes and any "persistent" connections fail
             debug(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`);
-            await apps.restartAppsUsingAddons(changedAddons);
+            apps.restartAppsUsingAddons(changedAddons, callback);
         } else {
             debug('markApps: apps are already uptodate');
+            callback();
         }
     }
 }

src/provision.js

@@ -8,8 +8,8 @@ exports = module.exports = {
 };
 
 const assert = require('assert'),
+    async = require('async'),
     backups = require('./backups.js'),
-    backuptask = require('./backuptask.js'),
     BoxError = require('./boxerror.js'),
     branding = require('./branding.js'),
     constants = require('./constants.js'),
@@ -29,8 +29,10 @@ const assert = require('assert'),
     tokens = require('./tokens.js'),
     _ = require('underscore');
 
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
 // we cannot use tasks since the tasks table gets overwritten when db is imported
-const gProvisionStatus = {
+let gProvisionStatus = {
     setup: {
         active: false,
         message: '',
@@ -46,150 +48,145 @@ const gProvisionStatus = {
 function setProgress(task, message, callback) {
     debug(`setProgress: ${task} - ${message}`);
     gProvisionStatus[task].message = message;
-    if (callback) callback();
+    callback();
 }
 
-async function unprovision() {
+function unprovision(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('unprovision');
+
     // TODO: also cancel any existing configureWebadmin task
-    await settings.setDashboardLocation('', '');
-    await mail.clearDomains();
-    await domains.clear();
+    async.series([
+        settings.setDashboardLocation.bind(null, '', ''),
+        mail.clearDomains,
+        domains.clear
+    ], callback);
 }
 
-async function setupTask(domain, auditSource) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof auditSource, 'object');
 
-    try {
-        await cloudron.setupDnsAndCert(constants.DASHBOARD_LOCATION, domain, auditSource, (progress) => setProgress('setup', progress.message));
-        await cloudron.setDashboardDomain(domain, auditSource);
-        setProgress('setup', 'Done'),
-        await eventlog.add(eventlog.ACTION_PROVISION, auditSource, {});
-    } catch (error) {
-        gProvisionStatus.setup.errorMessage = error ? error.message : '';
-    }
-
-    gProvisionStatus.setup.active = false;
-}
-
-async function setup(dnsConfig, sysinfoConfig, auditSource) {
+function setup(dnsConfig, sysinfoConfig, auditSource, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof sysinfoConfig, 'object');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (gProvisionStatus.setup.active || gProvisionStatus.restore.active) throw new BoxError(BoxError.BAD_STATE, 'Already setting up or restoring');
+    if (gProvisionStatus.setup.active || gProvisionStatus.restore.active) return callback(new BoxError(BoxError.BAD_STATE, 'Already setting up or restoring'));
 
     gProvisionStatus.setup = { active: true, errorMessage: '', message: 'Adding domain' };
 
-    try {
-        const activated = await users.isActivated();
-        if (activated) throw new BoxError(BoxError.CONFLICT, 'Already activated', { activate: true });
-
-        await unprovision();
-
-        const domain = dnsConfig.domain.toLowerCase();
-        const zoneName = dnsConfig.zoneName ? dnsConfig.zoneName : (tld.getDomain(domain) || domain);
-
-        debug(`setup: Setting up Cloudron with domain ${domain} and zone ${zoneName}`);
-
-        const data = {
-            zoneName: zoneName,
-            provider: dnsConfig.provider,
-            config: dnsConfig.config,
-            fallbackCertificate: dnsConfig.fallbackCertificate || null,
-            tlsConfig: dnsConfig.tlsConfig || { provider: 'letsencrypt-prod' },
-            dkimSelector: 'cloudron'
-        };
-
-        await settings.setMailLocation(domain, `${constants.DASHBOARD_LOCATION}.${domain}`); // default mail location. do this before we add the domain for upserting mail DNS
-        await domains.add(domain, data, auditSource);
-        await settings.setSysinfoConfig(sysinfoConfig);
-
-        safe(setupTask(domain, auditSource), { debug }); // now that args are validated run the task in the background
-    } catch (error) {
-        debug('setup: error', error);
+    function done(error) {
         gProvisionStatus.setup.active = false;
-        gProvisionStatus.setup.errorMessage = error.message;
-        throw error;
+        gProvisionStatus.setup.errorMessage = error ? error.message : '';
+        callback(error);
     }
+
+    users.isActivated(function (error, activated) {
+        if (error) return done(error);
+        if (activated) return done(new BoxError(BoxError.CONFLICT, 'Already activated', { activate: true }));
+
+        unprovision(function (error) {
+            if (error) return done(error);
+
+            const domain = dnsConfig.domain.toLowerCase();
+            const zoneName = dnsConfig.zoneName ? dnsConfig.zoneName : (tld.getDomain(domain) || domain);
+
+            debug(`provision: Setting up Cloudron with domain ${domain} and zone ${zoneName}`);
+
+            let data = {
+                zoneName: zoneName,
+                provider: dnsConfig.provider,
+                config: dnsConfig.config,
+                fallbackCertificate: dnsConfig.fallbackCertificate || null,
+                tlsConfig: dnsConfig.tlsConfig || { provider: 'letsencrypt-prod' },
+                dkimSelector: 'cloudron'
+            };
+
+            async.series([
+                settings.setMailLocation.bind(null, domain, `${constants.DASHBOARD_LOCATION}.${domain}`), // default mail location. do this before we add the domain for upserting mail DNS
+                domains.add.bind(null, domain, data, auditSource),
+                sysinfo.testConfig.bind(null, sysinfoConfig)
+            ], function (error) {
+                if (error) return done(error);
+
+                callback(); // now that args are validated run the task in the background
+
+                async.series([
+                    settings.setSysinfoConfig.bind(null, sysinfoConfig),
+                    cloudron.setupDnsAndCert.bind(null, constants.DASHBOARD_LOCATION, domain, auditSource, (progress) => setProgress('setup', progress.message, NOOP_CALLBACK)),
+                    cloudron.setDashboardDomain.bind(null, domain, auditSource),
+                    setProgress.bind(null, 'setup', 'Done'),
+                    async () => eventlog.add(eventlog.ACTION_PROVISION, auditSource, { })
+                ], function (error) {
+                    gProvisionStatus.setup.active = false;
+                    gProvisionStatus.setup.errorMessage = error ? error.message : '';
+                });
+            });
+        });
+    });
 }
 
-async function activate(username, password, email, displayName, ip, auditSource) {
+function activate(username, password, email, displayName, ip, auditSource, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof displayName, 'string');
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    debug(`activate: user: ${username} email:${email}`);
+    debug('activating user:%s email:%s', username, email);
 
-    const [error, ownerId] = await safe(users.createOwner(email, username, password, displayName, auditSource));
-    if (error && error.reason === BoxError.ALREADY_EXISTS) throw new BoxError(BoxError.CONFLICT, 'Already activated');
-    if (error) throw error;
+    users.createOwner(username, password, email, displayName, auditSource, async function (error, userObject) {
+        if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(new BoxError(BoxError.CONFLICT, 'Already activated'));
+        if (error) return callback(error);
 
-    const token = { clientId: tokens.ID_WEBADMIN, identifier: ownerId, expires: Date.now() + constants.DEFAULT_TOKEN_EXPIRATION_MSECS };
-    const result = await tokens.add(token);
+        const token = { clientId: tokens.ID_WEBADMIN, identifier: userObject.id, expires: Date.now() + constants.DEFAULT_TOKEN_EXPIRATION_MSECS };
+        let result;
+        [error, result] = await safe(tokens.add(token));
+        if (error) return callback(error);
 
-    eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, {});
+        eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
 
-    setImmediate(() => safe(cloudron.onActivated({}), { debug }));
+        callback(null, {
+            userId: userObject.id,
+            token: result.accessToken,
+            expires: result.expires
+        });
 
-    return {
-        userId: ownerId,
-        token: result.accessToken,
-        expires: result.expires
-    };
+        setImmediate(cloudron.onActivated.bind(null, {}, NOOP_CALLBACK)); // hack for now to not block the above http response
+    });
 }
 
-async function restoreTask(backupConfig, backupId, sysinfoConfig, options, auditSource) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof backupId, 'string');
-    assert.strictEqual(typeof sysinfoConfig, 'object');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof auditSource, 'object');
-
-    try {
-        setProgress('restore', 'Downloading backup');
-        await backuptask.restore(backupConfig, backupId, (progress) => setProgress('restore', progress.message));
-        await settings.setSysinfoConfig(sysinfoConfig);
-        await reverseProxy.restoreFallbackCertificates();
-
-        const dashboardDomain = settings.dashboardDomain(); // load this fresh from after the backup.restore
-        if (!options.skipDnsSetup) await cloudron.setupDnsAndCert(constants.DASHBOARD_LOCATION, dashboardDomain, auditSource, (progress) => setProgress('restore', progress.message));
-        await cloudron.setDashboardDomain(dashboardDomain, auditSource);
-        await settings.setBackupCredentials(backupConfig); // update just the credentials and not the policy and flags
-        await eventlog.add(eventlog.ACTION_RESTORE, auditSource, { backupId });
-
-        setImmediate(() => safe(cloudron.onRestored(options), { debug }));
-    } catch (error) {
-        gProvisionStatus.restore.errorMessage = error ? error.message : '';
-    }
-    gProvisionStatus.restore.active = false;
-}
-
-async function restore(backupConfig, backupId, version, sysinfoConfig, options, auditSource) {
+function restore(backupConfig, backupId, version, sysinfoConfig, options, auditSource, callback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof version, 'string');
     assert.strictEqual(typeof sysinfoConfig, 'object');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (!semver.valid(version)) throw new BoxError(BoxError.BAD_FIELD, 'version is not a valid semver', { field: 'version' });
-    if (constants.VERSION !== version) throw new BoxError(BoxError.BAD_STATE, `Run "cloudron-setup --version ${version}" on a fresh Ubuntu installation to restore from this backup`);
+    if (!semver.valid(version)) return callback(new BoxError(BoxError.BAD_FIELD, 'version is not a valid semver', { field: 'version' }));
+    if (constants.VERSION !== version) return callback(new BoxError(BoxError.BAD_STATE, `Run "cloudron-setup --version ${version}" on a fresh Ubuntu installation to restore from this backup`));
 
-    if (gProvisionStatus.setup.active || gProvisionStatus.restore.active) throw new BoxError(BoxError.BAD_STATE, 'Already setting up or restoring');
+    if (gProvisionStatus.setup.active || gProvisionStatus.restore.active) return callback(new BoxError(BoxError.BAD_STATE, 'Already setting up or restoring'));
 
     gProvisionStatus.restore = { active: true, errorMessage: '', message: 'Testing backup config' };
 
-    try {
-        const activated = await users.isActivated();
-        if (activated) throw new BoxError(BoxError.CONFLICT, 'Already activated. Restore with a fresh Cloudron installation.');
+    function done(error) {
+        gProvisionStatus.restore.active = false;
+        gProvisionStatus.restore.errorMessage = error ? error.message : '';
+        callback(error);
+    }
 
-        if (mounts.isMountProvider(backupConfig.provider)) {
+    users.isActivated(async function (error, activated) {
+        if (error) return done(error);
+        if (activated) return done(new BoxError(BoxError.CONFLICT, 'Already activated. Restore with a fresh Cloudron installation.'));
+
+        if (backups.isMountProvider(backupConfig.provider)) {
             error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions);
-            if (error) throw error;
+            if (error) return done(error);
 
             const newMount = {
                 name: 'backup',
@@ -198,44 +195,75 @@ async function restore(backupConfig, backupId, version, sysinfoConfig, options,
                 mountOptions: backupConfig.mountOptions
             };
 
-            await safe(mounts.tryAddMount(newMount, { timeout: 10 })); // 10 seconds
+            [error] = await safe(mounts.tryAddMount(newMount, { timeout: 10 })); // 10 seconds
+            if (error) return done(error);
         }
 
-        let error = await backups.testProviderConfig(backupConfig);
-        if (error) throw error;
+        backups.testProviderConfig(backupConfig, function (error) {
+            if (error) return done(error);
 
-        if ('password' in backupConfig) {
-            backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.password);
-            delete backupConfig.password;
-        } else {
-            backupConfig.encryption = null;
-        }
+            if ('password' in backupConfig) {
+                backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.password);
+                delete backupConfig.password;
+            } else {
+                backupConfig.encryption = null;
+            }
 
-        error = await sysinfo.testConfig(sysinfoConfig);
-        if (error) throw error;
+            sysinfo.testConfig(sysinfoConfig, function (error) {
+                if (error) return done(error);
 
-        safe(restoreTask(backupConfig, backupId, sysinfoConfig, options, auditSource), { debug }); // now that args are validated run the task in the background
-    } catch (error) {
-        gProvisionStatus.restore.active = false;
-        gProvisionStatus.restore.errorMessage = error ? error.message : '';
-        throw error;
-    }
+                debug(`restore: restoring from ${backupId} from provider ${backupConfig.provider} with format ${backupConfig.format}`);
+
+                callback(); // now that the fields are validated, continue task in the background
+
+                async.series([
+                    setProgress.bind(null, 'restore', 'Downloading backup'),
+                    backups.restore.bind(null, backupConfig, backupId, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)),
+                    settings.setSysinfoConfig.bind(null, sysinfoConfig),
+                    reverseProxy.restoreFallbackCertificates,
+                    (done) => {
+                        const dashboardDomain = settings.dashboardDomain(); // load this fresh from after the backup.restore
+                        async.series([
+                            (next) => {
+                                if (options.skipDnsSetup) return next();
+                                cloudron.setupDnsAndCert(constants.DASHBOARD_LOCATION, dashboardDomain, auditSource, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK), next);
+                            },
+                            cloudron.setDashboardDomain.bind(null, dashboardDomain, auditSource)
+                        ], done);
+                    },
+                    settings.setBackupCredentials.bind(null, backupConfig), // update just the credentials and not the policy and flags
+                    async () => eventlog.add(eventlog.ACTION_RESTORE, auditSource, { backupId }),
+                ], function (error) {
+                    gProvisionStatus.restore.active = false;
+                    gProvisionStatus.restore.errorMessage = error ? error.message : '';
+
+                    if (!error) cloudron.onActivated(options, NOOP_CALLBACK);
+                });
+            });
+        });
+    });
 }
 
-async function getStatus() {
-    const activated = await users.isActivated();
+function getStatus(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    const allSettings = await settings.list();
+    users.isActivated(function (error, activated) {
+        if (error) return callback(error);
 
-    return _.extend({
-        version: constants.VERSION,
-        apiServerOrigin: settings.apiServerOrigin(), // used by CaaS tool
-        webServerOrigin: settings.webServerOrigin(), // used by CaaS tool
-        cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
-        footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
-        adminFqdn: settings.dashboardDomain() ? settings.dashboardFqdn() : null,
-        language: allSettings[settings.LANGUAGE_KEY],
-        activated: activated,
-        provider: settings.provider() // used by setup wizard of marketplace images
-    }, gProvisionStatus);
+        settings.getAll(function (error, allSettings) {
+            if (error) return callback(error);
+
+            callback(null, _.extend({
+                version: constants.VERSION,
+                apiServerOrigin: settings.apiServerOrigin(), // used by CaaS tool
+                webServerOrigin: settings.webServerOrigin(), // used by CaaS tool
+                cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
+                footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
+                adminFqdn: settings.dashboardDomain() ? settings.dashboardFqdn() : null,
+                language: allSettings[settings.LANGUAGE_KEY],
+                activated: activated,
+                provider: settings.provider() // used by setup wizard of marketplace images
+            }, gProvisionStatus));
+        });
+    });
 }

src/proxyauth.js

@@ -26,8 +26,7 @@ const apps = require('./apps.js'),
     safe = require('safetydance'),
     speakeasy = require('speakeasy'),
     translation = require('./translation.js'),
-    users = require('./users.js'),
-    util = require('util');
+    users = require('./users.js');
 
 let gHttpServer = null;
 let TOKEN_SECRET = null;
@@ -49,56 +48,60 @@ function jwtVerify(req, res, next) {
     });
 }
 
-async function basicAuthVerify(req, res, next) {
+function basicAuthVerify(req, res, next) {
     const appId = req.headers['x-app-id'] || '';
     const credentials = basicAuth(req);
     if (!appId || !credentials) return next();
 
-    const [error, app] = await safe(apps.get(appId));
-    if (error) return next(new HttpError(503, error.message));
+    const api = credentials.name.indexOf('@') !== -1 ? users.verifyWithEmail : users.verifyWithUsername;
 
-    if (!app.manifest.addons.proxyAuth.basicAuth) return next();
+    apps.get(appId, function (error, app) {
+        if (error) return next(new HttpError(503, error.message));
 
-    const verifyFunc = credentials.name.indexOf('@') !== -1 ? users.verifyWithEmail : users.verifyWithUsername;
-    const [verifyError, user] = await safe(verifyFunc(credentials.name, credentials.pass, appId));
-    if (verifyError) return next(new HttpError(403, 'Invalid username or password' ));
+        if (!app.manifest.addons.proxyAuth.basicAuth) return next();
 
-    req.user = user;
-    next();
+        api(credentials.name, credentials.pass, appId, function (error, user) {
+            if (error) return next(new HttpError(403, 'Invalid username or password' ));
+
+            req.user = user;
+            next();
+        });
+    });
 }
 
-async function loginPage(req, res, next) {
+function loginPage(req, res, next) {
     const appId = req.headers['x-app-id'] || '';
     if (!appId) return next(new HttpError(503, 'Nginx misconfiguration'));
 
-    const [error, translationAssets] = await safe(translation.getTranslations());
-    if (error) return next(new HttpError(500, 'No translation found'));
+    translation.getTranslations(function (error, translationAssets) {
+        if (error) return next(new HttpError(500, 'No translation found'));
 
-    const raw = safe.fs.readFileSync(path.join(paths.DASHBOARD_DIR, 'templates/proxyauth-login.ejs'), 'utf8');
-    if (raw === null) return next(new HttpError(500, 'Login template not found'));
+        const raw = safe.fs.readFileSync(path.join(paths.DASHBOARD_DIR, 'templates/proxyauth-login.ejs'), 'utf8');
+        if (raw === null) return next(new HttpError(500, 'Login template not found'));
 
-    const translatedContent = translation.translate(raw, translationAssets.translations || {}, translationAssets.fallback || {});
-    let finalContent = '';
+        const translatedContent = translation.translate(raw, translationAssets.translations || {}, translationAssets.fallback || {});
+        var finalContent = '';
 
-    const [getError, app] = await safe(apps.get(appId));
-    if (getError) return next(new HttpError(503, getError.message));
+        apps.get(appId, function (error, app) {
+            if (error) return next(new HttpError(503, error.message));
 
-    const title = app.label || app.manifest.title;
+            const title = app.label || app.manifest.title;
 
-    const [iconError, iconBuffer] = await safe(apps.getIcon(app, {}));
-    if (iconError || !iconBuffer) return next(new HttpError(500, 'Icon rendering error'));
+            apps.getIcon(app, {}, function (error, iconBuffer) {
+                const icon = 'data:image/png;base64,' + iconBuffer.toString('base64');
 
-    const icon = 'data:image/png;base64,' + iconBuffer.toString('base64');
+                try {
+                    finalContent = ejs.render(translatedContent, { title, icon });
+                } catch (e) {
+                    debug('Error rendering proxyauth-login.ejs', e);
+                    return next(new HttpError(500, 'Login template error'));
+                }
 
-    try {
-        finalContent = ejs.render(translatedContent, { title, icon });
-    } catch (e) {
-        debug('Error rendering proxyauth-login.ejs', e);
-        return next(new HttpError(500, 'Login template error'));
-    }
-
-    res.set('Content-Type', 'text/html');
-    return res.send(finalContent);
+                res.set('Content-Type', 'text/html');
+                return res.send(finalContent);
+            });
+        });
+    });
 }
 
 // someday this can be more sophisticated and check for a real browser
@@ -107,7 +110,7 @@ function isBrowser(req) {
     if (!userAgent) return false;
 
     // https://github.com/docker/engine/blob/master/dockerversion/useragent.go#L18
-    return !userAgent.toLowerCase().includes('docker') && !userAgent.toLowerCase().includes('container');
+    return !userAgent.toLowerCase().includes('docker');
 }
 
 // called by nginx to authorize any protected route. this route must return only 2xx or 401/403 (http://nginx.org/en/docs/http/ngx_http_auth_request_module.html)
@@ -133,7 +136,7 @@ function auth(req, res, next) {
 }
 
 // endpoint called by login page, username and password posted as JSON body
-async function passwordAuth(req, res, next) {
+function passwordAuth(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
     const appId = req.headers['x-app-id'] || '';
@@ -145,53 +148,60 @@ async function passwordAuth(req, res, next) {
 
     const { username, password, totpToken } = req.body;
 
-    const verifyFunc = username.indexOf('@') !== -1 ? users.verifyWithEmail : users.verifyWithUsername;
-    const [error, user] = await safe(verifyFunc(username, password, appId));
-    if (error) return next(new HttpError(403, 'Invalid username or password' ));
+    const api = username.indexOf('@') !== -1 ? users.verifyWithEmail : users.verifyWithUsername;
 
-    if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
-        if (!totpToken) return next(new HttpError(403, 'A totpToken must be provided'));
+    api(username, password, appId, function (error, user) {
+        if (error) return next(new HttpError(403, 'Invalid username or password' ));
 
-        let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
-        if (!verified) return next(new HttpError(403, 'Invalid totpToken'));
-    }
+        if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
+            if (!totpToken) return next(new HttpError(403, 'A totpToken must be provided'));
 
-    req.user = user;
-    next();
-}
+            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
+            if (!verified) return next(new HttpError(403, 'Invalid totpToken'));
+        }
 
-async function authorize(req, res, next) {
-    const appId = req.headers['x-app-id'] || '';
-    if (!appId) return next(new HttpError(503, 'Nginx misconfiguration'));
-
-    const [error, app] = await safe(apps.get(appId));
-    if (error) return next(new HttpError(403, 'No such app' ));
-
-    if (!apps.canAccess(app, req.user)) return next(new HttpError(403, 'Forbidden' ));
-
-    const token = jwt.sign({ user: users.removePrivateFields(req.user) }, TOKEN_SECRET, { expiresIn: `${constants.DEFAULT_TOKEN_EXPIRATION_DAYS}d` });
-
-    res.cookie('authToken', token, {
-        httpOnly: true,
-        maxAge: constants.DEFAULT_TOKEN_EXPIRATION_MSECS,
-        secure: true
+        req.user = user;
+        next();
     });
-
-    res.redirect(302, '/');
 }
 
-async function logoutPage(req, res, next) {
+function authorize(req, res, next) {
     const appId = req.headers['x-app-id'] || '';
     if (!appId) return next(new HttpError(503, 'Nginx misconfiguration'));
 
-    const [error, app] = await safe(apps.get(appId));
-    if (error) return next(new HttpError(503, error.message));
+    apps.get(appId, function (error, app) {
+        if (error) return next(new HttpError(403, 'No such app' ));
 
-    res.clearCookie('authToken');
+        apps.hasAccessTo(app, req.user, function (error, hasAccess) {
+            if (error) return next(new HttpError(403, 'Forbidden' ));
+            if (!hasAccess) return next(new HttpError(403, 'Forbidden' ));
 
-    // when we have no path, redirect to the login page. we cannot redirect to '/' because browsers will immediately serve up the cached page
-    // if a path is set, we can assume '/' is a public page
-    res.redirect(302, app.manifest.addons.proxyAuth.path ? '/' : '/login');
+            const token = jwt.sign({ user: users.removePrivateFields(req.user) }, TOKEN_SECRET, { expiresIn: `${constants.DEFAULT_TOKEN_EXPIRATION_DAYS}d` });
+
+            res.cookie('authToken', token, {
+                httpOnly: true,
+                maxAge: constants.DEFAULT_TOKEN_EXPIRATION_MSECS,
+                secure: true
+            });
+
+            res.redirect(302, '/');
+        });
+    });
+}
+
+function logoutPage(req, res, next) {
+    const appId = req.headers['x-app-id'] || '';
+    if (!appId) return next(new HttpError(503, 'Nginx misconfiguration'));
+
+    apps.get(appId, function (error, app) {
+        if (error) return next(new HttpError(503, error.message));
+
+        res.clearCookie('authToken');
+
+        // when we have no path, redirect to the login page. we cannot redirect to '/' because browsers will immediately serve up the cached page
+        // if a path is set, we can assume '/' is a public page
+        res.redirect(302, app.manifest.addons.proxyAuth.path ? '/' : '/login');
+    });
 }
 
 function logout(req, res, next) {
@@ -201,17 +211,17 @@ function logout(req, res, next) {
 
 // provides webhooks for the auth wall
 function initializeAuthwallExpressSync() {
-    const app = express();
-    const httpServer = http.createServer(app);
+    let app = express();
+    let httpServer = http.createServer(app);
 
-    const QUERY_LIMIT = '1mb'; // max size for json and urlencoded queries
-    const REQUEST_TIMEOUT = 10000; // timeout for all requests
+    let QUERY_LIMIT = '1mb'; // max size for json and urlencoded queries
+    let REQUEST_TIMEOUT = 10000; // timeout for all requests
 
-    const json = middleware.json({ strict: true, limit: QUERY_LIMIT }); // application/json
+    let json = middleware.json({ strict: true, limit: QUERY_LIMIT }); // application/json
 
     if (process.env.BOX_ENV !== 'test') app.use(middleware.morgan('proxyauth :method :url :status :response-time ms - :res[content-length]', { immediate: false }));
 
-    const router = new express.Router();
+    var router = new express.Router();
     router.del = router.delete; // amend router.del for readability further on
 
     app
@@ -229,7 +239,8 @@ function initializeAuthwallExpressSync() {
     return httpServer;
 }
 
-async function start() {
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
     assert.strictEqual(gHttpServer, null, 'Authwall is already up and running.');
 
     if (!fs.existsSync(paths.PROXY_AUTH_TOKEN_SECRET_FILE)) {
@@ -241,12 +252,14 @@ async function start() {
 
     gHttpServer = initializeAuthwallExpressSync();
 
-    await util.promisify(gHttpServer.listen.bind(gHttpServer))(constants.AUTHWALL_PORT, '127.0.0.1');
+    gHttpServer.listen(constants.AUTHWALL_PORT, '127.0.0.1', callback);
 }
 
-async function stop() {
-    if (!gHttpServer) return;
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    await util.promisify(gHttpServer.close.bind(gHttpServer))();
+    if (!gHttpServer) return callback(null);
+
+    gHttpServer.close(callback);
     gHttpServer = null;
 }