more changes

chatwoot requires this

https://www.chatwoot.com/docs/self-hosted/deployment/caprover#api-requests-failing-with-you-need-to-sign-in-or-sign-up-before-continuing

They are apparently disabled by default since they conflict with some CGI headers:

https://stackoverflow.com/questions/22856136/why-do-http-servers-forbid-underscores-in-http-header-names
https://www.nginx.com/resources/wiki/start/topics/tutorials/config_pitfalls/?highlight=disappearing%20http%20headers#missing-disappearing-http-headers

.eslintrc.json

@@ -5,7 +5,7 @@
     },
     "extends": "eslint:recommended",
     "parserOptions": {
-        "ecmaVersion": 8
+        "ecmaVersion": 2020
     },
     "rules": {
         "indent": [

.gitignore

@@ -1,5 +1,6 @@
 node_modules/
 coverage/
+.nyc_output/
 webadmin/dist/
 installer/src/certs/server.key
 

.jshintrc

@@ -1,5 +1,5 @@
 {
     "node": true,
     "unused": true,
-    "esversion": 8
+    "esversion": 11
 }

CHANGES

@@ -2295,3 +2295,157 @@
 * mail: enable sieve extension editheader
 * mail: update solr to 8.9.0
 
+[6.3.4]
+* Fix issue where old nginx configs where not removed before upgrade
+
+[6.3.5]
+* Fix permission issues with sshfs
+* filemanager: reset selection if directory has changed
+* branding: fix error highlight with empty cloudron name
+* better text instead of "Cloudron in the wild"
+* Make sso login hint translatable
+* Give unread notifications a small left border
+* Fix issue where clicking update indicator opened app in new tab
+* Ensure notifications are only fetched and shown for at least admins
+* setupaccount: Show input field errors below input field
+* Set focus automatically for new alias or redirect
+* eventlog: fix issue where old events are not periodically removed
+* ssfs: fix chown
+
+[6.3.6]
+* Fix broken reboot button
+* app updated notification shown despite failure
+* Update translation for sso login information
+* Hide groups/tags/state filter in app listing for normal users
+* filemanager: Ensure breadcrumbs and hash are correctly updated on folder navigation
+* cloudron-setup: check if nginx/docker is already installed
+* Use the addresses of all available interfaces for port 53 binding
+* refresh config on appstore login
+* password reset: check 2fa when enabled
+
+[7.0.0]
+* Ubuntu 16 is not supported anymore
+* Do not use Gravatar as the default but only an option
+* redis: suppress password warning
+* setup UI: fix dark mode
+* wellknown: response to .wellknown/matrix/client
+* purpose field is not required anymore during appstore signup
+* sftp: fix symlink deletion
+* Show correct/new app version info in updated finished notification
+* Make new login email translatable
+* Hide ticket form if cloudron.io mail is not verified
+* Refactor code to use async/await
+* postgresql: bump shm size and disable parallel queries
+* update nodejs to 14.17.6
+* external ldap: If we detect a local user with the same username as found on LDAP/AD we map it
+* add basic eventlog for apps in app view
+* Enable sshfs/cifs/nfs in app import UI
+* Require password for fallback email change
+* Make password reset logic translatable
+* support: only verified email address can open support tickets
+* Logout users without 2FA when mandatory 2fa is enabled
+* notifications: better oom message for redis
+* Add way to impersonate users for presetup
+* mail: open up port 465 for mail submission (TLS)
+* Implement operator role for apps
+* sftp: normal users do not have SFTP access anymore. Use operator role instead
+* eventlog: add service rebuild/restart/configure events
+* upcloud: add object storage integration
+* Each app can now have a custom crontab
+* services: add recovery mode
+* postgresql: fix restore issue with long table names
+* recvmail: make the addon work again
+* mail: update solr to 8.10.0
+* mail: POP3 support
+* update docker to 20.10.7
+* volumes: add remount button
+* mail: add spam eventlog filter type
+* mail: configure dnsbl
+* mail: add duplication detection for lists
+* mail: add SRS for Sieve Forwarding
+
+[7.0.1]
+* Fix matrix wellKnown client migration
+
+[7.0.2]
+* mail: POP3 flag was not returned correctly
+* external ldap: fix crash preventing users from logging in
+* volumes: ensure we don't crash if mount status is unexpected
+* backups: set default backup memory limit to 800
+* users: allow admins to specify password recovery email
+* retry startup tasks on database error
+
+[7.0.3]
+* support: fix remoe support not working for 'root' user
+* Fix cog icon on app grid item hover for darkmode
+* Disable password reset and impersonate button for self user instead of hiding them
+* pop3: fix crash with auth of non-existent mailbox
+* mail: fix direction field in eventlog of deferred mails
+* mail: fix eventlog search
+* mail: save message-id in eventlog
+* backups: fix issue which resulted in incomplete backups when an app has backups disabled
+* restore: do not redirect until mail data has been restored
+* proxyauth: set viewport meta tag in login view
+
+[7.0.4]
+* Add password reveal button to login pages
+* appstore: fix crash if account already registered
+* Do not nuke all the logrotate configs on update
+* Remove unused httpPaths from manifest
+* cloudron-support: add option to reset cloudron.io account
+* Fix flicker in login page
+* Fix LE account key re-use issue in DO 1-click image
+* mail: add non-tls ports for recvmail addon
+* backups: fix issue where mail backups where not cleaned up
+* notifications: fix automatic app update notifications
+
+[7.1.0]
+* Add mail manager role
+* mailbox: app can be set as owner when recvmail addon enabled
+* domains: add well known config UI (for jitsi configuration)
+* Prefix email addon variables with CLOUDRON_EMAIL instead of CLOUDRON_MAIL
+* remove support for manifest version 1
+* Add option to enable/disable mailbox sharing
+* base image 3.2.0
+* Update node to 16.13.1
+* mongodb: update to 4.4
+* Add `upstreamVersion` to manifest
+* Add `logPaths` to manifest
+* Add cifs seal support for backup and volume mounts
+* add a way for admins to set username when profiles are locked
+* Add support for secondary domains
+* postgresql: enable postgis
+* remove nginx config of stopped apps
+* mail: use port25check.cloudron.io to check outbound port 25 connectivity
+* Add import/export of mailboxes and users
+* LDAP server can now be exposed
+* Update monaco-editor to 0.32.1
+* Update xterm.js to 4.17.0
+* Update docker to 20.10.12
+* IPv6 support
+
+[7.1.1]
+* Fix issue where dkimKey of a mail domain is sometimes null
+* firewall: add retry for xtables lock
+* redis: fix issue where protected mode was enabled with no password
+
+[7.1.2]
+* Fix crash in cloudron-firewall when ports are whitelisted
+* eventlog: add event for certificate cleanup
+* eventlog: log event for mailbox alias update
+* backups: fix incorrect mountpoint check with managed mounts
+
+[7.1.3]
+* Fix security issue where an admin can impersonate an owner
+* block list: can upload up to 2MB
+* dns: fix issue where link local address was picked up for ipv6
+* setup: ufw may not be installed
+* mysql: fix default collation of databases
+
+[7.1.4]
+* wildcard dns: fix handling of ENODATA
+* cloudflare: fix error handling
+* openvpn: ipv6 support
+* dyndns: fix issue where eventlog was getting filled with empty entries
+* mandatory 2fa: Fix typo in 2FA check
+

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2020 Cloudron UG
+Copyright (c) 2022 Cloudron UG
 
 With regard to the Cloudron Software:
 

baseimage/initializeBaseUbuntuImage.sh

@@ -32,6 +32,7 @@ debconf-set-selections <<< 'mysql-server mysql-server/root_password_again passwo
 
 gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
 mysql_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "mysql-server-8.0" || echo "mysql-server-5.7")
+ntpd_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "systemd-timesyncd" || echo "")
 apt-get -y install --no-install-recommends \
     acl \
     apparmor \
@@ -49,6 +50,7 @@ apt-get -y install --no-install-recommends \
     logrotate \
     $mysql_package \
     nfs-common \
+    $ntpd_package \
     openssh-server \
     pwgen \
     resolvconf \
@@ -74,7 +76,7 @@ apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends
 cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
 echo "==> Installing node.js"
-readonly node_version=14.15.4
+readonly node_version=16.13.1
 mkdir -p /usr/local/node-${node_version}
 curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-${node_version}
 ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
@@ -87,11 +89,11 @@ echo "==> Installing Docker"
 
 # create systemd drop-in file. if you channge options here, be sure to fixup installer.sh as well
 mkdir -p /etc/systemd/system/docker.service.d
-echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
+echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
 
 # there are 3 packages for docker - containerd, CLI and the daemon
-readonly docker_version=20.10.3
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
+readonly docker_version=20.10.12
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.9-1_amd64.deb" -o /tmp/containerd.deb
 curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
 curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 # apt install with install deps (as opposed to dpkg -i)
@@ -155,6 +157,17 @@ if [ -f "/etc/default/motd-news" ]; then
     sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
 fi
 
+# If privacy extensions are not disabled on server, this breaks IPv6 detection
+# https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756
+if [[ ! -f /etc/sysctl.d/99-cloudimg-ipv6.conf ]]; then
+    echo "==> Disable temporary address (IPv6)"
+    echo -e "# See https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756\nnet.ipv6.conf.all.use_tempaddr = 0\nnet.ipv6.conf.default.use_tempaddr = 0\n\n" > /etc/sysctl.d/99-cloudimg-ipv6.conf
+fi
+
+# Disable exim4 (1blu.de)
+systemctl stop exim4 || true
+systemctl disable exim4 || true
+
 # Disable bind for good measure (on online.net, kimsufi servers these are pre-installed)
 systemctl stop bind9 || true
 systemctl disable bind9 || true
@@ -171,8 +184,9 @@ systemctl disable postfix || true
 systemctl stop systemd-resolved || true
 systemctl disable systemd-resolved || true
 
-# ubuntu's default config for unbound does not work if ipv6 is disabled. this config is overwritten in start.sh
+# on vultr, ufw is enabled by default. we have our own firewall
+ufw disable || true
+
 # we need unbound to work as this is required for installer.sh to do any DNS requests
-ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
-echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: ${ip6}" > /etc/unbound/unbound.conf.d/cloudron-network.conf
+echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: no" > /etc/unbound/unbound.conf.d/cloudron-network.conf
 systemctl restart unbound

box.js

@@ -2,65 +2,76 @@
 
 'use strict';
 
-let async = require('async'),
-    dockerProxy = require('./src/dockerproxy.js'),
-    fs = require('fs'),
+const fs = require('fs'),
     ldap = require('./src/ldap.js'),
     paths = require('./src/paths.js'),
     proxyAuth = require('./src/proxyauth.js'),
-    server = require('./src/server.js');
+    safe = require('safetydance'),
+    server = require('./src/server.js'),
+    settings = require('./src/settings.js'),
+    userdirectory = require('./src/userdirectory.js');
 
-const NOOP_CALLBACK = function () { };
+let logFd;
 
-function setupLogging(callback) {
-    if (process.env.BOX_ENV === 'test') return callback();
+async function setupLogging() {
+    if (process.env.BOX_ENV === 'test') return;
 
-    const logfileStream = fs.createWriteStream(paths.BOX_LOG_FILE, { flags:'a' });
-    process.stdout.write = process.stderr.write = logfileStream.write.bind(logfileStream);
-
-    callback();
+    logFd = fs.openSync(paths.BOX_LOG_FILE, 'a');
+    // we used to write using a stream before but it caches internally and there is no way to flush it when things crash
+    process.stdout.write = process.stderr.write = function (...args) {
+        const callback = typeof args[args.length-1] === 'function' ? args.pop() : function () {}; // callback is required for fs.write
+        fs.write.apply(fs, [logFd, ...args, callback]);
+    };
 }
 
-async.series([
-    setupLogging,
-    server.start, // do this first since it also inits the database
-    proxyAuth.start,
-    ldap.start,
-    dockerProxy.start
-], function (error) {
-    if (error) {
-        console.log('Error starting server', error);
-        process.exit(1);
-    }
+// this is also used as the 'uncaughtException' handler which can only have synchronous functions
+function exitSync(status) {
+    if (status.error) fs.write(logFd, status.error.stack + '\n', function () {});
+    fs.fsyncSync(logFd);
+    fs.closeSync(logFd);
+    process.exit(status.code);
+}
 
-    // require those here so that logging handler is already setup
-    require('supererror');
+async function startServers() {
+    await setupLogging();
+    await server.start(); // do this first since it also inits the database
+    await proxyAuth.start();
+    await ldap.start();
+
+    const conf = await settings.getUserDirectoryConfig();
+    if (conf.enabled) await userdirectory.start();
+}
+
+async function main() {
+    const [error] = await safe(startServers());
+    if (error) return exitSync({ error: new Error(`Error starting server: ${JSON.stringify(error)}`), code: 1 });
+
+    // require this here so that logging handler is already setup
     const debug = require('debug')('box:box');
 
-    process.on('SIGINT', function () {
+    process.on('SIGINT', async function () {
         debug('Received SIGINT. Shutting down.');
 
-        proxyAuth.stop(NOOP_CALLBACK);
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
+        await proxyAuth.stop();
+        await server.stop();
+        await userdirectory.stop();
+        await ldap.stop();
         setTimeout(process.exit.bind(process), 3000);
     });
 
-    process.on('SIGTERM', function () {
+    process.on('SIGTERM', async function () {
         debug('Received SIGTERM. Shutting down.');
 
-        proxyAuth.stop(NOOP_CALLBACK);
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
+        await proxyAuth.stop();
+        await server.stop();
+        await userdirectory.stop();
+        await ldap.stop();
         setTimeout(process.exit.bind(process), 3000);
     });
 
-    process.on('uncaughtException', function (error) {
-        console.error((error && error.stack) ? error.stack : error);
-        setTimeout(process.exit.bind(process, 1), 3000);
-    });
+    process.on('uncaughtException', (error) => exitSync({ error, code: 1 }));
 
     console.log(`Cloudron is up and running. Logs are at ${paths.BOX_LOG_FILE}`); // this goes to journalctl
-});
+}
+
+main();

crashnotifierservice.js

@@ -2,27 +2,21 @@
 
 'use strict';
 
-var database = require('./src/database.js');
+const database = require('./src/database.js');
 
-var crashNotifier = require('./src/crashnotifier.js');
+const crashNotifier = require('./src/crashnotifier.js');
 
 // This is triggered by systemd with the crashed unit name as argument
-function main() {
+async function main() {
     if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <unitName>');
 
-    var unitName = process.argv[2];
+    const unitName = process.argv[2];
     console.log('Started crash notifier for', unitName);
 
     // eventlog api needs the db
-    database.initialize(function (error) {
-        if (error) return console.error('Cannot connect to database. Unable to send crash log.', error);
+    await database.initialize();
 
-        crashNotifier.sendFailureLogs(unitName, function (error) {
-            if (error) console.error(error);
-
-            process.exit();
-        });
-    });
+    await crashNotifier.sendFailureLogs(unitName);
 }
 
 main();

migrations/20210707154724-users-migrate-avatar.js

@@ -0,0 +1,13 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE users SET avatar="gravatar" WHERE avatar IS NULL', function (error) {
+        if (error) return callback(error);
+
+        db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB NOT NULL', callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB', callback);
+};

migrations/20210729191138-domains-update-wellKnown-matrix-client.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * from domains', [], function (error, results) {
+        if (error) return callback(error);
+
+        async.eachSeries(results, function (r, iteratorDone) {
+            if (!r.wellKnownJson) return iteratorDone();
+
+            const wellKnown = safe.JSON.parse(r.wellKnownJson);
+            if (!wellKnown || !wellKnown['matrix/server']) return iteratorDone();
+            const matrixHostname = JSON.parse(wellKnown['matrix/server'])['m.server'];
+
+            wellKnown['matrix/client'] = JSON.stringify({
+                'm.homeserver': {
+                    'base_url': 'https://' + matrixHostname
+                }
+            });
+
+            db.runSql('UPDATE domains SET wellKnownJson=? WHERE domain=?', [ JSON.stringify(wellKnown), r.domain ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210809203855-appAddonConfigs-alter-value.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appAddonConfigs MODIFY value TEXT NOT NULL', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appAddonConfigs MODIFY value VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210820172800-users-alter-loginLocationsJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY loginLocationsJson MEDIUMTEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY loginLocationsJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210921022843-apps-add-operatorsJson.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN operatorsJson TEXT', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN operatorsJson', callback);
+};

migrations/20210927211658-apps-add-crontab.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN crontab TEXT', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN crontab', callback);
+};

migrations/20211001095256-users-add-inviteToken.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN inviteToken VARCHAR(128) DEFAULT ""', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN inviteToken', callback);
+};

migrations/20211001160903-apps-add-inbox.js

@@ -0,0 +1,19 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN enableInbox BOOLEAN DEFAULT 0'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxName VARCHAR(128)'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxDomain VARCHAR(128)'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN enableInbox'),
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxName'),
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxDomain'),
+    ], callback);
+};

migrations/20211006200150-domains-ensure-fallbackCertificate.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const async = require('async'),
+    reverseProxy = require('../src/reverseproxy.js'),
+    safe = require('safetydance');
+
+const NGINX_CERT_DIR = '/home/yellowtent/platformdata/nginx/cert';
+
+// ensure fallbackCertificate of domains are present in database and the cert dir. it seems a bad migration lost them.
+// https://forum.cloudron.io/topic/5683/data-argument-must-be-of-type-received-null-error-during-restore-process
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM domains', [ ], function (error, domains) {
+        if (error) return callback(error);
+
+        // this code is br0ken since async 3.x since async functions won't get iteratorDone anymore
+        // no point fixing this migration though since it won't run again in old cloudrons. and in new cloudron domains will be empty
+        async.eachSeries(domains, async function (domain, iteratorDone) {
+            let fallbackCertificate = safe.JSON.parse(domain.fallbackCertificateJson);
+            if (!fallbackCertificate || !fallbackCertificate.cert || !fallbackCertificate.key) {
+                let error;
+                [error, fallbackCertificate] = await safe(reverseProxy.generateFallbackCertificate(domain.domain));
+                if (error) return iteratorDone(error);
+            }
+
+            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.cert`, fallbackCertificate.cert, 'utf8')) return iteratorDone(safe.error);
+            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.key`, fallbackCertificate.key, 'utf8')) return iteratorDone(safe.error);
+
+            db.runSql('UPDATE domains SET fallbackCertificateJson=? WHERE domain=?', [ JSON.stringify(fallbackCertificate), domain.domain ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20211008171126-mailboxes-add-enablePop3.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes ADD COLUMN enablePop3 BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN enablePop3', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20211012020255-mail-add-dkimKeyJson.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    path = require('path'),
+    safe = require('safetydance');
+
+const MAIL_DATA_DIR = '/home/yellowtent/boxdata/mail';
+const DKIM_DIR = `${MAIL_DATA_DIR}/dkim`;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mail ADD COLUMN dkimKeyJson MEDIUMTEXT', function (error) {
+        if (error) return callback(error);
+
+        fs.readdir(DKIM_DIR, function (error, filenames) {
+            if (error && error.code === 'ENOENT') return callback();
+            if (error) return callback(error);
+
+            async.eachSeries(filenames, function (filename, iteratorCallback) {
+                const domain = filename;
+                const publicKey = safe.fs.readFileSync(path.join(DKIM_DIR, domain, 'public'), 'utf8');
+                const privateKey = safe.fs.readFileSync(path.join(DKIM_DIR, domain, 'private'), 'utf8');
+                if (!publicKey || !privateKey) return iteratorCallback();
+
+                const dkimKey = {
+                    publicKey,
+                    privateKey
+                };
+
+                db.runSql('UPDATE mail SET dkimKeyJson=? WHERE domain=?', [ JSON.stringify(dkimKey), domain ], iteratorCallback);
+            }, function (error) {
+                if (error) return callback(error);
+
+                fs.rmdir(DKIM_DIR, { recursive: true }, callback);
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.run(db, 'ALTER TABLE mail DROP COLUMN dkimKeyJson')
+    ], callback);
+};

migrations/20211117070204-blobs-delete-dhparams.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM blobs WHERE id=?', [ 'dhparams' ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20211117201805-eventlog-rename-source-to-sourceJson.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE source sourceJson TEXT', []),
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE data dataJson TEXT', []),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE sourceJson source TEXT', []),
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE dataJson data TEXT', []),
+    ], callback);
+};

migrations/20220107015249-settings-sysinfo-ip-to-ipv4.js

@@ -0,0 +1,17 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('SELECT value FROM settings WHERE name=?', [ 'sysinfo_config' ], function (error, result) {
+        if (error || result.length === 0) return callback(error);
+        const sysinfoConfig = JSON.parse(result[0].value);
+        if (sysinfoConfig.provider !== 'fixed' || !sysinfoConfig.ip) return callback();
+        sysinfoConfig.ipv4 = sysinfoConfig.ip;
+        delete sysinfoConfig.ip;
+
+        db.runSql('REPLACE INTO settings (name, value) VALUES(?, ?)', [ 'sysinfo_config', JSON.stringify(sysinfoConfig) ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220113223425-settings-rename-directory-config-to-profile-config.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'directory_config', 'profile_config' ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'profile_config', 'directory_config' ], callback);
+};

migrations/20220115070646-subdomains-add-environmentVariable.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains ADD COLUMN environmentVariable VARCHAR(128)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains DROP COLUMN environmentVariable', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20220202011148-blobs-migrate-token-secret.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const safe = require('safetydance');
+
+const PROXY_AUTH_TOKEN_SECRET_FILE = '/home/yellowtent/platformdata/proxy-auth-token-secret';
+
+exports.up = function (db, callback) {
+    const token = safe.fs.readFileSync(PROXY_AUTH_TOKEN_SECRET_FILE);
+    if (!token) return callback();
+    db.runSql('INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'proxy_auth_token_secret', token ], function (error) {
+        if (error) return callback(error);
+        safe.fs.unlinkSync(PROXY_AUTH_TOKEN_SECRET_FILE);
+        callback();
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220207214956-rename-subdomains-to-locations.js

@@ -0,0 +1,12 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('RENAME TABLE subdomains TO locations', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220219220751-mail-ensure-dkimKey.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const async = require('async'),
+    mail = require('../src/mail.js'),
+    safe = require('safetydance'),
+    util = require('util');
+
+// it seems some mail domains do not have dkimKey in the database for some reason because of some previous bad migration
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM mail', [ ], function (error, mailDomains) {
+        if (error) return callback(error);
+
+        async.eachSeries(mailDomains, function (mailDomain, iteratorDone) {
+            let dkimKey = safe.JSON.parse(mailDomain.dkimKeyJson);
+            if (dkimKey && dkimKey.publicKey && dkimKey.privateKey) return iteratorDone();
+            console.log(`${mailDomain.domain} has no dkim key in the database. generating a new one`);
+            util.callbackify(mail.generateDkimKey)(function (error, dkimKey) {
+                if (error) return iteratorDone(error);
+                db.runSql('UPDATE mail SET dkimKeyJson=? WHERE domain=?', [ JSON.stringify(dkimKey), mailDomain.domain ], iteratorDone);
+            });
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/schema.sql

@@ -28,11 +28,12 @@ CREATE TABLE IF NOT EXISTS users(
     twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
     source VARCHAR(128) DEFAULT "",
     role VARCHAR(32),
+    inviteToken VARCHAR(128) DEFAULT "",
     resetToken VARCHAR(128) DEFAULT "",
     resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     active BOOLEAN DEFAULT 1,
-    avatar MEDIUMBLOB,
-    locationJson TEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
+    avatar MEDIUMBLOB NOT NULL,
+    loginLocationsJson MEDIUMTEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
 
     INDEX creationTime_index (creationTime),
     PRIMARY KEY(id));
@@ -85,6 +86,9 @@ CREATE TABLE IF NOT EXISTS apps(
     enableMailbox BOOLEAN DEFAULT 1, // whether sendmail addon is enabled
     mailboxName VARCHAR(128), // mailbox of this app
     mailboxDomain VARCHAR(128), // mailbox domain of this apps
+    enableInbox BOOLEAN DEFAULT 0, // whether recvmail addon is enabled
+    inboxName VARCHAR(128), // mailbox of this app
+    inboxDomain VARCHAR(128), // mailbox domain of this apps
     label VARCHAR(128),     // display name
     tagsJson VARCHAR(2048), // array of tags
     dataDir VARCHAR(256) UNIQUE,
@@ -94,6 +98,7 @@ CREATE TABLE IF NOT EXISTS apps(
     containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair'
     appStoreIcon MEDIUMBLOB,
     icon MEDIUMBLOB,
+    crontab TEXT,
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
@@ -117,7 +122,7 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
     appId VARCHAR(128) NOT NULL,
     addonId VARCHAR(32) NOT NULL,
     name VARCHAR(128) NOT NULL,
-    value VARCHAR(512) NOT NULL,
+    value TEXT NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id));
 
 CREATE TABLE IF NOT EXISTS appEnvVars(
@@ -145,8 +150,8 @@ CREATE TABLE IF NOT EXISTS backups(
 CREATE TABLE IF NOT EXISTS eventlog(
     id VARCHAR(128) NOT NULL,
     action VARCHAR(128) NOT NULL,
-    source TEXT, /* { userId, username, ip }. userId can be null for cron,sysadmin */
-    data TEXT, /* free flowing json based on action */
+    sourceJson TEXT, /* { userId, username, ip }. userId can be null for cron,sysadmin */
+    dataJson TEXT, /* free flowing json based on action */
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
 
     INDEX creationTime_index (creationTime),
@@ -176,6 +181,7 @@ CREATE TABLE IF NOT EXISTS mail(
     relayJson TEXT,
     bannerJson TEXT,
 
+    dkimKeyJson MEDIUMTEXT,
     dkimSelector VARCHAR(128) NOT NULL DEFAULT "cloudron",
 
     FOREIGN KEY(domain) REFERENCES domains(domain),
@@ -202,16 +208,18 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     domain VARCHAR(128),
     active BOOLEAN DEFAULT 1,
+    enablePop3 BOOLEAN DEFAULT 0,
 
     FOREIGN KEY(domain) REFERENCES mail(domain),
     FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
     UNIQUE (name, domain));
 
-CREATE TABLE IF NOT EXISTS subdomains(
+CREATE TABLE IF NOT EXISTS locations(
     appId VARCHAR(128) NOT NULL,
     domain VARCHAR(128) NOT NULL,
     subdomain VARCHAR(128) NOT NULL,
-    type VARCHAR(128) NOT NULL, /* primary or redirect */
+    type VARCHAR(128) NOT NULL, /* primary, secondary, redirect, alias */
+    environmentVariable VARCHAR(128), /* only set for secondary */
 
     certificateJson MEDIUMTEXT,
 
@@ -222,6 +230,7 @@ CREATE TABLE IF NOT EXISTS subdomains(
 CREATE TABLE IF NOT EXISTS tasks(
     id int NOT NULL AUTO_INCREMENT,
     type VARCHAR(32) NOT NULL,
+    argsJson TEXT,
     percent INTEGER DEFAULT 0,
     message TEXT,
     errorJson TEXT,
@@ -278,7 +287,7 @@ CREATE TABLE IF NOT EXISTS appMounts(
 
 CREATE TABLE IF NOT EXISTS blobs(
     id VARCHAR(128) NOT NULL UNIQUE,
-    value TEXT,
+    value MEDIUMBLOB,
     PRIMARY KEY(id));
 
 CHARACTER SET utf8 COLLATE utf8_bin;

package.json

@@ -11,77 +11,73 @@
     "url": "https://git.cloudron.io/cloudron/box.git"
   },
   "dependencies": {
-    "@google-cloud/dns": "^2.1.0",
-    "@google-cloud/storage": "^5.8.5",
+    "@google-cloud/dns": "^2.2.4",
+    "@google-cloud/storage": "^5.16.1",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^3.2.0",
-    "aws-sdk": "^2.906.0",
+    "async": "^3.2.2",
+    "aws-sdk": "^2.1053.0",
     "basic-auth": "^2.0.1",
-    "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.10.2",
+    "body-parser": "^1.19.1",
+    "cloudron-manifestformat": "^5.15.0",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.1.1",
     "connect-timeout": "^1.9.0",
-    "cookie-parser": "^1.4.5",
-    "cookie-session": "^1.4.0",
+    "cookie-parser": "^1.4.6",
+    "cookie-session": "^2.0.0",
     "cron": "^1.8.2",
-    "db-migrate": "^0.11.12",
-    "db-migrate-mysql": "^2.1.2",
-    "debug": "^4.3.1",
+    "db-migrate": "^0.11.13",
+    "db-migrate-mysql": "^2.2.0",
+    "debug": "^4.3.3",
     "delay": "^5.0.0",
-    "dockerode": "^3.3.0",
+    "dockerode": "^3.3.1",
     "ejs": "^3.1.6",
-    "ejs-cli": "^2.2.1",
-    "express": "^4.17.1",
-    "ipaddr.js": "^2.0.0",
+    "ejs-cli": "^2.2.3",
+    "express": "^4.17.2",
+    "ipaddr.js": "^2.0.1",
     "js-yaml": "^4.1.0",
     "json": "^11.0.0",
     "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.2.4",
+    "ldapjs": "^2.3.2",
     "lodash": "^4.17.21",
     "lodash.chunk": "^4.2.0",
-    "mime": "^2.5.2",
     "moment": "^2.29.1",
-    "moment-timezone": "^0.5.33",
+    "moment-timezone": "^0.5.34",
     "morgan": "^1.10.0",
     "multiparty": "^4.2.2",
-    "mustache-express": "^1.3.0",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.6.0",
+    "nodemailer": "^6.7.2",
     "nodemailer-smtp-transport": "^2.7.4",
     "once": "^1.4.0",
     "pretty-bytes": "^5.6.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
-    "qrcode": "^1.4.4",
+    "qrcode": "^1.5.0",
     "readdirp": "^3.6.0",
-    "request": "^2.88.2",
-    "rimraf": "^3.0.2",
     "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^2.0.1",
+    "safetydance": "^2.2.0",
     "semver": "^7.3.5",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^6.1.0",
-    "supererror": "^0.7.2",
+    "superagent": "^7.0.1",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
-    "ua-parser-js": "^0.7.28",
-    "underscore": "^1.13.1",
+    "ua-parser-js": "^1.0.2",
+    "underscore": "^1.13.2",
     "uuid": "^8.3.2",
-    "validator": "^13.6.0",
-    "ws": "^7.4.5",
+    "validator": "^13.7.0",
+    "ws": "^8.4.0",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.4.1",
-    "js2xmlparser": "^4.0.1",
-    "mocha": "^8.4.0",
+    "js2xmlparser": "^4.0.2",
+    "mocha": "^9.1.3",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^13.0.11",
-    "node-sass": "^6.0.0",
+    "nock": "^13.2.1",
+    "node-sass": "^7.0.1",
+    "nyc": "^15.1.0",
     "recursive-readdir": "^2.2.2"
   },
   "scripts": {

runTests

@@ -22,7 +22,7 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p boxdata/box boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
 mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update
 sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
 
@@ -37,14 +37,15 @@ openssl req -x509 -newkey rsa:2048 -keyout platformdata/nginx/cert/host.key -out
 # clear out any containers if FAST is unset
 if [[ -z ${FAST+x} ]]; then
     echo "=> Delete all docker containers first"
-    docker ps -qa | xargs --no-run-if-empty docker rm -f
+    docker ps -qa --filter "label=isCloudronManaged" | xargs --no-run-if-empty docker rm -f
+    docker rm -f mysql-server
     echo "==> To skip this run with: FAST=1 ./runTests"
 else
     echo "==> WARNING!! Skipping docker container cleanup, the database might not be pristine!"
 fi
 
 # create docker network (while the infra code does this, most tests skip infra setup)
-docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
+docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 --gateway 172.18.0.1 cloudron --ipv6 --subnet=fd00:c107:d509::/64 || true
 
 # create the same mysql server version to test with
 OUT=`docker inspect mysql-server` || true
@@ -62,6 +63,9 @@ while ! mysqladmin ping -h"${MYSQL_IP}" --silent; do
     sleep 1
 done
 
+echo "=> Ensure local base image"
+docker pull cloudron/base:3.0.0@sha256:455c70428723e3a823198c57472785437eb6eab082e79b3ff04ea584faf46e92
+
 echo "=> Create iptables blocklist"
 sudo ipset create cloudron_blocklist hash:net || true
 
@@ -75,10 +79,15 @@ echo "=> Run database migrations"
 cd "${source_dir}"
 BOX_ENV=test DATABASE_URL=mysql://root:password@${MYSQL_IP}/box node_modules/.bin/db-migrate up
 
-echo "=> Run tests with mocha"
 TESTS=${DEFAULT_TESTS}
 if [[ $# -gt 0 ]]; then
     TESTS="$*"
 fi
 
-BOX_ENV=test ./node_modules/mocha/bin/_mocha --bail --no-timeouts --exit -R spec ${TESTS}
+if [[ -z ${COVERAGE+x} ]]; then
+    echo "=> Run tests with mocha"
+    BOX_ENV=test ./node_modules/.bin/mocha --bail --no-timeouts --exit -R spec ${TESTS}
+else
+    echo "=> Run tests with mocha and coverage"
+    BOX_ENV=test ./node_modules/.bin/nyc --reporter=html  ./node_modules/.bin/mocha --no-timeouts --exit -R spec ${TESTS}
+fi

scripts/cloudron-setup

@@ -41,6 +41,11 @@ if [[ "${disk_size_gb}" -lt "${MINIMUM_DISK_SIZE_GB}" ]]; then
     exit 1
 fi
 
+if [[ "$(uname -m)" != "x86_64" ]]; then
+    echo "Error: Cloudron only supports amd64/x86_64"
+    exit 1
+fi
+
 # do not use is-active in case box service is down and user attempts to re-install
 if systemctl cat box.service >/dev/null 2>&1; then
     echo "Error: Cloudron is already installed. To reinstall, start afresh"
@@ -99,6 +104,11 @@ if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubu
     exit 1
 fi
 
+if which nginx >/dev/null || which docker >/dev/null || which node > /dev/null; then
+    echo "Error: Some packages like nginx/docker/nodejs are already installed. Cloudron requires specific versions of these packages and will install them as part of it's installation. Please start with a fresh Ubuntu install and run this script again." > /dev/stderr
+    exit 1
+fi
+
 # Install MOTD file for stack script style installations. this is removed by the trap exit handler. Heredoc quotes prevents parameter expansion
 cat > /etc/update-motd.d/91-cloudron-install-in-progress <<'EOF'
 #!/bin/bash
@@ -163,7 +173,7 @@ if ! sourceTarballUrl=$(echo "${releaseJson}" | python3 -c 'import json,sys;obj=
     exit 1
 fi
 
-echo "=> Downloading version ${version} ..."
+echo "=> Downloading Cloudron version ${version} ..."
 box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)
 
 if ! $curl -sL "${sourceTarballUrl}" | tar -zxf - -C "${box_src_tmp_dir}"; then
@@ -182,7 +192,7 @@ if [[ "${initBaseImage}" == "true" ]]; then
 fi
 
 # The provider flag is still used for marketplace images
-echo "=> Installing version ${version} (this takes some time) ..."
+echo "=> Installing Cloudron version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
 echo "${provider}" > /etc/cloudron/PROVIDER
 [[ ! -z "${setupToken}" ]] && echo "${setupToken}" > /etc/cloudron/SETUP_TOKEN
@@ -204,7 +214,7 @@ while true; do
     sleep 10
 done
 
-if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
     ip='<IP>'
 fi
 if [[ -z "${setupToken}" ]]; then

scripts/cloudron-support

@@ -10,12 +10,13 @@ OUT="/tmp/cloudron-support.log"
 LINE="\n========================================================\n"
 CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQVilclYAIu+ioDp/sgzzFz6YU0hPcRYY7ze/LiF/lC7uQqK062O54BFXTvQ3ehtFZCx3bNckjlT2e6gB8Qq07OM66De4/S/g+HJW4TReY2ppSPMVNag0TNGxDzVH8pPHOysAm33LqT2b6L/wEXwC6zWFXhOhHjcMqXvi8Ejaj20H1HVVcf/j8qs5Thkp9nAaFTgQTPu8pgwD8wDeYX1hc9d0PYGesTADvo6HF4hLEoEnefLw7PaStEbzk2fD3j7/g5r5HcgQQXBe74xYZ/1gWOX2pFNuRYOBSEIrNfJEjFJsqk3NR1+ZoMGK7j+AZBR4k0xbrmncQLcQzl6MMDzkp support@cloudron.io"
 HELP_MESSAGE="
-This script collects diagnostic information to help debug server related issues
+This script collects diagnostic information to help debug server related issues.
 
  Options:
-   --owner-login   Login as owner
-   --enable-ssh    Enable SSH access for the Cloudron support team
-   --help          Show this message
+   --owner-login             Login as owner
+   --enable-ssh              Enable SSH access for the Cloudron support team
+   --reset-appstore-account  Reset associated cloudron.io account
+   --help                    Show this message
 "
 
 # We require root
@@ -26,7 +27,7 @@ fi
 
 enableSSH="false"
 
-args=$(getopt -o "" -l "help,enable-ssh,admin-login,owner-login" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,enable-ssh,admin-login,owner-login,reset-appstore-account" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
@@ -39,11 +40,18 @@ while true; do
     --owner-login)
         admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY creationTime LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
-        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/nul)
-        ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
-        printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > "${ghost_file}"
-        chown yellowtent:yellowtent "${ghost_file}" && chmod o-r,g-r "${ghost_file}"
-        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
+        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/null)
+        mysql -NB -uroot -ppassword -e "INSERT INTO box.settings (name, value) VALUES ('ghosts_config', '{\"${admin_username}\":\"${admin_password}\"}') ON DUPLICATE KEY UPDATE name='ghosts_config', value='{\"${admin_username}\":\"${admin_password}\"}'" 2>/dev/null
+        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once."
+        exit 0
+        ;;
+    --reset-appstore-account)
+        echo -e "This will reset the Cloudron.io account associated with this Cloudron. Once reset, you can re-login with a different account in the Cloudron Dashboard. See https://docs.cloudron.io/appstore/#change-account for more information.\n"
+        read -e -p "Reset the Cloudron.io account? [y/N] " choice
+        [[ "$choice" != [Yy]* ]] && exit 1
+        mysql -uroot -ppassword -e "DELETE FROM box.settings WHERE name='cloudron_token';" 2>/dev/null
+        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/null)
+        echo "Account reset. Please re-login at https://${dashboard_domain}/#/appstore"
         exit 0
         ;;
     --) break;;
@@ -143,8 +151,7 @@ if [[ "${enableSSH}" == "true" ]]; then
 fi
 
 echo -n "Uploading information..."
-# for some reason not using $(cat $OUT) will not contain newlines!?
-paste_key=$(curl -X POST ${PASTEBIN}/documents --silent -d "$(cat $OUT)" | python3 -c "import sys, json; print(json.load(sys.stdin)['key'])")
+paste_key=$(curl -X POST ${PASTEBIN}/documents --silent --data-binary "@$OUT" | python3 -c "import sys, json; print(json.load(sys.stdin)['key'])")
 echo "Done"
 
 echo ""

scripts/createReleaseTarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v14.15.4" ]]; then
-    echo "This script requires node 14.15.4"
+if [[ "$(node --version)" != "v16.13.1" ]]; then
+    echo "This script requires node 16.13.1"
     exit 1
 fi
 

scripts/installer.sh

@@ -73,20 +73,16 @@ log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION
 
 log "updating docker"
 
-readonly docker_version=20.10.3
+readonly docker_version=20.10.12
 if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
     # there are 3 packages for docker - containerd, CLI and the daemon
-    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.9-1_amd64.deb" -o /tmp/containerd.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
+    log "installing docker"
     prepare_apt_once
-
-    while ! apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb; do
-        log "Failed to install docker. Retry"
-        sleep 1
-    done
-
+    apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
     rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
 fi
 
@@ -115,13 +111,13 @@ if ! which sshfs; then
 fi
 
 log "updating node"
-readonly node_version=14.15.4
+readonly node_version=16.13.1
 if [[ "$(node --version)" != "v${node_version}" ]]; then
     mkdir -p /usr/local/node-${node_version}
     $curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-${node_version}
     ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
     ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-10.18.1
+    rm -rf /usr/local/node-14.17.6
 fi
 
 # this is here (and not in updater.js) because rebuild requires the above node
@@ -159,7 +155,7 @@ done
 log "update cloudron-syslog"
 CLOUDRON_SYSLOG_DIR=/usr/local/cloudron-syslog
 CLOUDRON_SYSLOG="${CLOUDRON_SYSLOG_DIR}/bin/cloudron-syslog"
-CLOUDRON_SYSLOG_VERSION="1.0.3"
+CLOUDRON_SYSLOG_VERSION="1.1.0"
 while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLOUDRON_SYSLOG_VERSION} ]]; do
     rm -rf "${CLOUDRON_SYSLOG_DIR}"
     mkdir -p "${CLOUDRON_SYSLOG_DIR}"

setup/start.sh

@@ -16,7 +16,7 @@ readonly HOME_DIR="/home/${USER}"
 readonly BOX_SRC_DIR="${HOME_DIR}/box"
 readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata"
 readonly APPS_DATA_DIR="${HOME_DIR}/appsdata"
-readonly BOX_DATA_DIR="${HOME_DIR}/boxdata"
+readonly BOX_DATA_DIR="${HOME_DIR}/boxdata/box"
 readonly MAIL_DATA_DIR="${HOME_DIR}/boxdata/mail"
 
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -37,12 +37,17 @@ systemctl enable apparmor
 systemctl restart apparmor
 
 usermod ${USER} -a -G docker
-# unbound (which starts after box code) relies on this interface to exist. dockerproxy also relies on this.
-docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
+
+if ! grep -q ip6tables /etc/systemd/system/docker.service.d/cloudron.conf; then
+    log "Adding ip6tables flag to docker" # https://github.com/moby/moby/pull/41622
+    echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
+    systemctl daemon-reload
+    systemctl restart docker
+fi
 
 mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
-mkdir -p "${MAIL_DATA_DIR}/dkim"
+mkdir -p "${MAIL_DATA_DIR}"
 
 # keep these in sync with paths.js
 log "Ensuring directories"
@@ -52,7 +57,8 @@ mkdir -p "${PLATFORM_DATA_DIR}/mysql"
 mkdir -p "${PLATFORM_DATA_DIR}/postgresql"
 mkdir -p "${PLATFORM_DATA_DIR}/mongodb"
 mkdir -p "${PLATFORM_DATA_DIR}/redis"
-mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner"
+mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner" \
+         "${PLATFORM_DATA_DIR}/addons/mail/dkim"
 mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
 mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
 mkdir -p "${PLATFORM_DATA_DIR}/acme"
@@ -70,10 +76,6 @@ mkdir -p "${PLATFORM_DATA_DIR}/firewall"
 mkdir -p /var/backups
 chmod 777 /var/backups
 
-# can be removed after 6.3
-[[ -f "${BOX_DATA_DIR}/updatechecker.json" ]] && mv "${BOX_DATA_DIR}/updatechecker.json" "${PLATFORM_DATA_DIR}/update/updatechecker.json"
-rm -rf "${BOX_DATA_DIR}/well-known"
-
 log "Configuring journald"
 sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
     -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
@@ -84,23 +86,19 @@ sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
 sed -e "s/^WatchdogSec=.*$/WatchdogSec=3min/" \
     -i /lib/systemd/system/systemd-journald.service
 
-# Give user access to system logs
-usermod -a -G systemd-journal ${USER}
-mkdir -p /var/log/journal  # in some images, this directory is not created making system log to /run/systemd instead
-chown root:systemd-journal /var/log/journal
+usermod -a -G systemd-journal ${USER} # Give user access to system logs
+if [[ ! -d /var/log/journal ]]; then # in some images, this directory is not created making system log to /run/systemd instead
+    mkdir -p /var/log/journal
+    chown root:systemd-journal /var/log/journal
+    chmod g+s /var/log/journal  # sticky bit for group propagation
+fi
 systemctl daemon-reload
 systemctl restart systemd-journald
-setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
 
 # Give user access to nginx logs (uses adm group)
 usermod -a -G adm ${USER}
 
 log "Setting up unbound"
-# DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org)
-# We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
-# We listen on 0.0.0.0 because there is no way control ordering of docker (which creates the 172.18.0.0/16) and unbound
-# If IP6 is not enabled, dns queries seem to fail on some hosts. -s returns false if file missing or 0 size
-ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
 cp -f "${script_dir}/start/unbound.conf" /etc/unbound/unbound.conf.d/cloudron-network.conf
 # update the root anchor after a out-of-disk-space situation (see #269)
 unbound-anchor -a /var/lib/unbound/root.key
@@ -116,7 +114,7 @@ systemctl enable box
 systemctl enable cloudron-firewall
 systemctl enable --now cloudron-disable-thp
 
-# update firewall rules
+# update firewall rules. this must be done after docker created it's rules
 systemctl restart cloudron-firewall
 
 # For logrotate
@@ -144,11 +142,19 @@ if [[ "${ubuntu_version}" == "20.04" ]]; then
 fi
 systemctl restart collectd
 
+log "Configuring sysctl"
+# If privacy extensions are not disabled on server, this breaks IPv6 detection
+# https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756
+if [[ ! -f /etc/sysctl.d/99-cloudimg-ipv6.conf ]]; then
+    echo "==> Disable temporary address (IPv6)"
+    echo -e "# See https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756\nnet.ipv6.conf.all.use_tempaddr = 0\nnet.ipv6.conf.default.use_tempaddr = 0\n\n" > /etc/sysctl.d/99-cloudimg-ipv6.conf
+    sysctl -p
+fi
+
 log "Configuring logrotate"
 if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then
     echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf
 fi
-rm -f "${PLATFORM_DATA_DIR}/logrotate.d/"*
 cp "${script_dir}/start/logrotate/"* "${PLATFORM_DATA_DIR}/logrotate.d/"
 
 # logrotate files have to be owned by root, this is here to fixup existing installations where we were resetting the owner to yellowtent
@@ -233,11 +239,9 @@ chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 chown "${USER}:${USER}" "${APPS_DATA_DIR}"
 
-# do not chown the boxdata/mail directory; dovecot gets upset
-chown "${USER}:${USER}" "${BOX_DATA_DIR}"
-find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${MAIL_DATA_DIR}" -exec chown -R "${USER}:${USER}" {} \;
+chown "${USER}:${USER}" -R "${BOX_DATA_DIR}"
+# do not chown the boxdata/mail directory entirely; dovecot gets upset
 chown "${USER}:${USER}" "${MAIL_DATA_DIR}"
-chown "${USER}:${USER}" -R "${MAIL_DATA_DIR}/dkim" # this is owned by box currently since it generates the keys
 
 log "Starting Cloudron"
 systemctl start box

setup/start/cloudron-firewall.sh

@@ -3,100 +3,148 @@
 set -eu -o pipefail
 
 echo "==> Setting up firewall"
-iptables -t filter -N CLOUDRON || true
-iptables -t filter -F CLOUDRON # empty any existing rules
+
+has_ipv6=$(cat /proc/net/if_inet6 >/dev/null 2>&1 && echo "yes" || echo "no")
+
+# wait for 120 seconds for xtables lock, checking every 1 second
+readonly iptables="iptables --wait 120 --wait-interval 1"
+readonly ip6tables="ip6tables --wait 120 --wait-interval 1"
+
+function ipxtables() {
+    $iptables "$@"
+    [[ "${has_ipv6}" == "yes" ]] && $ip6tables "$@"
+}
+
+ipxtables -t filter -N CLOUDRON || true
+ipxtables -t filter -F CLOUDRON # empty any existing rules
 
 # first setup any user IP block lists
 ipset create cloudron_blocklist hash:net || true
+ipset create cloudron_blocklist6 hash:net family inet6 || true
+
 /home/yellowtent/box/src/scripts/setblocklist.sh
 
-iptables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist src -j DROP
+$iptables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist src -j DROP
 # the DOCKER-USER chain is not cleared on docker restart
-if ! iptables -t filter -C DOCKER-USER -m set --match-set cloudron_blocklist src -j DROP; then
-    iptables -t filter -I DOCKER-USER 1 -m set --match-set cloudron_blocklist src -j DROP
+if ! $iptables -t filter -C DOCKER-USER -m set --match-set cloudron_blocklist src -j DROP; then
+    $iptables -t filter -I DOCKER-USER 1 -m set --match-set cloudron_blocklist src -j DROP
 fi
 
+$ip6tables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist6 src -j DROP
+# there is no DOCKER-USER chain in ip6tables, bug?
+$ip6tables -D FORWARD -m set --match-set cloudron_blocklist6 src -j DROP || true
+$ip6tables -I FORWARD 1 -m set --match-set cloudron_blocklist6 src -j DROP
+
 # allow related and establisted connections
-iptables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
-iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,443 -j ACCEPT # 202 is the alternate ssh port
+ipxtables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,443 -j ACCEPT # 202 is the alternate ssh port
 
 # whitelist any user ports. we used to use --dports but it has a 15 port limit (XT_MULTI_PORTS)
 ports_json="/home/yellowtent/platformdata/firewall/ports.json"
-if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(','))" 2>/dev/null); then
-    IFS=',' arr=(${allowed_tcp_ports})
-    for p in "${arr[@]}"; do
-        iptables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
+if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(' '))" 2>/dev/null); then
+    for p in $allowed_tcp_ports; do
+        ipxtables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
     done
 fi
 
-if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(','))" 2>/dev/null); then
-    IFS=',' arr=(${allowed_udp_ports})
-    for p in "${arr[@]}"; do
-        iptables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
+if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(' '))" 2>/dev/null); then
+    for p in $allowed_udp_ports; do
+        ipxtables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
     done
 fi
 
+# LDAP user directory allow list
+ipset create cloudron_ldap_allowlist hash:net || true
+ipset flush cloudron_ldap_allowlist
+
+ipset create cloudron_ldap_allowlist6 hash:net family inet6 || true
+ipset flush cloudron_ldap_allowlist6
+
+ldap_allowlist_json="/home/yellowtent/platformdata/firewall/ldap_allowlist.txt"
+if [[ -f "${ldap_allowlist_json}" ]]; then
+    # without the -n block, any last line without a new line won't be read it!
+    while read -r line || [[ -n "$line" ]]; do
+        [[ -z "${line}" ]] && continue # ignore empty lines
+        [[ "$line" =~ ^#.*$ ]] && continue # ignore lines starting with #
+        if [[ "$line" == *":"* ]]; then
+            ipset add -! cloudron_ldap_allowlist6 "${line}" # the -! ignore duplicates
+        else
+            ipset add -! cloudron_ldap_allowlist "${line}" # the -! ignore duplicates
+        fi
+    done < "${ldap_allowlist_json}"
+
+    # ldap server we expose 3004 and also redirect from standard ldaps port 636
+    ipxtables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
+    $iptables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist src -p tcp --dport 3004 -j ACCEPT
+    $ip6tables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist6 src -p tcp --dport 3004 -j ACCEPT
+fi
+
 # turn and stun service
-iptables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
 
-iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
-iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT
-iptables -t filter -A CLOUDRON -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
-iptables -t filter -A CLOUDRON -i lo -j ACCEPT # required for localhost connections (mysql)
+# ICMPv6 is very fundamental to IPv6 connectivity unlike ICMPv4
+$iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
+$iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
+$ip6tables -t filter -A CLOUDRON -p ipv6-icmp -j ACCEPT
+
+ipxtables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT
+$iptables -t filter -A CLOUDRON -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+ipxtables -t filter -A CLOUDRON -i lo -j ACCEPT # required for localhost connections (mysql)
 
 # log dropped incoming. keep this at the end of all the rules
-iptables -t filter -A CLOUDRON -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
-iptables -t filter -A CLOUDRON -j DROP
+ipxtables -t filter -A CLOUDRON -m limit --limit 2/min -j LOG --log-prefix "Packet dropped: " --log-level 7
+ipxtables -t filter -A CLOUDRON -j DROP
 
-if ! iptables -t filter -C INPUT -j CLOUDRON 2>/dev/null; then
-    iptables -t filter -I INPUT -j CLOUDRON
-fi
+# prepend our chain to the filter table
+$iptables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $iptables -t filter -I INPUT -j CLOUDRON
+$ip6tables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $ip6tables -t filter -I INPUT -j CLOUDRON
 
 # Setup rate limit chain (the recent info is at /proc/net/xt_recent)
-iptables -t filter -N CLOUDRON_RATELIMIT || true
-iptables -t filter -F CLOUDRON_RATELIMIT # empty any existing rules
+ipxtables -t filter -N CLOUDRON_RATELIMIT || true
+ipxtables -t filter -F CLOUDRON_RATELIMIT # empty any existing rules
 
 # log dropped incoming. keep this at the end of all the rules
-iptables -t filter -N CLOUDRON_RATELIMIT_LOG || true
-iptables -t filter -F CLOUDRON_RATELIMIT_LOG # empty any existing rules
-iptables -t filter -A CLOUDRON_RATELIMIT_LOG -m limit --limit 2/min -j LOG --log-prefix "IPTables RateLimit: " --log-level 7
-iptables -t filter -A CLOUDRON_RATELIMIT_LOG -j DROP
+ipxtables -t filter -N CLOUDRON_RATELIMIT_LOG || true
+ipxtables -t filter -F CLOUDRON_RATELIMIT_LOG # empty any existing rules
+ipxtables -t filter -A CLOUDRON_RATELIMIT_LOG -m limit --limit 2/min -j LOG --log-prefix "IPTables RateLimit: " --log-level 7
+ipxtables -t filter -A CLOUDRON_RATELIMIT_LOG -j DROP
 
 # http https
 for port in 80 443; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
 # ssh
 for port in 22 202; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
-    iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
+done
+
+# ldaps
+for port in 636 3004; do
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
 # docker translates (dnat) 25, 587, 993, 4190 in the PREROUTING step
 for port in 2525 4190 9993; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
+    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
 done
 
-# msa, ldap, imap, sieve
-for port in 2525 3002 4190 9993; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
+# msa, ldap, imap, sieve, pop3
+for port in 2525 3002 4190 9993 9995; do
+    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
 done
 
 # cloudron docker network: mysql postgresql redis mongodb
 for port in 3306 5432 6379 27017; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
+    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
-if ! iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null; then
-    iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
-fi
+$iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
+$ip6tables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $ip6tables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
 
 # Workaround issue where Docker insists on adding itself first in FORWARD table
-iptables -D FORWARD -j CLOUDRON_RATELIMIT || true
-iptables -I FORWARD 1 -j CLOUDRON_RATELIMIT
-
-echo "==> Setting up firewall done"
+ipxtables -D FORWARD -j CLOUDRON_RATELIMIT || true
+ipxtables -I FORWARD 1 -j CLOUDRON_RATELIMIT

setup/start/cloudron-motd

@@ -4,13 +4,17 @@
 
 printf "**********************************************************************\n\n"
 
-if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
-    if [[ -f /tmp/.cloudron-motd-cache ]]; then
-        ip=$(cat /tmp/.cloudron-motd-cache)
-    elif ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+cache_file="/var/cache/cloudron-motd-cache"
+
+if [[ -z "$(ls -A /home/yellowtent/platformdata/addons/mail/dkim)" ]]; then
+    if [[ ! -f "${cache_file}" ]]; then
+        curl --fail --connect-timeout 2 --max-time 2 -q https://ipv4.api.cloudron.io/api/v1/helper/public_ip --output "${cache_file}" || true
+    fi
+    if [[ -f "${cache_file}" ]]; then
+        ip=$(sed -n -e 's/.*"ip": "\(.*\)"/\1/p' /var/cache/cloudron-motd-cache)
+    else
         ip='<IP>'
     fi
-    echo "${ip}" > /tmp/.cloudron-motd-cache
 
     if [[ ! -f /etc/cloudron/SETUP_TOKEN ]]; then
         url="https://${ip}"

setup/start/nginx/nginx.conf

@@ -19,15 +19,10 @@ http {
     include       mime.types;
     default_type  application/octet-stream;
 
-    # the collectd config depends on this log format
-    log_format combined2 '$remote_addr - [$time_local] '
-        '"$request" $status $body_bytes_sent $request_time '
-        '"$http_referer" "$host" "$http_user_agent"';
-
     # required for long host names
     server_names_hash_bucket_size 128;
 
-    access_log /var/log/nginx/access.log combined2;
+    access_log /var/log/nginx/access.log combined;
 
     sendfile        on;
 

setup/start/sudoers

@@ -53,9 +53,14 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/stoptask.sh
 Defaults!/home/yellowtent/box/src/scripts/setblocklist.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setblocklist.sh
 
+Defaults!/home/yellowtent/box/src/scripts/setldapallowlist.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setldapallowlist.sh
+
 Defaults!/home/yellowtent/box/src/scripts/addmount.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/addmount.sh
 
 Defaults!/home/yellowtent/box/src/scripts/rmmount.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmount.sh
 
+Defaults!/home/yellowtent/box/src/scripts/remountmount.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/remountmount.sh

setup/start/systemd/unbound.service

@@ -2,7 +2,7 @@
 
 [Unit]
 Description=Unbound DNS Resolver
-After=network-online.target docker.service
+After=network-online.target
 Before=nss-lookup.target
 Wants=network-online.target nss-lookup.target
 

setup/start/unbound.conf

@@ -1,7 +1,11 @@
+# Unbound is used primarily for RBL queries (host 2.0.0.127.zen.spamhaus.org)
+# We cannot use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
+
 server:
         port: 53
         interface: 127.0.0.1
         interface: 172.18.0.1
+        ip-freebind: yes
         do-ip6: no
         access-control: 127.0.0.1 allow
         access-control: 172.18.0.1/16 allow
@@ -10,4 +14,3 @@ server:
         # enable below for logging to journalctl -u unbound
         # verbosity: 5
         # log-queries: yes
-

src/accesscontrol.js

@@ -8,10 +8,7 @@ const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     safe = require('safetydance'),
     tokens = require('./tokens.js'),
-    users = require('./users.js'),
-    util = require('util');
-
-const userGet = util.promisify(users.get);
+    users = require('./users.js');
 
 async function verifyToken(accessToken) {
     assert.strictEqual(typeof accessToken, 'string');
@@ -19,10 +16,8 @@ async function verifyToken(accessToken) {
     const token = await tokens.getByAccessToken(accessToken);
     if (!token) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'No such token');
 
-    const [error, user] = await safe(userGet(token.identifier));
-    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not found');
-    if (error) throw error;
-
+    const user = await users.get(token.identifier);
+    if (!user) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not found');
     if (!user.active) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not active');
 
     await safe(tokens.update(token.id, { lastUsedTime: new Date() })); // ignore any error

src/acme2.js

@@ -9,11 +9,11 @@ exports = module.exports = {
 };
 
 const assert = require('assert'),
-    async = require('async'),
+    blobs = require('./blobs.js'),
     BoxError = require('./boxerror.js'),
     crypto = require('crypto'),
     debug = require('debug')('box:cert/acme2'),
-    domains = require('./domains.js'),
+    dns = require('./dns.js'),
     fs = require('fs'),
     os = require('os'),
     path = require('path'),
@@ -32,7 +32,7 @@ const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
 function Acme2(options) {
     assert.strictEqual(typeof options, 'object');
 
-    this.accountKeyPem = options.accountKeyPem; // Buffer
+    this.accountKeyPem = null; // Buffer .
     this.email = options.email;
     this.keyId = null;
     this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
@@ -88,10 +88,10 @@ Acme2.prototype.sendSignedRequest = async function (url, payload) {
 
     let [error, response] = await safe(superagent.get(this.directory.newNonce).timeout(30000).ok(() => true));
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`);
-    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code when fetching nonce : ${response.status}`);
+    if (response.status !== 204) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching nonce : ${response.status}`);
 
     const nonce = response.headers['Replay-Nonce'.toLowerCase()];
-    if (!nonce) throw new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response');
+    if (!nonce) throw new BoxError(BoxError.ACME_ERROR, 'No nonce in response');
 
     debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
 
@@ -129,23 +129,43 @@ Acme2.prototype.updateContact = async function (registrationUri) {
     };
 
     const result = await this.sendSignedRequest(registrationUri, JSON.stringify(payload));
-    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
+    if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
 
     debug(`updateContact: contact of user updated to ${this.email}`);
 };
 
-Acme2.prototype.registerUser = async function () {
+async function generateAccountKey() {
+    const acmeAccountKey = safe.child_process.execSync('openssl genrsa 4096');
+    if (!acmeAccountKey) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate acme account key: ${safe.error.message}`);
+    return acmeAccountKey;
+}
+
+Acme2.prototype.ensureAccount = async function () {
     const payload = {
         termsOfServiceAgreed: true
     };
 
-    debug('registerUser: registering user');
+    debug('ensureAccount: registering user');
+
+    this.accountKeyPem = await blobs.get(blobs.ACME_ACCOUNT_KEY);
+    if (!this.accountKeyPem) {
+        debug('ensureAccount: generating new account keys');
+        this.accountKeyPem = await generateAccountKey();
+        await blobs.set(blobs.ACME_ACCOUNT_KEY, this.accountKeyPem);
+    }
+
+    let result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
+    if (result.status === 403 && result.body.type === 'urn:ietf:params:acme:error:unauthorized') {
+        debug(`ensureAccount: key was revoked. ${result.status} ${JSON.stringify(result.body)}. generating new account key`);
+        this.accountKeyPem = await generateAccountKey();
+        await blobs.set(blobs.ACME_ACCOUNT_KEY, this.accountKeyPem);
+        result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
+    }
 
-    const result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
     // 200 if already exists. 201 for new accounts
-    if (result.status !== 200 && result.status !== 201) return new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${JSON.stringify(result.body)}`);
+    if (result.status !== 200 && result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${JSON.stringify(result.body)}`);
 
-    debug(`registerUser: user registered keyid: ${result.headers.location}`);
+    debug(`ensureAccount: user registered keyid: ${result.headers.location}`);
 
     this.keyId = result.headers.location;
 
@@ -166,15 +186,15 @@ Acme2.prototype.newOrder = async function (domain) {
 
     const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload));
     if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`);
-    if (result.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+    if (result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`);
 
     debug('newOrder: created order %s %j', domain, result.body);
 
     const order = result.body, orderUrl = result.headers.location;
 
-    if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid authorizations in order');
-    if (typeof order.finalize !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid finalize in order');
-    if (typeof orderUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid order location in order header');
+    if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.ACME_ERROR, 'invalid authorizations in order');
+    if (typeof order.finalize !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid finalize in order');
+    if (typeof orderUrl !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid order location in order header');
 
     return { order, orderUrl };
 };
@@ -184,20 +204,20 @@ Acme2.prototype.waitForOrder = async function (orderUrl) {
 
     debug(`waitForOrder: ${orderUrl}`);
 
-    return await promiseRetry({ times: 15, interval: 20000 }, async () => {
+    return await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
         debug('waitForOrder: getting status');
 
         const result = await this.postAsGet(orderUrl);
         if (result.status !== 200) {
             debug(`waitForOrder: invalid response code getting uri ${result.status}`);
-            throw new BoxError(BoxError.EXTERNAL_ERROR, `Bad response code: ${result.status}`);
+            throw new BoxError(BoxError.ACME_ERROR, `Bad response when waiting for order. code: ${result.status}`);
         }
 
         debug('waitForOrder: status is "%s %j', result.body.status, result.body);
 
-        if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.TRY_AGAIN, `Request is in ${result.body.status} state`);
+        if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.ACME_ERROR, `Request is in ${result.body.status} state`);
         else if (result.body.status === 'valid' && result.body.certificate) return result.body.certificate;
-        else throw new BoxError(BoxError.EXTERNAL_ERROR, `Unexpected status or invalid response: ${JSON.stringify(result.body)}`);
+        else throw new BoxError(BoxError.ACME_ERROR, `Unexpected status or invalid response when waiting for order: ${JSON.stringify(result.body)}`);
     });
 };
 
@@ -229,7 +249,7 @@ Acme2.prototype.notifyChallengeReady = async function (challenge) {
     };
 
     const result = await this.sendSignedRequest(challenge.url, JSON.stringify(payload));
-    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+    if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
 };
 
 Acme2.prototype.waitForChallenge = async function (challenge) {
@@ -237,20 +257,20 @@ Acme2.prototype.waitForChallenge = async function (challenge) {
 
     debug('waitingForChallenge: %j', challenge);
 
-    await promiseRetry({ times: 15, interval: 20000 }, async () => {
+    await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
         debug('waitingForChallenge: getting status');
 
         const result = await this.postAsGet(challenge.url);
         if (result.status !== 200) {
             debug(`waitForChallenge: invalid response code getting uri ${result.status}`);
-            throw new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode);
+            throw new BoxError(BoxError.ACME_ERROR, `Bad response code when waiting for challenge : ${result.status}`);
         }
 
         debug(`waitForChallenge: status is "${result.body.status}" "${JSON.stringify(result.body)}"`);
 
-        if (result.body.status === 'pending') throw new BoxError(BoxError.TRY_AGAIN);
+        if (result.body.status === 'pending') throw new BoxError(BoxError.ACME_ERROR, 'Challenge is in pending state');
         else if (result.body.status === 'valid') return;
-        else throw new BoxError(BoxError.EXTERNAL_ERROR, `Unexpected status: ${result.body.status}`);
+        else throw new BoxError(BoxError.ACME_ERROR, `Unexpected status when waiting for challenge: ${result.body.status}`);
     });
 };
 
@@ -268,7 +288,7 @@ Acme2.prototype.signCertificate = async function (domain, finalizationUrl, csrDe
 
     const result = await this.sendSignedRequest(finalizationUrl, JSON.stringify(payload));
     // 429 means we reached the cert limit for this domain
-    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
+    if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to sign certificate. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
 };
 
 Acme2.prototype.createKeyAndCsr = async function (hostname, keyFilePath, csrFilePath) {
@@ -304,7 +324,7 @@ Acme2.prototype.createKeyAndCsr = async function (hostname, keyFilePath, csrFile
     if (!csrDer) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
     if (!safe.fs.writeFileSync(csrFilePath, csrDer)) throw new BoxError(BoxError.FS_ERROR, safe.error); // bookkeeping. inspect with openssl req -text -noout -in hostname.csr -inform der
 
-    await safe(fs.promises.rmdir(tmpdir, { recursive: true }));
+    await safe(fs.promises.rm(tmpdir, { recursive: true, force: true }));
 
     debug('createKeyAndCsr: csr file (DER) saved at %s', csrFilePath);
 
@@ -315,12 +335,12 @@ Acme2.prototype.downloadCertificate = async function (hostname, certUrl, certFil
     assert.strictEqual(typeof hostname, 'string');
     assert.strictEqual(typeof certUrl, 'string');
 
-    await promiseRetry({ times: 5, interval: 20000 }, async () => {
-        debug('downloadCertificate: downloading certificate');
+    await promiseRetry({ times: 5, interval: 20000, debug }, async () => {
+        debug(`downloadCertificate: downloading certificate of ${hostname}`);
 
         const result = await this.postAsGet(certUrl);
-        if (result.statusCode === 202) throw new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate');
-        if (result.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+        if (result.statusCode === 202) throw new BoxError(BoxError.ACME_ERROR, 'Retry downloading certificate');
+        if (result.statusCode !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
 
         const fullChainPem = result.body; // buffer
 
@@ -338,7 +358,7 @@ Acme2.prototype.prepareHttpChallenge = async function (hostname, domain, authori
 
     debug('prepareHttpChallenge: challenges: %j', authorization);
     let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
-    if (httpChallenges.length === 0) throw new BoxError(BoxError.EXTERNAL_ERROR, 'no http challenges');
+    if (httpChallenges.length === 0) throw new BoxError(BoxError.ACME_ERROR, 'no http challenges');
     let challenge = httpChallenges[0];
 
     debug('prepareHttpChallenge: preparing for challenge %j', challenge);
@@ -387,7 +407,7 @@ Acme2.prototype.prepareDnsChallenge = async function (hostname, domain, authoriz
 
     debug('prepareDnsChallenge: challenges: %j', authorization);
     const dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
-    if (dnsChallenges.length === 0) throw new BoxError(BoxError.EXTERNAL_ERROR, 'no dns challenges');
+    if (dnsChallenges.length === 0) throw new BoxError(BoxError.ACME_ERROR, 'no dns challenges');
     const challenge = dnsChallenges[0];
 
     const keyAuthorization = this.getKeyAuthorization(challenge.token);
@@ -399,17 +419,11 @@ Acme2.prototype.prepareDnsChallenge = async function (hostname, domain, authoriz
 
     debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
 
-    return new Promise((resolve, reject) => {
-        domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-            if (error) return reject(error);
+    await dns.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
 
-            domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 }, function (error) {
-                if (error) return reject(error);
+    await dns.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 });
 
-                resolve(challenge);
-            });
-        });
-    });
+    return challenge;
 };
 
 Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challenge) {
@@ -426,13 +440,7 @@ Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challeng
 
     debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
 
-    return new Promise((resolve, reject) => {
-        domains.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-            if (error) return reject(error);
-
-            resolve(null);
-        });
-    });
+    await dns.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
 };
 
 Acme2.prototype.prepareChallenge = async function (hostname, domain, authorizationUrl, acmeChallengesDir) {
@@ -444,7 +452,7 @@ Acme2.prototype.prepareChallenge = async function (hostname, domain, authorizati
     debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
 
     const response = await this.postAsGet(authorizationUrl);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code getting authorization : ${response.status}`);
+    if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code getting authorization : ${response.status}`);
 
     const authorization = response.body;
 
@@ -477,7 +485,7 @@ Acme2.prototype.acmeFlow = async function (hostname, domain, paths) {
 
     const { certFilePath, keyFilePath, csrFilePath, acmeChallengesDir } = paths;
 
-    await this.registerUser();
+    await this.ensureAccount();
     const { order, orderUrl } = await this.newOrder(hostname);
 
     for (let i = 0; i < order.authorizations.length; i++) {
@@ -501,14 +509,14 @@ Acme2.prototype.acmeFlow = async function (hostname, domain, paths) {
 };
 
 Acme2.prototype.loadDirectory = async function () {
-    await promiseRetry({ times: 3, interval: 20000 }, async () => {
+    await promiseRetry({ times: 3, interval: 20000, debug }, async () => {
         const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true);
 
-        if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code when fetching directory : ${response.status}`);
+        if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching directory : ${response.status}`);
 
         if (typeof response.body.newNonce !== 'string' ||
             typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`);
+            typeof response.body.newAccount !== 'string') throw new BoxError(BoxError.ACME_ERROR, `Invalid response body : ${response.body}`);
 
         this.directory = response.body;
     });
@@ -522,7 +530,7 @@ Acme2.prototype.getCertificate = async function (vhost, domain, paths) {
     debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
 
     if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
-        vhost = domains.makeWildcard(vhost);
+        vhost = dns.makeWildcard(vhost);
         debug(`getCertificate: will get wildcard cert for ${vhost}`);
     }
 
@@ -530,18 +538,16 @@ Acme2.prototype.getCertificate = async function (vhost, domain, paths) {
     await this.acmeFlow(vhost, domain, paths);
 };
 
-function getCertificate(vhost, domain, paths, options, callback) {
+async function getCertificate(vhost, domain, paths, options) {
     assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof paths, 'object');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    let attempt = 1;
-    async.retry({ times: 3, interval: 0 }, function (retryCallback) {
-        debug(`getCertificate: attempt ${attempt++}`);
+    await promiseRetry({ times: 3, interval: 0, debug }, async function () {
+        debug(`getCertificate: for vhost ${vhost} and domain ${domain}`);
 
-        let acme = new Acme2(options || { });
-        acme.getCertificate(vhost, domain, paths).then(callback).catch(retryCallback);
-    }, callback);
+        const acme = new Acme2(options || { });
+        return await acme.getCertificate(vhost, domain, paths);
+    });
 }

src/addonconfigs.js

@@ -0,0 +1,81 @@
+'use strict';
+
+exports = module.exports =  {
+    get,
+    set,
+    unset,
+
+    getByAppId,
+    getByName,
+    unsetByAppId,
+    getAppIdByValue,
+};
+
+const assert = require('assert'),
+    database = require('./database.js');
+
+async function set(appId, addonId, env) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert(Array.isArray(env));
+
+    await unset(appId, addonId);
+    if (env.length === 0) return;
+
+    const query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
+    const args = [ ], queryArgs = [ ];
+    for (let i = 0; i < env.length; i++) {
+        args.push(appId, addonId, env[i].name, env[i].value);
+        queryArgs.push('(?, ?, ?, ?)');
+    }
+
+    await database.query(query + queryArgs.join(','), args);
+}
+
+async function unset(appId, addonId) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+
+    await database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
+}
+
+async function unsetByAppId(appId) {
+    assert.strictEqual(typeof appId, 'string');
+
+    await database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ]);
+}
+
+async function get(appId, addonId) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+
+    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
+    return results;
+}
+
+async function getByAppId(appId) {
+    assert.strictEqual(typeof appId, 'string');
+
+    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ]);
+    return results;
+}
+
+async function getAppIdByValue(addonId, namePattern, value) {
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+    assert.strictEqual(typeof value, 'string');
+
+    const results = await database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ]);
+    if (results.length === 0) return null;
+    return results[0].appId;
+}
+
+async function getByName(appId, addonId, namePattern) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+
+    const results = await database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ]);
+    if (results.length === 0) return null;
+    return results[0].value;
+}

src/appdb.js

@@ -1,597 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get,
-    add,
-    exists,
-    del,
-    update,
-    getAll,
-    getPortBindings,
-    delPortBinding,
-
-    setAddonConfig,
-    getAddonConfig,
-    getAddonConfigByAppId,
-    getAddonConfigByName,
-    unsetAddonConfig,
-    unsetAddonConfigByAppId,
-    getAppIdByAddonConfigValue,
-    getByIpAddress,
-
-    getIcons,
-
-    setHealth,
-    setTask,
-    getAppStoreIds,
-
-    // subdomain table types
-    SUBDOMAIN_TYPE_PRIMARY: 'primary',
-    SUBDOMAIN_TYPE_REDIRECT: 'redirect',
-    SUBDOMAIN_TYPE_ALIAS: 'alias',
-
-    _clear: clear
-};
-
-const assert = require('assert'),
-    async = require('async'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js'),
-    safe = require('safetydance'),
-    util = require('util');
-
-const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
-    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
-    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson',
-    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp',
-    'apps.creationTime', 'apps.updateTime', 'apps.enableMailbox', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
-    'apps.dataDir', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
-
-const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
-
-function postProcess(result) {
-    assert.strictEqual(typeof result, 'object');
-
-    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
-    result.manifest = safe.JSON.parse(result.manifestJson);
-    delete result.manifestJson;
-
-    assert(result.tagsJson === null || typeof result.tagsJson === 'string');
-    result.tags = safe.JSON.parse(result.tagsJson) || [];
-    delete result.tagsJson;
-
-    assert(result.reverseProxyConfigJson === null || typeof result.reverseProxyConfigJson === 'string');
-    result.reverseProxyConfig = safe.JSON.parse(result.reverseProxyConfigJson) || {};
-    delete result.reverseProxyConfigJson;
-
-    assert(result.hostPorts === null || typeof result.hostPorts === 'string');
-    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');
-
-    result.portBindings = { };
-    let hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
-    let environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
-    let portTypes = result.portTypes === null ? [ ] : result.portTypes.split(',');
-
-    delete result.hostPorts;
-    delete result.environmentVariables;
-    delete result.portTypes;
-
-    for (let i = 0; i < environmentVariables.length; i++) {
-        result.portBindings[environmentVariables[i]] = { hostPort: parseInt(hostPorts[i], 10), type: portTypes[i] };
-    }
-
-    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
-    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
-    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
-    delete result.accessRestrictionJson;
-
-    result.sso = !!result.sso;
-    result.enableBackup = !!result.enableBackup;
-    result.enableAutomaticUpdate = !!result.enableAutomaticUpdate;
-    result.enableMailbox = !!result.enableMailbox;
-    result.proxyAuth = !!result.proxyAuth;
-    result.hasIcon = !!result.hasIcon;
-    result.hasAppStoreIcon = !!result.hasAppStoreIcon;
-
-    assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
-    result.debugMode = safe.JSON.parse(result.debugModeJson);
-    delete result.debugModeJson;
-
-    assert(result.servicesConfigJson === null || typeof result.servicesConfigJson === 'string');
-    result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
-    delete result.servicesConfigJson;
-
-    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
-    delete result.subdomains;
-    delete result.domains;
-    delete result.subdomainTypes;
-
-    result.alternateDomains = [];
-    result.aliasDomains = [];
-    for (let i = 0; i < subdomainTypes.length; i++) {
-        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
-            result.location = subdomains[i];
-            result.domain = domains[i];
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
-            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
-            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
-        }
-    }
-
-    let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
-    delete result.envNames;
-    delete result.envValues;
-    result.env = {};
-    for (let i = 0; i < envNames.length; i++) { // NOTE: envNames is [ null ] when env of an app is empty
-        if (envNames[i]) result.env[envNames[i]] = envValues[i];
-    }
-
-    let volumeIds = JSON.parse(result.volumeIds);
-    delete result.volumeIds;
-    let volumeReadOnlys = JSON.parse(result.volumeReadOnlys);
-    delete result.volumeReadOnlys;
-
-    result.mounts = volumeIds[0] === null ? [] : volumeIds.map((v, idx) => { return { volumeId: v, readOnly: !!volumeReadOnlys[idx] }; }); // NOTE: volumeIds is [null] when volumes of an app is empty
-
-    result.error = safe.JSON.parse(result.errorJson);
-    delete result.errorJson;
-
-    result.taskId = result.taskId ? String(result.taskId) : null;
-}
-
-// each query simply join apps table with another table by id. we then join the full result together
-const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
-const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
-const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
-const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
-const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
-    + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
-    + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
-    + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
-    + ` LEFT JOIN (${MOUNTS_QUERY}) AS q4 on q4.id = apps.id`;
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`${APPS_QUERY} WHERE apps.id = ?`, [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
-function getByIpAddress(ip, callback) {
-    assert.strictEqual(typeof ip, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`${APPS_QUERY} WHERE apps.containerIp = ?`, [ ip ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        results.forEach(postProcess);
-
-        callback(null, results);
-    });
-}
-
-function add(id, appStoreId, manifest, location, domain, portBindings, data, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appStoreId, 'string');
-    assert(manifest && typeof manifest === 'object');
-    assert.strictEqual(typeof manifest.version, 'string');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert(data && typeof data === 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    portBindings = portBindings || { };
-
-    var manifestJson = JSON.stringify(manifest);
-
-    const accessRestriction = data.accessRestriction || null;
-    const accessRestrictionJson = JSON.stringify(accessRestriction);
-    const memoryLimit = data.memoryLimit || 0;
-    const cpuShares = data.cpuShares || 512;
-    const installationState = data.installationState;
-    const runState = data.runState;
-    const sso = 'sso' in data ? data.sso : null;
-    const debugModeJson = data.debugMode ? JSON.stringify(data.debugMode) : null;
-    const env = data.env || {};
-    const label = data.label || null;
-    const tagsJson = data.tags ? JSON.stringify(data.tags) : null;
-    const mailboxName = data.mailboxName || null;
-    const mailboxDomain = data.mailboxDomain || null;
-    const reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null;
-    const servicesConfigJson = data.servicesConfig ? JSON.stringify(data.servicesConfig) : null;
-    const enableMailbox = data.enableMailbox || false;
-    const icon = data.icon || null;
-
-    let queries = [];
-
-    queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
-            + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
-            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox ]
-    });
-
-    queries.push({
-        query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-        args: [ id, domain, location, exports.SUBDOMAIN_TYPE_PRIMARY ]
-    });
-
-    Object.keys(portBindings).forEach(function (env) {
-        queries.push({
-            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, type, appId) VALUES (?, ?, ?, ?)',
-            args: [ env, portBindings[env].hostPort, portBindings[env].type, id ]
-        });
-    });
-
-    Object.keys(env).forEach(function (name) {
-        queries.push({
-            query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
-            args: [ id, name, env[name] ]
-        });
-    });
-
-    if (data.alternateDomains) {
-        data.alternateDomains.forEach(function (d) {
-            queries.push({
-                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]
-            });
-        });
-    }
-
-    if (data.aliasDomains) {
-        data.aliasDomains.forEach(function (d) {
-            queries.push({
-                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
-            });
-        });
-    }
-
-    database.transaction(queries, function (error) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
-        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'no such domain'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function exists(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        return callback(null, result.length !== 0);
-    });
-}
-
-function getPortBindings(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        var portBindings = { };
-        for (let i = 0; i < results.length; i++) {
-            portBindings[results[i].environmentVariable] = { hostPort: results[i].hostPort, type: results[i].type };
-        }
-
-        callback(null, portBindings);
-    });
-}
-
-function getIcons(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT icon, appStoreIcon FROM apps WHERE id = ?', [ id ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, { icon: results[0].icon, appStoreIcon: results[0].appStoreIcon });
-    });
-}
-
-function delPortBinding(hostPort, type, callback) {
-    assert.strictEqual(typeof hostPort, 'number');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM appPortBindings WHERE hostPort=? AND type=?', [ hostPort, type ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var queries = [
-        { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
-        { query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
-    ];
-
-    database.transaction(queries, function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[5].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        database.query.bind(null, 'DELETE FROM subdomains'),
-        database.query.bind(null, 'DELETE FROM appPortBindings'),
-        database.query.bind(null, 'DELETE FROM appAddonConfigs'),
-        database.query.bind(null, 'DELETE FROM appEnvVars'),
-        database.query.bind(null, 'DELETE FROM apps')
-    ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        return callback(null);
-    });
-}
-
-function update(id, app, callback) {
-    // ts is useful as a versioning mechanism (for example, icon changed). update the timestamp explicity in code instead of db.
-    // this way health and healthTime can be updated without changing ts
-    app.ts = new Date();
-    updateWithConstraints(id, app, '', callback);
-}
-
-function updateWithConstraints(id, app, constraints, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof constraints, 'string');
-    assert.strictEqual(typeof callback, 'function');
-    assert(!('portBindings' in app) || typeof app.portBindings === 'object');
-    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
-    assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
-    assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
-    assert(!('tags' in app) || Array.isArray(app.tags));
-    assert(!('env' in app) || typeof app.env === 'object');
-
-    var queries = [ ];
-
-    if ('portBindings' in app) {
-        var portBindings = app.portBindings || { };
-        // replace entries by app id
-        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
-        Object.keys(portBindings).forEach(function (env) {
-            var values = [ portBindings[env].hostPort, portBindings[env].type, env, id ];
-            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, type, environmentVariable, appId) VALUES(?, ?, ?, ?)', args: values });
-        });
-    }
-
-    if ('env' in app) {
-        queries.push({ query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] });
-
-        Object.keys(app.env).forEach(function (name) {
-            queries.push({
-                query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
-                args: [ id, name, app.env[name] ]
-            });
-        });
-    }
-
-    if ('location' in app && 'domain' in app) { // must be updated together as they are unique together
-        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ]}); // all locations of an app must be updated together
-        queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, app.domain, app.location, exports.SUBDOMAIN_TYPE_PRIMARY ]});
-
-        if ('alternateDomains' in app) {
-            app.alternateDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
-            });
-        }
-
-        if ('aliasDomains' in app) {
-            app.aliasDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
-            });
-        }
-    }
-
-    if ('mounts' in app) {
-        queries.push({ query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ]});
-        app.mounts.forEach(function (m) {
-            queries.push({ query: 'INSERT INTO appMounts (appId, volumeId, readOnly) VALUES (?, ?, ?)', args: [ id, m.volumeId, m.readOnly ]});
-        });
-    }
-
-    var fields = [ ], values = [ ];
-    for (let p in app) {
-        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig') {
-            fields.push(`${p}Json = ?`);
-            values.push(JSON.stringify(app[p]));
-        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
-            fields.push(p + ' = ?');
-            values.push(app[p]);
-        }
-    }
-
-    if (values.length !== 0) {
-        values.push(id);
-        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
-    }
-
-    database.transaction(queries, function (error, results) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[results.length - 1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        return callback(null);
-    });
-}
-
-function setHealth(appId, health, healthTime, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof health, 'string');
-    assert(util.types.isDate(healthTime));
-    assert.strictEqual(typeof callback, 'function');
-
-    const values = { health, healthTime };
-
-    updateWithConstraints(appId, values, '', callback);
-}
-
-function setTask(appId, values, options, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof values, 'object');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    values.ts = new Date();
-
-    if (!options.requireNullTaskId) return updateWithConstraints(appId, values, '', callback);
-
-    if (options.requiredState === null) {
-        updateWithConstraints(appId, values, 'AND taskId IS NULL', callback);
-    } else {
-        updateWithConstraints(appId, values, `AND taskId IS NULL AND installationState = "${options.requiredState}"`, callback);
-    }
-}
-
-function getAppStoreIds(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT id, appStoreId FROM apps', function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function setAddonConfig(appId, addonId, env, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert(Array.isArray(env));
-    assert.strictEqual(typeof callback, 'function');
-
-    unsetAddonConfig(appId, addonId, function (error) {
-        if (error) return callback(error);
-
-        if (env.length === 0) return callback(null);
-
-        var query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
-        var args = [ ], queryArgs = [ ];
-        for (var i = 0; i < env.length; i++) {
-            args.push(appId, addonId, env[i].name, env[i].value);
-            queryArgs.push('(?, ?, ?, ?)');
-        }
-
-        database.query(query + queryArgs.join(','), args, function (error) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            return callback(null);
-        });
-    });
-}
-
-function unsetAddonConfig(appId, addonId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function unsetAddonConfigByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function getAddonConfig(appId, addonId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAddonConfigByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAppIdByAddonConfigValue(addonId, namePattern, value, callback) {
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-    assert.strictEqual(typeof value, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null, results[0].appId);
-    });
-}
-
-function getAddonConfigByName(appId, addonId, namePattern, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null, results[0].value);
-    });
-}

src/apphealthmonitor.js

@@ -1,10 +1,8 @@
 'use strict';
 
-const appdb = require('./appdb.js'),
-    apps = require('./apps.js'),
+const apps = require('./apps.js'),
     assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:apphealthmonitor'),
@@ -17,17 +15,15 @@ exports = module.exports = {
     run
 };
 
-const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
 const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes
 
 const OOM_EVENT_LIMIT = 60 * 60 * 1000; // will only raise 1 oom event every hour
 let gStartTime = null; // time when apphealthmonitor was started
 let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago
 
-function setHealth(app, health, callback) {
+async function setHealth(app, health) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof health, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // app starts out with null health
     // if it became healthy, we update immediately. this is required for ui to say "running" etc
@@ -42,79 +38,74 @@ function setHealth(app, health, callback) {
             debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, auditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app: app });
         }
     } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
         if (lastHealth === apps.HEALTH_HEALTHY) {
             debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, auditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app: app });
         }
     } else {
         debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);
-        return callback(null);
+        return;
     }
 
-    appdb.setHealth(app.id, health, healthTime, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null); // app uninstalled?
-        if (error) return callback(error);
+    const [error] = await safe(apps.setHealth(app.id, health, healthTime));
+    if (error && error.reason === BoxError.NOT_FOUND) return; // app uninstalled?
+    if (error) throw error;
 
-        app.health = health;
-        app.healthTime = healthTime;
-
-        callback(null);
-    });
+    app.health = health;
+    app.healthTime = healthTime;
 }
 
 
 // callback is called with error for fatal errors and not if health check failed
-function checkAppHealth(app, callback) {
+async function checkAppHealth(app, options) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
+    assert.strictEqual(typeof options, 'object');
 
-    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) {
-        return callback(null);
-    }
+    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) return;
 
     const manifest = app.manifest;
 
-    docker.inspect(app.containerId, function (error, data) {
-        if (error || !data || !data.State) return setHealth(app, apps.HEALTH_ERROR, callback);
-        if (data.State.Running !== true) return setHealth(app, apps.HEALTH_DEAD, callback);
+    const [error, data] = await safe(docker.inspect(app.containerId));
+    if (error || !data || !data.State) return await setHealth(app, apps.HEALTH_ERROR);
+    if (data.State.Running !== true) return await setHealth(app, apps.HEALTH_DEAD);
 
-        // non-appstore apps may not have healthCheckPath
-        if (!manifest.healthCheckPath) return setHealth(app, apps.HEALTH_HEALTHY, callback);
+    // non-appstore apps may not have healthCheckPath
+    if (!manifest.healthCheckPath) return await setHealth(app, apps.HEALTH_HEALTHY);
 
-        const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
-        superagent
-            .get(healthCheckUrl)
-            .set('Host', app.fqdn) // required for some apache configs with rewrite rules
-            .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
-            .redirects(0)
-            .timeout(HEALTHCHECK_INTERVAL)
-            .end(function (error, res) {
-                if (error && !error.response) {
-                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
-                } else if (res.statusCode > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
-                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
-                } else {
-                    setHealth(app, apps.HEALTH_HEALTHY, callback);
-                }
-            });
-    });
+    const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
+    const [healthCheckError, response] = await safe(superagent
+        .get(healthCheckUrl)
+        .set('Host', app.fqdn) // required for some apache configs with rewrite rules
+        .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
+        .redirects(0)
+        .ok(() => true)
+        .timeout(options.timeout * 1000));
+
+    if (healthCheckError) {
+        await setHealth(app, apps.HEALTH_UNHEALTHY);
+    } else if (response.status > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
+        await setHealth(app, apps.HEALTH_UNHEALTHY);
+    } else {
+        await setHealth(app, apps.HEALTH_HEALTHY);
+    }
 }
 
-function getContainerInfo(containerId, callback) {
-    docker.inspect(containerId, function (error, result) {
-        if (error) return callback(error);
+async function getContainerInfo(containerId) {
+    const result = await docker.inspect(containerId);
 
-        const appId = safe.query(result, 'Config.Labels.appId', null);
+    const appId = safe.query(result, 'Config.Labels.appId', null);
+    if (appId) return { app: await apps.get(appId) }; // don't get by container id as this can be an exec container
 
-        if (!appId) return callback(null, null /* app */, { name: result.Name.slice(1) }); // addon . Name has a '/' in the beginning for some reason
-
-        apps.get(appId, callback); // don't get by container id as this can be an exec container
-    });
+    if (result.Name.startsWith('/redis-')) {
+        return { app: await apps.get(result.Name.slice('/redis-'.length)), addonName: 'redis' };
+    } else {
+        return { addonName: result.Name.slice(1) }; // addon . Name has a '/' in the beginning for some reason
+    }
 }
 
 /*
@@ -122,81 +113,69 @@ function getContainerInfo(containerId, callback) {
         docker run -ti -m 100M cloudron/base:3.0.0 /bin/bash
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
-function processDockerEvents(intervalSecs, callback) {
-    assert.strictEqual(typeof intervalSecs, 'number');
-    assert.strictEqual(typeof callback, 'function');
+async function processDockerEvents(options) {
+    assert.strictEqual(typeof options, 'object');
 
-    const since = ((new Date().getTime() / 1000) - intervalSecs).toFixed(0);
+    const since = ((new Date().getTime() / 1000) - options.intervalSecs).toFixed(0);
     const until = ((new Date().getTime() / 1000) - 1).toFixed(0);
 
-    docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
-        if (error) return callback(error);
+    const stream = await docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) });
+    stream.setEncoding('utf8');
+    stream.on('data', async function (data) { // this is actually ldjson, we only process the first line for now
+        const event = safe.JSON.parse(data);
+        if (!event) return;
+        const containerId = String(event.id);
 
-        stream.setEncoding('utf8');
-        stream.on('data', function (data) {
-            const event = JSON.parse(data);
-            const containerId = String(event.id);
+        const [error, info] = await safe(getContainerInfo(containerId));
+        const program = error ? containerId : (info.addonName || info.app.fqdn);
+        const now = Date.now();
 
-            getContainerInfo(containerId, function (error, app, addon) {
-                const program = error ? containerId : (app ? app.fqdn : addon.name);
-                const now = Date.now();
-                const notifyUser = !(app && app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
+        // do not send mails for dev apps
+        const notifyUser = !(info.app && info.app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
 
-                debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
+        debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
 
-                // do not send mails for dev apps
-                if (notifyUser) {
-                    // app can be null for addon containers
-                    eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event, containerId, addon: addon || null, app: app || null });
+        if (notifyUser) {
+            await eventlog.add(eventlog.ACTION_APP_OOM, AuditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null });
 
-                    gLastOomMailTime = now;
-                }
-            });
-        });
-
-        stream.on('error', function (error) {
-            debug('Error reading docker events', error);
-            callback();
-        });
-
-        stream.on('end', callback);
-
-        // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
-        setTimeout(stream.destroy.bind(stream), 3000); // https://github.com/apocas/dockerode/issues/179
+            gLastOomMailTime = now;
+        }
     });
+
+    stream.on('error', function (error) {
+        debug('Error reading docker events', error);
+    });
+
+    stream.on('end', function () {
+        // debug('Event stream ended');
+    });
+
+    // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
+    setTimeout(stream.destroy.bind(stream), options.timeout); // https://github.com/apocas/dockerode/issues/179
 }
 
-function processApp(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function processApp(options) {
+    assert.strictEqual(typeof options, 'object');
 
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(error);
+    const allApps = await apps.list();
 
-        async.each(allApps, checkAppHealth, function (error) {
-            const alive = allApps
-                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
+    const healthChecks = allApps.map((app) => checkAppHealth(app, options)); // start healthcheck in parallel
 
-            debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.` + (error ? ` ${error.reason}` : ''));
+    await Promise.allSettled(healthChecks); // wait for all promises to finish
 
-            callback(null);
-        });
-    });
+    const alive = allApps
+        .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
+
+    debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.`);
 }
 
-function run(intervalSecs, callback) {
+async function run(intervalSecs) {
     assert.strictEqual(typeof intervalSecs, 'number');
-    assert.strictEqual(typeof callback, 'function');
 
     if (constants.TEST) return;
 
     if (!gStartTime) gStartTime = new Date();
 
-    async.series([
-        processApp, // this is first because docker.getEvents seems to get 'stuck' sometimes
-        processDockerEvents.bind(null, intervalSecs)
-    ], function (error) {
-        if (error) debug(`run: could not check app health. ${error.message}`);
-
-        callback();
-    });
+    await processApp({ timeout: (intervalSecs - 3) * 1000 });
+    await processDockerEvents({ intervalSecs, timeout: 3000 });
 }

src/apppasswords.js

@@ -0,0 +1,88 @@
+'use strict';
+
+exports = module.exports = {
+    get,
+    add,
+    list,
+    del,
+
+    removePrivateFields
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    crypto = require('crypto'),
+    database = require('./database.js'),
+    hat = require('./hat.js'),
+    safe = require('safetydance'),
+    uuid = require('uuid'),
+    _ = require('underscore');
+
+const APP_PASSWORD_FIELDS = [ 'id', 'name', 'userId', 'identifier', 'hashedPassword', 'creationTime' ].join(',');
+
+function validateAppPasswordName(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 char');
+    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'name too long');
+
+    return null;
+}
+
+function removePrivateFields(appPassword) {
+    return _.pick(appPassword, 'id', 'name', 'userId', 'identifier', 'creationTime');
+}
+
+async function get(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE id = ?', [ id ]);
+    if (result.length === 0) return null;
+    return result[0];
+}
+
+async function add(userId, identifier, name) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof name, 'string');
+
+    let error = validateAppPasswordName(name);
+    if (error) throw error;
+
+    if (identifier.length < 1) throw new BoxError(BoxError.BAD_FIELD, 'identifier must be atleast 1 char');
+
+    const password = hat(16 * 4);
+    const hashedPassword = crypto.createHash('sha256').update(password).digest('base64');
+
+    const appPassword = {
+        id: 'uid-' + uuid.v4(),
+        name,
+        userId,
+        identifier,
+        password,
+        hashedPassword
+    };
+
+    const query = 'INSERT INTO appPasswords (id, userId, identifier, name, hashedPassword) VALUES (?, ?, ?, ?, ?)';
+    const args = [ appPassword.id, appPassword.userId, appPassword.identifier, appPassword.name, appPassword.hashedPassword ];
+
+    [error] = await safe(database.query(query, args));
+    if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('appPasswords_name_userId_identifier') !== -1) throw new BoxError(BoxError.ALREADY_EXISTS, 'name/app combination already exists');
+    if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.indexOf('userId')) throw new BoxError(BoxError.NOT_FOUND, 'user not found');
+    if (error) throw error;
+
+    return { id: appPassword.id, password: appPassword.password };
+}
+
+async function list(userId) {
+    assert.strictEqual(typeof userId, 'string');
+
+    return await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE userId = ?', [ userId ]);
+}
+
+async function del(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query('DELETE FROM appPasswords WHERE id = ?', [ id ]);
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'password not found');
+}

src/appstore.js

@@ -13,7 +13,7 @@ exports = module.exports = {
     purchaseApp,
     unpurchaseApp,
 
-    getUserToken,
+    createUserToken,
     getSubscription,
     isFreePlan,
 
@@ -23,9 +23,8 @@ exports = module.exports = {
     createTicket
 };
 
-var apps = require('./apps.js'),
+const apps = require('./apps.js'),
     assert = require('assert'),
-    async = require('async'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:appstore'),
@@ -35,6 +34,7 @@ var apps = require('./apps.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     settings = require('./settings.js'),
+    sysinfo = require('./sysinfo.js'),
     superagent = require('superagent'),
     support = require('./support.js'),
     util = require('util');
@@ -50,7 +50,7 @@ let gFeatures = {
     privateDockerRegistry: false,
     branding: false,
     support: false,
-    directoryConfig: false,
+    profileConfig: false,
     mailboxMaxCount: 5,
     emailPremium: false
 };
@@ -74,99 +74,86 @@ function isAppAllowed(appstoreId, listingConfig) {
     return true;
 }
 
-function getCloudronToken(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    settings.getCloudronToken(function (error, token) {
-        if (error) return callback(error);
-        if (!token) return callback(new BoxError(BoxError.LICENSE_ERROR, 'Missing token'));
-
-        callback(null, token);
-    });
-}
-
-function login(email, password, totpToken, callback) {
+async function login(email, password, totpToken) {
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof totpToken, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var data = {
-        email: email,
-        password: password,
-        totpToken: totpToken
-    };
+    const data = { email, password, totpToken };
 
     const url = settings.apiServerOrigin() + '/api/v1/login';
-    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${result.statusCode}`));
+    const [error, response] = await safe(superagent.post(url)
+        .send(data)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-        callback(null, result.body); // { userId, accessToken }
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${response.status}`);
+
+    return response.body; // { userId, accessToken }
 }
 
-function registerUser(email, password, callback) {
+async function registerUser(email, password) {
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof password, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var data = {
-        email: email,
-        password: password,
-    };
+    const data = { email, password };
 
     const url = settings.apiServerOrigin() + '/api/v1/register_user';
-    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${result.statusCode}`));
+    const [error, response] = await safe(superagent.post(url)
+        .send(data)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-        callback(null);
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+
+    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, 'account already exists');
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${response.status}`);
 }
 
-function getUserToken(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function createUserToken() {
+    if (settings.isDemo()) throw new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode');
 
-    if (settings.isDemo()) return callback(new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode'));
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
 
-        const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
+    const [error, response] = await safe(superagent.post(url)
+        .send({})
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-        superagent.post(url).send({}).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${result.status}`));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${response.status}`);
 
-            callback(null, result.body.accessToken);
-        });
-    });
+    return response.body.accessToken;
 }
 
-function getSubscription(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function getSubscription() {
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const url = settings.apiServerOrigin() + '/api/v1/subscription';
 
-        const url = settings.apiServerOrigin() + '/api/v1/subscription';
-        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR));
-            if (result.statusCode === 502) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`));
+    const [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            // update the features cache
-            gFeatures = result.body.features;
-            safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR);
+    if (response.status === 502) throw new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`);
 
-            callback(null, result.body);
-        });
-    });
+    // update the features cache
+    gFeatures = response.body.features;
+    safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
+
+    return response.body;
 }
 
 function isFreePlan(subscription) {
@@ -174,225 +161,212 @@ function isFreePlan(subscription) {
 }
 
 // See app.js install it will create a db record first but remove it again if appstore purchase fails
-function purchaseApp(data, callback) {
+async function purchaseApp(data) {
     assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId, appId }
     assert(data.appstoreId || data.manifestId);
     assert.strictEqual(typeof data.appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps`;
+    const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps`;
 
-        superagent.post(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); // appstoreId does not exist
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 402) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            // 200 if already purchased, 201 is newly purchased
-            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', result.status, result.body)));
+    const [error, response] = await safe(superagent.post(url)
+        .send(data)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            callback(null);
-        });
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND); // appstoreId does not exist
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    // 200 if already purchased, 201 is newly purchased
+    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', response.status, response.body));
 }
 
-function unpurchaseApp(appId, data, callback) {
+async function unpurchaseApp(appId, data) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId }
     assert(data.appstoreId || data.manifestId);
-    assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
+    const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
 
-        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 404) return callback(null);   // was never purchased
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
+    let [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            superagent.del(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
-                if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 404) return;   // was never purchased
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
 
-                callback(null);
-            });
-        });
-    });
+    [error, response] = await safe(superagent.del(url)
+        .send(data)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
+
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
 }
 
-function getBoxUpdate(options, callback) {
+async function getBoxUpdate(options) {
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-        const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
+    const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
 
-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            automatic: options.automatic
-        };
+    const query = {
+        accessToken: token,
+        boxVersion: constants.VERSION,
+        automatic: options.automatic
+    };
 
-        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 204) return callback(null, null); // no update
-            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+    const [error, response] = await safe(superagent.get(url)
+        .query(query)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            var updateInfo = result.body;
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status === 204) return; // no update
+    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-            if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', result.statusCode, result.text)));
-            }
+    const updateInfo = response.body;
 
-            // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
-            if (!updateInfo.version || typeof updateInfo.version !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', result.statusCode, result.text)));
+    if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
+        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', response.status, response.text));
+    }
 
-            callback(null, updateInfo);
-        });
-    });
+    // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
+    if (!updateInfo.version || typeof updateInfo.version !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
+    if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
+    if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', response.status, response.text));
+    if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', response.status, response.text));
+    if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', response.status, response.text));
+    if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', response.status, response.text));
+
+    return updateInfo;
 }
 
-function getAppUpdate(app, options, callback) {
+async function getAppUpdate(app, options) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-        const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            appId: app.appStoreId,
-            appVersion: app.manifest.version,
-            automatic: options.automatic
-        };
+    const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
+    const query = {
+        accessToken: token,
+        boxVersion: constants.VERSION,
+        appId: app.appStoreId,
+        appVersion: app.manifest.version,
+        automatic: options.automatic
+    };
 
-        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 204) return callback(null); // no update
-            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+    const [error, response] = await safe(superagent.get(url)
+        .query(query)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            const updateInfo = result.body;
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status === 204) return; // no update
+    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-            // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
-            const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
+    const updateInfo = response.body;
 
-            // do some sanity checks
-            if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
-                debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', result.statusCode, result.text)));
-            }
+    // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
+    const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
 
-            updateInfo.unstable = !!updateInfo.unstable;
+    // do some sanity checks
+    if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
+        debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
+        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', response.status, response.text));
+    }
 
-            // { id, creationDate, manifest, unstable }
-            callback(null, updateInfo);
-        });
-    });
+    updateInfo.unstable = !!updateInfo.unstable;
+
+    // { id, creationDate, manifest, unstable }
+    return updateInfo;
 }
 
-function registerCloudron(data, callback) {
+async function registerCloudron(data) {
     assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     const url = `${settings.apiServerOrigin()}/api/v1/register_cloudron`;
 
-    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${result.statusCode} ${error.message}`));
+    const [error, response] = await safe(superagent.post(url)
+        .send(data)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-        // cloudronId, token, licenseKey
-        if (!result.body.cloudronId) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id'));
-        if (!result.body.cloudronToken) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token'));
-        if (!result.body.licenseKey) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no license'));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${response.statusCode} ${error.message}`);
 
-        async.series([
-            settings.setCloudronId.bind(null, result.body.cloudronId),
-            settings.setCloudronToken.bind(null, result.body.cloudronToken),
-            settings.setLicenseKey.bind(null, result.body.licenseKey),
-        ], function (error) {
-            if (error) return callback(error);
+    // cloudronId, token, licenseKey
+    if (!response.body.cloudronId) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id');
+    if (!response.body.cloudronToken) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token');
+    if (!response.body.licenseKey) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no license');
 
-            debug(`registerCloudron: Cloudron registered with id ${result.body.cloudronId}`);
+    await settings.setCloudronId(response.body.cloudronId);
+    await settings.setCloudronToken(response.body.cloudronToken);
+    await settings.setLicenseKey(response.body.licenseKey);
 
-            callback();
-        });
-    });
+    debug(`registerCloudron: Cloudron registered with id ${response.body.cloudronId}`);
 }
 
-function updateCloudron(data, callback) {
+async function updateCloudron(data) {
     assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
-        if (error && error.reason === BoxError.LICENSE_ERROR) return callback(null); // missing token. not registered yet
-        if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-        const url = `${settings.apiServerOrigin()}/api/v1/update_cloudron`;
-        const query = {
-            accessToken: token
-        };
+    const url = `${settings.apiServerOrigin()}/api/v1/update_cloudron`;
+    const query = {
+        accessToken: token
+    };
 
-        superagent.post(url).query(query).send(data).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+    const [error, response] = await safe(superagent.post(url)
+        .query(query)
+        .send(data)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-            callback();
-        });
-    });
+    debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
 }
 
-function registerWithLoginCredentials(options, callback) {
+async function registerWithLoginCredentials(options) {
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    function maybeSignup(done) {
-        if (!options.signup) return done();
+    const token = await settings.getCloudronToken();
+    if (token) throw new BoxError(BoxError.CONFLICT, 'Cloudron is already registered');
 
-        registerUser(options.email, options.password, done);
-    }
+    if (options.signup) await registerUser(options.email, options.password);
 
-    getCloudronToken(function (error, token) {
-        if (token) return callback(new BoxError(BoxError.CONFLICT, 'Cloudron is already registered'));
-
-        maybeSignup(function (error) {
-            if (error) return callback(error);
-
-            login(options.email, options.password, options.totpToken || '', function (error, result) {
-                if (error) return callback(error);
-
-                registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
-            });
-        });
-    });
+    const result = await login(options.email, options.password, options.totpToken || '');
+    await registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION });
 }
 
-function createTicket(info, auditSource, callback) {
+async function createTicket(info, auditSource) {
     assert.strictEqual(typeof info, 'object');
     assert.strictEqual(typeof info.email, 'string');
     assert.strictEqual(typeof info.displayName, 'string');
@@ -400,128 +374,101 @@ function createTicket(info, auditSource, callback) {
     assert.strictEqual(typeof info.subject, 'string');
     assert.strictEqual(typeof info.description, 'string');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    function collectAppInfoIfNeeded(callback) {
-        if (!info.appId) return callback();
-        apps.get(info.appId, callback);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+
+    if (info.enableSshSupport) {
+        await safe(support.enableRemoteSupport(true, auditSource));
+        info.ipv4 = await sysinfo.getServerIPv4();
     }
 
-    function enableSshIfNeeded(callback) {
-        if (!info.enableSshSupport) return callback();
+    info.app = info.appId ? await apps.get(info.appId) : null;
+    info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
 
-        support.enableRemoteSupport(true, auditSource, function (error) {
-            // ensure we can at least get the ticket through
-            if (error) debug('Unable to enable SSH support.', error);
+    const request = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true);
 
-            callback();
-        });
+    // either send as JSON through body or as multipart, depending on attachments
+    if (info.app) {
+        request.field('infoJSON', JSON.stringify(info));
+
+        const logPaths = await apps.getLogPaths(info.app);
+        for (const logPath of logPaths) {
+            const logs = safe.child_process.execSync(`tail --lines=1000 ${logPath}`);
+            if (logs) request.attach(path.basename(logPath), logs, path.basename(logPath));
+        }
+    } else {
+        request.send(info);
     }
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const [error, response] = await safe(request);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-        enableSshIfNeeded(function (error) {
-            if (error) return callback(error);
+    await eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
 
-            collectAppInfoIfNeeded(function (error, app) {
-                if (error) return callback(error);
-                if (app) info.app = app;
-
-                info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
-
-                var req = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
-                    .query({ accessToken: token })
-                    .timeout(30 * 1000);
-
-                // either send as JSON through body or as multipart, depending on attachments
-                if (info.app) {
-                    req.field('infoJSON', JSON.stringify(info));
-
-                    apps.getLocalLogfilePaths(info.app).forEach(function (filePath) {
-                        var logs = safe.child_process.execSync(`tail --lines=1000 ${filePath}`);
-                        if (logs) req.attach(path.basename(filePath), logs, path.basename(filePath));
-                    });
-                } else {
-                    req.send(info);
-                }
-
-                req.end(function (error, result) {
-                    if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                    if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                    if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                    if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
-
-                    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
-
-                    callback(null, { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
-                });
-            });
-        });
-    });
+    return { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` };
 }
 
-function getApps(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function getApps() {
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const unstable = await settings.getUnstableAppsConfig();
 
-        settings.getUnstableAppsConfig(function (error, unstable) {
-            if (error) return callback(error);
+    const url = `${settings.apiServerOrigin()}/api/v1/apps`;
 
-            const url = `${settings.apiServerOrigin()}/api/v1/apps`;
-            superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable }).timeout(30 * 1000).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
-                if (!result.body.apps) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+    const [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-                settings.getAppstoreListingConfig(function (error, listingConfig) {
-                    if (error) return callback(error);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 403 || response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', response.status, response.body));
+    if (!response.body.apps) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-                    const filteredApps = result.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
-
-                    callback(null, filteredApps);
-                });
-            });
-        });
-    });
+    const listingConfig = await settings.getAppstoreListingConfig();
+    const filteredApps = response.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
+    return filteredApps;
 }
 
-function getAppVersion(appId, version, callback) {
+async function getAppVersion(appId, version) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof version, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    settings.getAppstoreListingConfig(function (error, listingConfig) {
-        if (error) return callback(error);
+    const listingConfig = await settings.getAppstoreListingConfig();
 
-        if (!isAppAllowed(appId, listingConfig)) return callback(new BoxError(BoxError.FEATURE_DISABLED));
+    if (!isAppAllowed(appId, listingConfig)) throw new BoxError(BoxError.FEATURE_DISABLED);
 
-        getCloudronToken(function (error, token) {
-            if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-            let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
-            if (version !== 'latest') url += `/versions/${version}`;
+    let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
+    if (version !== 'latest') url += `/versions/${version}`;
 
-            superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
-                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
+    const [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-                callback(null, result.body);
-            });
-        });
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 403 || response.statusCode === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', response.status, response.body));
+
+    return response.body;
 }
 
-function getApp(appId, callback) {
+async function getApp(appId) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    getAppVersion(appId, 'latest', callback);
+    return await getAppVersion(appId, 'latest');
 }

src/apptaskmanager.js

@@ -20,12 +20,11 @@ let gPendingTasks = [ ];
 let gInitialized = false;
 
 const TASK_CONCURRENCY = 3;
-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 function waitText(lockOperation) {
     if (lockOperation === locker.OP_BOX_UPDATE) return 'Waiting for Cloudron to finish updating. See the Settings view';
     if (lockOperation === locker.OP_PLATFORM_START) return 'Waiting for Cloudron to initialize';
-    if (lockOperation === locker.OP_FULL_BACKUP) return 'Wait for Cloudron to finish backup. See the Backups view';
+    if (lockOperation === locker.OP_FULL_BACKUP) return 'Waiting for Cloudron to finish backup. See the Backups view';
 
     return ''; // cannot happen
 }
@@ -36,31 +35,31 @@ function initializeSync() {
 }
 
 // callback is called when task is finished
-function scheduleTask(appId, taskId, options, callback) {
+function scheduleTask(appId, taskId, options, onFinished) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof taskId, 'string');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
+    assert.strictEqual(typeof onFinished, 'function');
 
     if (!gInitialized) initializeSync();
 
     if (appId in gActiveTasks) {
-        return callback(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
+        return onFinished(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
     }
 
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, options, callback });
+        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' });
+        gPendingTasks.push({ appId, taskId, options, onFinished });
         return;
     }
 
-    var lockError = locker.recursiveLock(locker.OP_APPTASK);
+    const lockError = locker.recursiveLock(locker.OP_APPTASK);
 
     if (lockError) {
         debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, options, callback });
+        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) });
+        gPendingTasks.push({ appId, taskId, options, onFinished });
         return;
     }
 
@@ -73,7 +72,7 @@ function scheduleTask(appId, taskId, options, callback) {
     scheduler.suspendJobs(appId);
 
     tasks.startTask(taskId, Object.assign(options, { logFile }), function (error, result) {
-        callback(error, result);
+        onFinished(error, result);
 
         delete gActiveTasks[appId];
         locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
@@ -88,5 +87,5 @@ function startNextTask() {
     assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
     const t = gPendingTasks.shift();
-    scheduleTask(t.appId, t.taskId, t.options, t.callback);
+    scheduleTask(t.appId, t.taskId, t.options, t.onFinished);
 }

src/auditsource.js

@@ -1,15 +1,24 @@
 'use strict';
 
-exports = module.exports = {
-    CRON:  { userId: null, username: 'cron' },
-    HEALTH_MONITOR: { userId: null, username: 'healthmonitor' },
-    EXTERNAL_LDAP_TASK: { userId: null, username: 'externalldap' },
-    EXTERNAL_LDAP_AUTO_CREATE: { userId: null, username: 'externalldap' },
+class AuditSource {
+    constructor(username, userId, ip) {
+        this.username = username;
+        this.userId = userId || null;
+        this.ip = ip || null;
+    }
 
-    fromRequest
-};
-
-function fromRequest(req) {
-    const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-    return { ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+    static fromRequest(req) {
+        const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+        return new AuditSource(req.user?.username, req.user?.id, ip);
+    }
 }
+
+// these can be static variables but see https://stackoverflow.com/questions/60046847/eslint-does-not-allow-static-class-properties#comment122122927_60464446
+AuditSource.CRON =  new AuditSource('cron');
+AuditSource.HEALTH_MONITOR = new AuditSource('healthmonitor');
+AuditSource.EXTERNAL_LDAP_TASK = new AuditSource('externalldap');
+AuditSource.EXTERNAL_LDAP_AUTO_CREATE = new AuditSource('externalldap');
+AuditSource.APPTASK = new AuditSource('apptask');
+AuditSource.PLATFORM = new AuditSource('platform');
+
+exports = module.exports = AuditSource;

src/autoconfig.xml.ejs

@@ -19,7 +19,13 @@
       <username>%EMAILADDRESS%</username>
       <addThisServer>true</addThisServer>
     </outgoingServer>
-
+    <incomingServer type="pop3">
+      <hostname><%= mailFqdn %></hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <username>%EMAILADDRESS%</username>
+      <authentication>password-cleartext</authentication>
+    </incomingServer>
     <documentation url="http://cloudron.io/email/#autodiscover">
        <descr lang="en">Cloudron Email</descr>
     </documentation>

src/backupcleaner.js

@@ -0,0 +1,306 @@
+'use strict';
+
+exports = module.exports = {
+    run,
+
+    _applyBackupRetentionPolicy: applyBackupRetentionPolicy
+};
+
+const apps = require('./apps.js'),
+    assert = require('assert'),
+    backups = require('./backups.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:backupcleaner'),
+    moment = require('moment'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    settings = require('./settings.js'),
+    storage = require('./storage.js'),
+    util = require('util'),
+    _ = require('underscore');
+
+function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
+    assert(Array.isArray(allBackups));
+    assert.strictEqual(typeof policy, 'object');
+    assert(Array.isArray(referencedBackupIds));
+
+    const now = new Date();
+
+    for (const backup of allBackups) {
+        if (backup.state === backups.BACKUP_STATE_ERROR) {
+            backup.discardReason = 'error';
+        } else if (backup.state === backups.BACKUP_STATE_CREATING) {
+            if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
+            else backup.discardReason = 'creating-too-long';
+        } else if (referencedBackupIds.includes(backup.id)) {
+            backup.keepReason = 'reference';
+        } else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) {
+            backup.keepReason = 'preserveSecs';
+        } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) {
+            backup.keepReason = 'keepWithinSecs';
+        }
+    }
+
+    const KEEP_FORMATS = {
+        keepDaily: 'Y-M-D',
+        keepWeekly: 'Y-W',
+        keepMonthly: 'Y-M',
+        keepYearly: 'Y'
+    };
+
+    for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) {
+        if (!(format in policy)) continue;
+
+        const n = policy[format]; // we want to keep "n" backups of format
+        if (!n) continue; // disabled rule
+
+        let lastPeriod = null, keptSoFar = 0;
+        for (const backup of allBackups) {
+            if (backup.discardReason) continue; // already discarded for some reason
+            if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason
+            const period = moment(backup.creationTime).format(KEEP_FORMATS[format]);
+            if (period === lastPeriod) continue; // already kept for this period
+
+            lastPeriod = period;
+            backup.keepReason = backup.keepReason ? `${backup.keepReason}+${format}` : format;
+            if (++keptSoFar === n) break;
+        }
+    }
+
+    if (policy.keepLatest) {
+        let latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL);
+        if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest';
+    }
+
+    for (const backup of allBackups) {
+        debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
+    }
+}
+
+async function cleanupBackup(backupConfig, backup, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof backup, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
+
+    return new Promise((resolve) => {
+        function done(error) {
+            if (error) {
+                debug('cleanupBackup: error removing backup %j : %s', backup, error.message);
+                return resolve();
+            }
+
+            // prune empty directory if possible
+            storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), async function (error) {
+                if (error) debug('cleanupBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message);
+
+                const [delError] = await safe(backups.del(backup.id));
+                if (delError) debug('cleanupBackup: error removing from database', delError);
+                else debug('cleanupBackup: removed %s', backup.id);
+
+                resolve();
+            });
+        }
+
+        if (backup.format ==='tgz') {
+            progressCallback({ message: `${backup.id}: Removing ${backupFilePath}`});
+            storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, done);
+        } else {
+            const events = storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath);
+            events.on('progress', (message) => progressCallback({ message: `${backup.id}: ${message}` }));
+            events.on('done', done);
+        }
+    });
+}
+
+async function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert(Array.isArray(referencedAppBackupIds));
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    let removedAppBackupIds = [];
+
+    const allApps = await apps.list();
+    const allAppIds = allApps.map(a => a.id);
+
+    const appBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000);
+
+    // collate the backups by app id. note that the app could already have been uninstalled
+    let appBackupsById = {};
+    for (const appBackup of appBackups) {
+        if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = [];
+        appBackupsById[appBackup.identifier].push(appBackup);
+    }
+
+    // apply backup policy per app. keep latest backup only for existing apps
+    let appBackupsToRemove = [];
+    for (const appId of Object.keys(appBackupsById)) {
+        applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds);
+        appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
+    }
+
+    for (const appBackup of appBackupsToRemove) {
+        await progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
+        removedAppBackupIds.push(appBackup.id);
+        await cleanupBackup(backupConfig, appBackup, progressCallback); // never errors
+    }
+
+    debug('cleanupAppBackups: done');
+
+    return removedAppBackupIds;
+}
+
+async function cleanupMailBackups(backupConfig, referencedAppBackupIds, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert(Array.isArray(referencedAppBackupIds));
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    let removedMailBackupIds = [];
+
+    const mailBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_MAIL, 1, 1000);
+
+    applyBackupRetentionPolicy(mailBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), referencedAppBackupIds);
+
+    for (const mailBackup of mailBackups) {
+        if (mailBackup.keepReason) continue;
+        await progressCallback({ message: `Removing mail backup ${mailBackup.id}`});
+        removedMailBackupIds.push(mailBackup.id);
+        await cleanupBackup(backupConfig, mailBackup, progressCallback); // never errors
+    }
+
+    debug('cleanupMailBackups: done');
+
+    return removedMailBackupIds;
+}
+
+async function cleanupBoxBackups(backupConfig, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    let referencedAppBackupIds = [], removedBoxBackupIds = [];
+
+    const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
+
+    applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */);
+
+    for (const boxBackup of boxBackups) {
+        if (boxBackup.keepReason) {
+            referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn);
+            continue;
+        }
+
+        await progressCallback({ message: `Removing box backup ${boxBackup.id}`});
+
+        removedBoxBackupIds.push(boxBackup.id);
+        await cleanupBackup(backupConfig, boxBackup, progressCallback);
+    }
+
+    debug('cleanupBoxBackups: done');
+
+    return { removedBoxBackupIds, referencedAppBackupIds };
+}
+
+async function cleanupMissingBackups(backupConfig, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const perPage = 1000;
+    let missingBackupIds = [];
+    const backupExists = util.promisify(storage.api(backupConfig.provider).exists);
+
+    if (constants.TEST) return missingBackupIds;
+
+    let page = 1, result = [];
+    do {
+        result = await backups.list(page, perPage);
+
+        for (const backup of result) {
+            let backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
+            if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
+
+            const [existsError, exists] = await safe(backupExists(backupConfig, backupFilePath));
+            if (existsError || exists) continue;
+
+            await progressCallback({ message: `Removing missing backup ${backup.id}`});
+
+            const [delError] = await safe(backups.del(backup.id));
+            if (delError) debug(`cleanupBackup: error removing ${backup.id} from database`, delError);
+
+            missingBackupIds.push(backup.id);
+        }
+
+        ++ page;
+    } while (result.length === perPage);
+
+    debug('cleanupMissingBackups: done');
+
+    return missingBackupIds;
+}
+
+// removes the snapshots of apps that have been uninstalled
+async function cleanupSnapshots(backupConfig) {
+    assert.strictEqual(typeof backupConfig, 'object');
+
+    const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
+    const info = safe.JSON.parse(contents);
+    if (!info) return;
+
+    delete info.box;
+
+    for (const appId of Object.keys(info)) {
+        const app = await apps.get(appId);
+        if (app) continue; // app is still installed
+
+        await new Promise((resolve) => {
+            async function done(/* ignoredError */) {
+                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
+                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`));
+
+                await safe(backups.setSnapshotInfo(appId, null /* info */), { debug });
+                debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`);
+
+                resolve();
+            }
+
+            if (info[appId].format ==='tgz') {
+                storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), done);
+            } else {
+                const events = storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format));
+                events.on('progress', function (detail) { debug(`cleanupSnapshots: ${detail}`); });
+                events.on('done', done);
+            }
+        });
+    }
+
+    debug('cleanupSnapshots: done');
+}
+
+async function run(progressCallback) {
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const backupConfig = await settings.getBackupConfig();
+
+    if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
+        debug('cleanup: keeping all backups');
+        return {};
+    }
+
+    await progressCallback({ percent: 10, message: 'Cleaning box backups' });
+    const { removedBoxBackupIds, referencedAppBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback);
+
+    await progressCallback({ percent: 20, message: 'Cleaning mail backups' });
+    const removedMailBackupIds = await cleanupMailBackups(backupConfig, referencedAppBackupIds, progressCallback);
+
+    await progressCallback({ percent: 40, message: 'Cleaning app backups' });
+    const removedAppBackupIds = await cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback);
+
+    await progressCallback({ percent: 70, message: 'Cleaning missing backups' });
+    const missingBackupIds = await cleanupMissingBackups(backupConfig, progressCallback);
+
+    await progressCallback({ percent: 90, message: 'Cleaning snapshots' });
+    await cleanupSnapshots(backupConfig);
+
+    return { removedBoxBackupIds, removedMailBackupIds, removedAppBackupIds, missingBackupIds };
+}

src/backupdb.js

@@ -1,176 +0,0 @@
-'use strict';
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js'),
-    safe = require('safetydance');
-
-const BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
-
-exports = module.exports = {
-    add,
-
-    getByTypePaged,
-    getByIdentifierPaged,
-    getByIdentifierAndStatePaged,
-
-    get,
-    del,
-    update,
-    list,
-
-    _clear: clear
-};
-
-function postProcess(result) {
-    assert.strictEqual(typeof result, 'object');
-
-    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
-
-    result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
-    delete result.manifestJson;
-}
-
-function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof state, 'string');
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, state, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function getByTypePaged(type, page, perPage, callback) {
-    assert.strictEqual(typeof type, 'string');
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ type, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function getByIdentifierPaged(identifier, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function list(page, perPage, callback) {
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups ORDER BY creationTime DESC LIMIT ?,?',
-        [ (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
-        [ id ], function (error, result) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-            if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
-
-            postProcess(result[0]);
-
-            callback(null, result[0]);
-        });
-}
-
-function add(id, data, callback) {
-    assert(data && typeof data === 'object');
-    assert.strictEqual(typeof id, 'string');
-    assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number');
-    assert.strictEqual(typeof data.packageVersion, 'string');
-    assert.strictEqual(typeof data.type, 'string');
-    assert.strictEqual(typeof data.identifier, 'string');
-    assert.strictEqual(typeof data.state, 'string');
-    assert(Array.isArray(data.dependsOn));
-    assert.strictEqual(typeof data.manifest, 'object');
-    assert.strictEqual(typeof data.format, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var creationTime = data.creationTime || new Date(); // allow tests to set the time
-    var manifestJson = JSON.stringify(data.manifest);
-
-    database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ],
-        function (error) {
-            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            callback(null);
-        });
-}
-
-function update(id, backup, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof backup, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var fields = [ ], values = [ ];
-    for (var p in backup) {
-        fields.push(p + ' = ?');
-        values.push(backup[p]);
-    }
-    values.push(id);
-
-    database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('TRUNCATE TABLE backups', [], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        callback(null);
-    });
-}

src/blobs.js

@@ -4,16 +4,16 @@
 
 exports = module.exports = {
     get,
+    getString,
     set,
+    setString,
     del,
 
-    initSecrets,
-
     ACME_ACCOUNT_KEY: 'acme_account_key',
     ADDON_TURN_SECRET: 'addon_turn_secret',
-    DHPARAMS: 'dhparams',
     SFTP_PUBLIC_KEY: 'sftp_public_key',
     SFTP_PRIVATE_KEY: 'sftp_private_key',
+    PROXY_AUTH_TOKEN_SECRET: 'proxy_auth_token_secret',
 
     CERT_PREFIX: 'cert',
 
@@ -21,13 +21,7 @@ exports = module.exports = {
 };
 
 const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    constants = require('./constants.js'),
-    crypto = require('crypto'),
-    database = require('./database.js'),
-    debug = require('debug')('box:blobs'),
-    paths = require('./paths.js'),
-    safe = require('safetydance');
+    database = require('./database.js');
 
 const BLOBS_FIELDS = [ 'id', 'value' ].join(',');
 
@@ -39,6 +33,14 @@ async function get(id) {
     return result[0].value;
 }
 
+async function getString(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query(`SELECT ${BLOBS_FIELDS} FROM blobs WHERE id = ?`, [ id ]);
+    if (result.length === 0) return null;
+    return result[0].value.toString('utf8');
+}
+
 async function set(id, value) {
     assert.strictEqual(typeof id, 'string');
     assert(value === null || Buffer.isBuffer(value));
@@ -46,6 +48,13 @@ async function set(id, value) {
     await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, value ]);
 }
 
+async function setString(id, value) {
+    assert.strictEqual(typeof id, 'string');
+    assert(value === null || typeof value === 'string');
+
+    await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, Buffer.from(value) ]);
+}
+
 async function del(id) {
     await database.query('DELETE FROM blobs WHERE id=?', [ id ]);
 }
@@ -53,50 +62,3 @@ async function del(id) {
 async function clear() {
     await database.query('DELETE FROM blobs');
 }
-
-async function initSecrets() {
-    let acmeAccountKey = await get(exports.ACME_ACCOUNT_KEY);
-    if (!acmeAccountKey) {
-        acmeAccountKey = safe.child_process.execSync('openssl genrsa 4096');
-        if (!acmeAccountKey) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate acme account key: ${safe.error.message}`);
-        await set(exports.ACME_ACCOUNT_KEY, acmeAccountKey);
-    }
-
-    let turnSecret = await get(exports.ADDON_TURN_SECRET);
-    if (!turnSecret) {
-        turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter
-        await set(exports.ADDON_TURN_SECRET, Buffer.from(turnSecret));
-    }
-
-    if (!constants.TEST) {
-        let dhparams = await get(exports.DHPARAMS);
-        if (!dhparams) {
-            debug('initSecrets: generating dhparams.pem. this takes forever');
-            dhparams = safe.child_process.execSync('openssl dhparam 2048');
-            if (!dhparams) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
-            if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
-            await set(exports.DHPARAMS, dhparams);
-        } else if (!safe.fs.existsSync(paths.DHPARAMS_FILE)) {
-            if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
-        }
-    }
-
-    let sftpPrivateKey = await get(exports.SFTP_PRIVATE_KEY);
-    let sftpPublicKey = await get(exports.SFTP_PUBLIC_KEY);
-
-    if (!sftpPrivateKey || !sftpPublicKey) {
-        debug('initSecrets: generate sftp keys');
-        if (constants.TEST) {
-            safe.fs.unlinkSync(paths.SFTP_PUBLIC_KEY_FILE);
-            safe.fs.unlinkSync(paths.SFTP_PRIVATE_KEY_FILE);
-        }
-        if (!safe.child_process.execSync(`ssh-keygen -m PEM -t rsa -f "${paths.SFTP_KEYS_DIR}/ssh_host_rsa_key" -q -N ""`)) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate sftp ssh keys: ${safe.error.message}`);
-        sftpPublicKey = safe.fs.readFileSync(paths.SFTP_PUBLIC_KEY_FILE);
-        await set(exports.SFTP_PUBLIC_KEY, sftpPublicKey);
-        sftpPrivateKey = safe.fs.readFileSync(paths.SFTP_PRIVATE_KEY_FILE);
-        await set(exports.SFTP_PRIVATE_KEY, sftpPrivateKey);
-    } else if (!safe.fs.existsSync(paths.SFTP_PUBLIC_KEY_FILE) || !safe.fs.existsSync(paths.SFTP_PRIVATE_KEY_FILE)) {
-        if (!safe.fs.writeFileSync(paths.SFTP_PUBLIC_KEY_FILE, sftpPublicKey)) throw new BoxError(BoxError.FS_ERROR, `Could not save sftp public key: ${safe.error.message}`);
-        if (!safe.fs.writeFileSync(paths.SFTP_PRIVATE_KEY_FILE, sftpPrivateKey)) throw new BoxError(BoxError.FS_ERROR, `Could not save sftp private key: ${safe.error.message}`);
-    }
-}

src/boxerror.js

@@ -33,6 +33,7 @@ function BoxError(reason, errorOrMessage, override) {
 }
 util.inherits(BoxError, Error);
 BoxError.ACCESS_DENIED = 'Access Denied';
+BoxError.ACME_ERROR = 'Acme Error';
 BoxError.ADDONS_ERROR = 'Addons Error';
 BoxError.ALREADY_EXISTS = 'Already Exists';
 BoxError.BAD_FIELD = 'Bad Field';
@@ -60,6 +61,7 @@ BoxError.NGINX_ERROR = 'Nginx Error';
 BoxError.NOT_FOUND = 'Not found';
 BoxError.NOT_IMPLEMENTED = 'Not implemented';
 BoxError.NOT_SIGNED = 'Not Signed';
+BoxError.NOT_SUPPORTED = 'Not Supported';
 BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.PLAN_LIMIT = 'Plan Limit';
 BoxError.SPAWN_ERROR = 'Spawn Error';
@@ -85,10 +87,12 @@ BoxError.toHttpError = function (error) {
     case BoxError.ALREADY_EXISTS:
     case BoxError.BAD_STATE:
     case BoxError.CONFLICT:
+    case BoxError.NOT_SUPPORTED:
         return new HttpError(409, error);
     case BoxError.INVALID_CREDENTIALS:
         return new HttpError(412, error);
     case BoxError.EXTERNAL_ERROR:
+    case BoxError.ACME_ERROR:
     case BoxError.NETWORK_ERROR:
     case BoxError.FS_ERROR:
     case BoxError.MOUNT_ERROR:

src/cloudron.js

@@ -25,14 +25,16 @@ exports = module.exports = {
 const apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     branding = require('./branding.js'),
     constants = require('./constants.js'),
     cron = require('./cron.js'),
     debug = require('debug')('box:cloudron'),
+    delay = require('delay'),
+    dns = require('./dns.js'),
+    dockerProxy = require('./dockerproxy.js'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
@@ -54,196 +56,160 @@ const apps = require('./apps.js'),
 
 const REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');
 
-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 async function initialize() {
-    runStartupTasks();
+    safe(runStartupTasks(), { debug }); // background
 
     await notifyUpdate();
 }
 
-function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        cron.stopJobs,
-        platform.stopAllTasks
-    ], callback);
+async function uninitialize() {
+    await cron.stopJobs();
+    await dockerProxy.stop();
+    await platform.stopAllTasks();
 }
 
-function onActivated(options, callback) {
+async function onActivated(options) {
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     debug('onActivated: running post activation tasks');
 
     // Starting the platform after a user is available means:
     // 1. mail bounces can now be sent to the cloudron owner
     // 2. the restore code path can run without sudo (since mail/ is non-root)
-    async.series([
-        platform.start.bind(null, options),
-        cron.startJobs,
-        // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
-        // the UI some time to query the dashboard domain in the restore code path
-        (done) => setTimeout(() => reverseProxy.writeDefaultConfig({ activated :true }, done), 30000)
-    ], callback);
+    await platform.start(options);
+    await cron.startJobs();
+    await dockerProxy.start(); // this relies on the 'cloudron' docker network interface to be available
+
+    // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
+    // the UI some time to query the dashboard domain in the restore code path
+    await delay(30000);
+    await reverseProxy.writeDefaultConfig({ activated :true });
 }
 
 async function notifyUpdate() {
     const version = safe.fs.readFileSync(paths.VERSION_FILE, 'utf8');
     if (version === constants.VERSION) return;
 
-    await eventlog.add(eventlog.ACTION_UPDATE_FINISH, auditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
+    await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
 
-    return new Promise((resolve, reject) => {
-        tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }, function (error) {
-            if (error && error.reason !== BoxError.NOT_FOUND) return reject(error); // when hotfixing, task may not exist
+    const [error] = await safe(tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }));
+    if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist
 
-            safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
-
-            resolve();
-        });
-    });
+    safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
 }
 
 // each of these tasks can fail. we will add some routes to fix/re-run them
-function runStartupTasks() {
-    const tasks = [
-        // stop all the systemd tasks
-        platform.stopAllTasks,
+async function runStartupTasks() {
+    const tasks = [];
 
-        // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
-        function (callback) {
-            settings.getBackupConfig(function (error, backupConfig) {
-                if (error) return callback(error);
+    // stop all the systemd tasks
+    tasks.push(platform.stopAllTasks);
 
-                backups.configureCollectd(backupConfig, callback);
-            });
-        },
+    // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
+    tasks.push(async function () {
+        const backupConfig = await settings.getBackupConfig();
+        await backups.configureCollectd(backupConfig);
+    });
 
-        // always generate webadmin config since we have no versioning mechanism for the ejs
-        function (callback) {
-            if (!settings.dashboardDomain()) return callback();
+    // always generate webadmin config since we have no versioning mechanism for the ejs
+    tasks.push(async function () {
+        if (!settings.dashboardDomain()) return;
 
-            reverseProxy.writeDashboardConfig(settings.dashboardDomain(), callback);
-        },
+        const domainObject = await domains.get(settings.dashboardDomain());
+        await reverseProxy.writeDashboardConfig(domainObject);
+    });
 
+    tasks.push(async function () {
         // check activation state and start the platform
-        function (callback) {
-            users.isActivated(function (error, activated) {
-                if (error) return callback(error);
+        const activated = await users.isActivated();
 
-                // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
-                // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
-                // we remove the config as a simple security measure to not expose IP <-> domain
-                if (!activated) {
-                    debug('runStartupTasks: not activated. generating IP based redirection config');
-                    return reverseProxy.writeDefaultConfig({ activated: false }, callback);
-                }
-
-                onActivated({}, callback);
-            });
+        // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
+        // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
+        // we remove the config as a simple security measure to not expose IP <-> domain
+        if (!activated) {
+            debug('runStartupTasks: not activated. generating IP based redirection config');
+            return await reverseProxy.writeDefaultConfig({ activated: false });
         }
-    ];
+
+        await onActivated({});
+    });
 
     // we used to run tasks in parallel but simultaneous nginx reloads was causing issues
-    async.series(async.reflectAll(tasks), function (error, results) {
-        results.forEach((result, idx) => {
-            if (result.error) debug(`Startup task at index ${idx} failed: ${result.error.message}`);
-        });
-    });
+    for (let i = 0; i < tasks.length; i++) {
+        const [error] = await safe(tasks[i]());
+        if (error) debug(`Startup task at index ${i} failed: ${error.message}`);
+    }
 }
 
-function getConfig(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function getConfig() {
     const release = safe.fs.readFileSync('/etc/lsb-release', 'utf-8');
-    if (release === null) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
+    if (release === null) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
     const ubuntuVersion = release.match(/DISTRIB_DESCRIPTION="(.*)"/)[1];
 
-    settings.getAll(function (error, allSettings) {
-        if (error) return callback(error);
+    const allSettings = await settings.list();
 
-        // be picky about what we send out here since this is sent for 'normal' users as well
-        callback(null, {
-            apiServerOrigin: settings.apiServerOrigin(),
-            webServerOrigin: settings.webServerOrigin(),
-            adminDomain: settings.dashboardDomain(),
-            adminFqdn: settings.dashboardFqdn(),
-            mailFqdn: settings.mailFqdn(),
-            version: constants.VERSION,
-            ubuntuVersion,
-            isDemo: settings.isDemo(),
-            cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
-            footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
-            features: appstore.getFeatures(),
-            profileLocked: allSettings[settings.DIRECTORY_CONFIG_KEY].lockUserProfiles,
-            mandatory2FA: allSettings[settings.DIRECTORY_CONFIG_KEY].mandatory2FA
-        });
-    });
+    // be picky about what we send out here since this is sent for 'normal' users as well
+    return {
+        apiServerOrigin: settings.apiServerOrigin(),
+        webServerOrigin: settings.webServerOrigin(),
+        adminDomain: settings.dashboardDomain(),
+        adminFqdn: settings.dashboardFqdn(),
+        mailFqdn: settings.mailFqdn(),
+        version: constants.VERSION,
+        ubuntuVersion,
+        isDemo: settings.isDemo(),
+        cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
+        footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
+        features: appstore.getFeatures(),
+        profileLocked: allSettings[settings.PROFILE_CONFIG_KEY].lockUserProfiles,
+        mandatory2FA: allSettings[settings.PROFILE_CONFIG_KEY].mandatory2FA
+    };
 }
 
-function reboot(callback) {
-    notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '', function (error) {
-        if (error) debug('reboot: failed to clear reboot notification.', error);
+async function reboot() {
+    await notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '');
 
-        shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
-    });
+    const [error] = await safe(shell.promises.sudo('reboot', [ REBOOT_CMD ], {}));
+    if (error) debug('reboot: could not reboot', error);
 }
 
-function isRebootRequired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function isRebootRequired() {
     // https://serverfault.com/questions/92932/how-does-ubuntu-keep-track-of-the-system-restart-required-flag-in-motd
-    callback(null, fs.existsSync('/var/run/reboot-required'));
+    return fs.existsSync('/var/run/reboot-required');
 }
 
-// called from cron.js
-function runSystemChecks(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function runSystemChecks() {
     debug('runSystemChecks: checking status');
 
-    async.parallel([
-        checkMailStatus,
-        checkRebootRequired,
-        checkUbuntuVersion
-    ], callback);
+    const checks = [
+        checkMailStatus(),
+        checkRebootRequired(),
+        checkUbuntuVersion()
+    ];
+
+    await Promise.allSettled(checks);
 }
 
-function checkMailStatus(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    mail.checkConfiguration(function (error, message) {
-        if (error) return callback(error);
-
-        notifications.alert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly', message, callback);
-    });
+async function checkMailStatus() {
+    const message = await mail.checkConfiguration();
+    await notifications.alert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly', message);
 }
 
-function checkRebootRequired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    isRebootRequired(function (error, rebootRequired) {
-        if (error) return callback(error);
-
-        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '', callback);
-    });
+async function checkRebootRequired() {
+    const rebootRequired = await isRebootRequired();
+    await notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '');
 }
 
-function checkUbuntuVersion(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function checkUbuntuVersion() {
     const isXenial = fs.readFileSync('/etc/lsb-release', 'utf-8').includes('16.04');
-    if (!isXenial) return callback();
+    if (!isXenial) return;
 
-    notifications.alert(notifications.ALERT_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 16.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-18/ to upgrade to Ubuntu 18 at the earliest.', callback);
+    await notifications.alert(notifications.ALERT_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 16.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-18/ to upgrade to Ubuntu 18 at the earliest.');
 }
 
-function getLogs(unit, options, callback) {
+async function getLogs(unit, options) {
     assert.strictEqual(typeof unit, 'string');
     assert(options && typeof options === 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     assert.strictEqual(typeof options.lines, 'number');
     assert.strictEqual(typeof options.format, 'string');
@@ -261,15 +227,15 @@ function getLogs(unit, options, callback) {
     // need to handle box.log without subdir
     if (unit === 'box') args.push(path.join(paths.LOG_DIR, 'box.log'));
     else if (unit.startsWith('crash-')) args.push(path.join(paths.CRASH_LOG_DIR, unit.slice(6) + '.log'));
-    else return callback(new BoxError(BoxError.BAD_FIELD, 'No such unit', { field: 'unit' }));
+    else throw new BoxError(BoxError.BAD_FIELD, `No such unit '${unit}'`);
 
-    var cp = spawn('/usr/bin/tail', args);
+    const cp = spawn('/usr/bin/tail', args);
 
-    var transformStream = split(function mapper(line) {
+    const transformStream = split(function mapper(line) {
         if (format !== 'json') return line + '\n';
 
-        var data = line.split(' '); // logs are <ISOtimestamp> <msg>
-        var timestamp = (new Date(data[0])).getTime();
+        const data = line.split(' '); // logs are <ISOtimestamp> <msg>
+        let timestamp = (new Date(data[0])).getTime();
         if (isNaN(timestamp)) timestamp = 0;
 
         return JSON.stringify({
@@ -283,142 +249,101 @@ function getLogs(unit, options, callback) {
 
     cp.stdout.pipe(transformStream);
 
-    return callback(null, transformStream);
+    return transformStream;
 }
 
-function prepareDashboardDomain(domain, auditSource, callback) {
+async function prepareDashboardDomain(domain, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`prepareDashboardDomain: ${domain}`);
 
-    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+    if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
 
-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await domains.get(domain);
+    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');
 
-        const fqdn = domains.fqdn(constants.DASHBOARD_LOCATION, domainObject);
+    const fqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
-        apps.getAll(function (error, result) {
-            if (error) return callback(error);
+    const result = await apps.list();
+    if (result.some(app => app.fqdn === fqdn)) throw new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app');
 
-            const conflict = result.filter(app => app.fqdn === fqdn);
-            if (conflict.length) return callback(new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app'));
+    const taskId = await tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_LOCATION, domain, auditSource ]);
 
-            tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_LOCATION, domain, auditSource ], function (error, taskId) {
-                if (error) return callback(error);
+    tasks.startTask(taskId, {});
 
-                tasks.startTask(taskId, {}, NOOP_CALLBACK);
-
-                callback(null, taskId);
-            });
-        });
-    });
+    return taskId;
 }
 
 // call this only pre activation since it won't start mail server
-function setDashboardDomain(domain, auditSource, callback) {
+async function setDashboardDomain(domain, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`setDashboardDomain: ${domain}`);
 
-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await domains.get(domain);
+    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');
 
-        reverseProxy.writeDashboardConfig(domain, function (error) {
-            if (error) return callback(error);
+    await reverseProxy.writeDashboardConfig(domainObject);
+    const fqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
-            const fqdn = domains.fqdn(constants.DASHBOARD_LOCATION, domainObject);
+    await settings.setDashboardLocation(domain, fqdn);
 
-            settings.setDashboardLocation(domain, fqdn, function (error) {
-                if (error) return callback(error);
+    await safe(appstore.updateCloudron({ domain }));
 
-                appstore.updateCloudron({ domain }, NOOP_CALLBACK);
-
-                eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain, fqdn });
-
-                callback(null);
-            });
-        });
-    });
+    await eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain, fqdn });
 }
 
 // call this only post activation because it will restart mail server
-function updateDashboardDomain(domain, auditSource, callback) {
+async function updateDashboardDomain(domain, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`updateDashboardDomain: ${domain}`);
 
-    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+    if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
 
-    setDashboardDomain(domain, auditSource, function (error) {
-        if (error) return callback(error);
+    await setDashboardDomain(domain, auditSource);
 
-        services.rebuildService('turn', NOOP_CALLBACK); // to update the realm variable
-
-        callback(null);
-    });
+    safe(services.rebuildService('turn', auditSource), { debug }); // to update the realm variable
 }
 
-function renewCerts(options, auditSource, callback) {
+async function renewCerts(options, auditSource) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ], function (error, taskId) {
-        if (error) return callback(error);
-
-        tasks.startTask(taskId, {}, NOOP_CALLBACK);
-
-        callback(null, taskId);
-    });
+    const taskId = await tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ]);
+    tasks.startTask(taskId, {});
+    return taskId;
 }
 
-function setupDnsAndCert(subdomain, domain, auditSource, progressCallback, callback) {
+async function setupDnsAndCert(subdomain, domain, auditSource, progressCallback) {
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
 
-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await domains.get(domain);
+    const dashboardFqdn = dns.fqdn(subdomain, domainObject);
 
-        const dashboardFqdn = domains.fqdn(subdomain, domainObject);
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();
 
-        sysinfo.getServerIp(function (error, ip) {
-            if (error) return callback(error);
-
-            async.series([
-                (done) => { progressCallback({ message: `Updating DNS of ${dashboardFqdn}` }); done(); },
-                domains.upsertDnsRecords.bind(null, subdomain, domain, 'A', [ ip ]),
-                (done) => { progressCallback({ message: `Waiting for DNS of ${dashboardFqdn}` }); done(); },
-                domains.waitForDnsRecord.bind(null, subdomain, domain, 'A', ip, { interval: 30000, times: 50000 }),
-                (done) => { progressCallback({ message: `Getting certificate of ${dashboardFqdn}` }); done(); },
-                reverseProxy.ensureCertificate.bind(null, domains.fqdn(subdomain, domainObject), domain, auditSource)
-            ], function (error) {
-                if (error) return callback(error);
-
-                callback(null);
-            });
-        });
-    });
+    progressCallback({ percent: 20, message: `Updating DNS of ${dashboardFqdn}` });
+    await dns.upsertDnsRecords(subdomain, domain, 'A', [ ipv4 ]);
+    if (ipv6) await dns.upsertDnsRecords(subdomain, domain, 'AAAA', [ ipv6 ]);
+    progressCallback({ percent: 40, message: `Waiting for DNS of ${dashboardFqdn}` });
+    await dns.waitForDnsRecord(subdomain, domain, 'A', ipv4, { interval: 30000, times: 50000 });
+    if (ipv6) await dns.waitForDnsRecord(subdomain, domain, 'AAAA', ipv6, { interval: 30000, times: 50000 });
+    progressCallback({ percent: 60, message: `Getting certificate of ${dashboardFqdn}` });
+    await reverseProxy.ensureCertificate(dns.fqdn(subdomain, domainObject), domain, auditSource);
 }
 
-function syncDnsRecords(options, callback) {
+async function syncDnsRecords(options) {
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ], function (error, taskId) {
-        if (error) return callback(error);
-
-        tasks.startTask(taskId, {}, NOOP_CALLBACK);
-
-        callback(null, taskId);
-    });
+    const taskId = await tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ]);
+    tasks.startTask(taskId, {});
+    return taskId;
 }

src/collectd.js

@@ -8,7 +8,6 @@ exports = module.exports = {
 const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('collectd'),
-    fs = require('fs'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -16,40 +15,29 @@ const assert = require('assert'),
 
 const CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh');
 
-function addProfile(name, profile, callback) {
+async function addProfile(name, profile) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof profile, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     const configFilePath = path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`);
 
     // skip restarting collectd if the profile already exists with the same contents
     const currentProfile = safe.fs.readFileSync(configFilePath, 'utf8') || '';
-    if (currentProfile === profile) return callback(null);
+    if (currentProfile === profile) return;
 
-    fs.writeFile(configFilePath, profile, function (error) {
-        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
-
-        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
-
-            callback(null);
-        });
-    });
+    if (!safe.fs.writeFileSync(configFilePath, profile)) throw new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${safe.error.message}`);
 
+    const [error] = await safe(shell.promises.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}));
+    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config');
 }
 
-function removeProfile(name, callback) {
+async function removeProfile(name) {
     assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`), function (error) {
-        if (error && error.code !== 'ENOENT') debug('Error removing collectd profile', error);
+    if (!safe.fs.unlinkSync(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`))) {
+        if (safe.error.code !== 'ENOENT') debug('Error removing collectd profile', safe.error);
+    }
 
-        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
-
-            callback(null);
-        });
-    });
+    const [error] = await safe(shell.promises.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}));
+    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config');
 }

src/constants.js

@@ -29,6 +29,7 @@ exports = module.exports = {
     AUTHWALL_PORT: 3001,
     LDAP_PORT: 3002,
     DOCKER_PROXY_PORT: 3003,
+    USER_DIRECTORY_LDAPS_PORT: 3004, // user directory LDAP with TLS rerouting in iptables, public port is 636
 
     NGINX_DEFAULT_CONFIG_FILE_NAME: 'default.conf',
 
@@ -46,18 +47,28 @@ exports = module.exports = {
         'io.github.sickchill.cloudronapp',
         'to.couchpota.cloudronapp'
     ],
+    DEMO_APP_LIMIT: 20,
 
     AUTOUPDATE_PATTERN_NEVER: 'never',
 
+    // the db field is a blob so we make this explicit
+    AVATAR_NONE: Buffer.from('', 'utf8'),
+    AVATAR_GRAVATAR: Buffer.from('gravatar', 'utf8'),
+    AVATAR_CUSTOM: Buffer.from('custom', 'utf8'),   // this is not used here just for reference. The field will contain a byte buffer instead of the type string
+
     SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8), // also used in dashboard client.js
 
     CLOUDRON: CLOUDRON,
     TEST: TEST,
 
+    PORT25_CHECK_SERVER: 'port25check.cloudron.io',
+
     SUPPORT_EMAIL: 'support@cloudron.io',
 
+    USER_DIRECTORY_LDAP_DN: 'cn=admin,ou=system,dc=cloudron',
+
     FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
 
-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '6.0.1-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '7.0.0-test'
 };
 

src/crashnotifier.js

@@ -5,7 +5,8 @@ exports = module.exports = {
 };
 
 const assert = require('assert'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
+    child_process = require('child_process'),
     eventlog = require('./eventlog.js'),
     safe = require('safetydance'),
     path = require('path'),
@@ -17,43 +18,36 @@ const COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
 const CRASH_LOG_TIMESTAMP_OFFSET = 1000 * 60 * 60; // 60 min
 const CRASH_LOG_TIMESTAMP_FILE = '/tmp/crashlog.timestamp';
 
-function collectLogs(unitName, callback) {
+async function collectLogs(unitName) {
     assert.strictEqual(typeof unitName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
-    if (!logs) return callback(safe.error);
-
-    callback(null, logs);
+    const logs = child_process.execSync(`sudo ${COLLECT_LOGS_CMD} ${unitName}`, { encoding: 'utf8' });
+    return logs;
 }
 
-function sendFailureLogs(unitName, callback) {
+async function sendFailureLogs(unitName) {
     assert.strictEqual(typeof unitName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // check if we already sent a mail in the last CRASH_LOG_TIME_OFFSET window
     const timestamp = safe.fs.readFileSync(CRASH_LOG_TIMESTAMP_FILE, 'utf8');
     if (timestamp && (parseInt(timestamp) + CRASH_LOG_TIMESTAMP_OFFSET) > Date.now()) {
         console.log('Crash log already sent within window');
-        return callback();
+        return;
     }
 
-    collectLogs(unitName, async function (error, logs) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            logs = util.format('Failed to collect logs.', error);
-        }
+    let [error, logs] = await safe(collectLogs(unitName));
+    if (error) {
+        console.error('Failed to collect logs.', error);
+        logs = util.format('Failed to collect logs.', error);
+    }
 
-        const crashId = `${new Date().toISOString()}`;
-        console.log(`Creating crash log for ${unitName} with id ${crashId}`);
+    const crashId = `${new Date().toISOString()}`;
+    console.log(`Creating crash log for ${unitName} with id ${crashId}`);
 
-        if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, `${crashId}.log`), logs)) console.log(`Failed to stash logs to ${crashId}.log:`, safe.error);
+    if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, `${crashId}.log`), logs)) console.log(`Failed to stash logs to ${crashId}.log:`, safe.error);
 
-        [error] = await safe(eventlog.add(eventlog.ACTION_PROCESS_CRASH, auditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }));
-        if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);
+    [error] = await safe(eventlog.add(eventlog.ACTION_PROCESS_CRASH, AuditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }));
+    if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);
 
-        safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
-
-        callback();
-    });
+    safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
 }

src/cron.js

@@ -19,8 +19,7 @@ exports = module.exports = {
 const appHealthMonitor = require('./apphealthmonitor.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     cloudron = require('./cloudron.js'),
     constants = require('./constants.js'),
@@ -29,11 +28,13 @@ const appHealthMonitor = require('./apphealthmonitor.js'),
     dyndns = require('./dyndns.js'),
     eventlog = require('./eventlog.js'),
     janitor = require('./janitor.js'),
+    safe = require('safetydance'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
     system = require('./system.js'),
     updater = require('./updater.js'),
     updateChecker = require('./updatechecker.js'),
+    userdirectory = require('./userdirectory.js'),
     _ = require('underscore');
 
 const gJobs = {
@@ -52,8 +53,6 @@ const gJobs = {
     appHealthMonitor: null
 };
 
-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 // cron format
 // Seconds: 0-59
 // Minutes: 0-59
@@ -62,87 +61,81 @@ const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 // Months: 0-11
 // Day of Week: 0-6
 
-function startJobs(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function startJobs() {
     debug('startJobs: starting cron jobs');
 
     const randomTick = Math.floor(60*Math.random());
     gJobs.systemChecks = new CronJob({
-        cronTime: '00 30 2 * * *', // once a day. if you change this interval, change the notification messages with correct duration
-        onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
+        cronTime: `${randomTick} ${randomTick} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
+        onTick: async () => await safe(cloudron.runSystemChecks(), { debug }),
         start: true
     });
 
     gJobs.diskSpaceChecker = new CronJob({
         cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
+        onTick: async () => await safe(system.checkDiskSpace(), { debug }),
         start: true
     });
 
     // this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
     gJobs.updateCheckerJob = new CronJob({
         cronTime: `${randomTick} ${randomTick} 1,5,9,13,17,21,23 * * *`,
-        onTick: () => updateChecker.checkForUpdates({ automatic: true }, NOOP_CALLBACK),
+        onTick: async () => await safe(updateChecker.checkForUpdates({ automatic: true }), { debug }),
         start: true
     });
 
     gJobs.cleanupTokens = new CronJob({
         cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: janitor.cleanupTokens,
+        onTick: async () => await safe(janitor.cleanupTokens(), { debug }),
         start: true
     });
 
     gJobs.cleanupBackups = new CronJob({
         cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN,
-        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        onTick: async () => await safe(backups.startCleanupTask(AuditSource.CRON), { debug }),
         start: true
     });
 
     gJobs.cleanupEventlog = new CronJob({
         cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: eventlog.cleanup.bind(null, new Date(Date.now() - 60 * 60 * 24 * 10 * 1000)), // 10 days ago
+        onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 60 * 60 * 24 * 10 * 1000) }), { debug }), // 10 days ago
         start: true
     });
 
     gJobs.dockerVolumeCleaner = new CronJob({
         cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: janitor.cleanupDockerVolumes,
+        onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug }),
         start: true
     });
 
     gJobs.schedulerSync = new CronJob({
         cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
-        onTick: scheduler.sync,
+        onTick: async () => await safe(scheduler.sync(), { debug }),
         start: true
     });
 
     gJobs.certificateRenew = new CronJob({
         cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
+        onTick: async () => await safe(cloudron.renewCerts({}, AuditSource.CRON), { debug }),
         start: true
     });
 
     gJobs.appHealthMonitor = new CronJob({
         cronTime: '*/10 * * * * *', // every 10 seconds
-        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
+        onTick: async () => await safe(appHealthMonitor.run(10), { debug }), // 10 is the max run time
         start: true
     });
 
-    settings.getAll(function (error, allSettings) {
-        if (error) return callback(error);
+    const allSettings = await settings.list();
 
-        const tz = allSettings[settings.TIME_ZONE_KEY];
-        backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
-        autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz);
-        dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
-
-        callback();
-    });
+    const tz = allSettings[settings.TIME_ZONE_KEY];
+    backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
+    autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz);
+    dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
 }
 
 // eslint-disable-next-line no-unused-vars
-function handleSettingsChanged(key, value) {
+async function handleSettingsChanged(key, value) {
     assert.strictEqual(typeof key, 'string');
     // value is a variant
 
@@ -152,10 +145,12 @@ function handleSettingsChanged(key, value) {
     case settings.AUTOUPDATE_PATTERN_KEY:
     case settings.DYNAMIC_DNS_KEY:
         debug('handleSettingsChanged: recreating all jobs');
-        async.series([
-            stopJobs,
-            startJobs
-        ], NOOP_CALLBACK);
+        await stopJobs();
+        await startJobs();
+        break;
+    case settings.USER_DIRECTORY_KEY:
+        if (value.enabled) await userdirectory.start();
+        else await userdirectory.stop();
         break;
     default:
         break;
@@ -172,7 +167,7 @@ function backupConfigChanged(value, tz) {
 
     gJobs.backup = new CronJob({
         cronTime: value.schedulePattern,
-        onTick: backups.startBackupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        onTick: async () => await safe(backups.startBackupTask(AuditSource.CRON), { debug }),
         start: true,
         timeZone: tz
     });
@@ -190,19 +185,21 @@ function autoupdatePatternChanged(pattern, tz) {
 
     gJobs.autoUpdater = new CronJob({
         cronTime: pattern,
-        onTick: function() {
+        onTick: async function() {
             const updateInfo = updateChecker.getUpdateInfo();
             // do box before app updates. for the off chance that the box logic fixes some app update logic issue
             if (updateInfo.box && !updateInfo.box.unstable) {
                 debug('Starting box autoupdate to %j', updateInfo.box);
-                updater.updateToLatest({ skipBackup: false }, auditSource.CRON, NOOP_CALLBACK);
+                const [error] = await safe(updater.updateToLatest({ skipBackup: false }, AuditSource.CRON));
+                if (error) debug(`Failed to box autoupdate: ${error.message}`);
                 return;
             }
 
             const appUpdateInfo = _.omit(updateInfo, 'box');
             if (Object.keys(appUpdateInfo).length > 0) {
                 debug('Starting app update to %j', appUpdateInfo);
-                apps.autoupdateApps(appUpdateInfo, auditSource.CRON, NOOP_CALLBACK);
+                const [error] = await safe(apps.autoupdateApps(appUpdateInfo, AuditSource.CRON));
+                if (error) debug(`Failed to app autoupdate: ${error.message}`);
             } else {
                 debug('No app auto updates available');
             }
@@ -221,7 +218,7 @@ function dynamicDnsChanged(enabled) {
     if (enabled) {
         gJobs.dynamicDns = new CronJob({
             cronTime: '5 * * * * *',    // we only update the records if the ip has changed.
-            onTick: dyndns.sync.bind(null, auditSource.CRON, NOOP_CALLBACK),
+            onTick: async () => await safe(dyndns.sync(AuditSource.CRON), { debug }),
             start: true
         });
     } else {
@@ -230,14 +227,10 @@ function dynamicDnsChanged(enabled) {
     }
 }
 
-function stopJobs(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    for (var job in gJobs) {
+async function stopJobs() {
+    for (const job in gJobs) {
         if (!gJobs[job]) continue;
         gJobs[job].stop();
         gJobs[job] = null;
     }
-
-    callback();
 }

src/database.js

@@ -15,14 +15,14 @@ exports = module.exports = {
 const assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
-    child_process = require('child_process'),
     constants = require('./constants.js'),
     debug = require('debug')('box:database'),
     mysql = require('mysql'),
-    once = require('once'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
     util = require('util');
 
-var gConnectionPool = null;
+let gConnectionPool = null;
 
 const gDatabase = {
     hostname: '127.0.0.1',
@@ -32,10 +32,8 @@ const gDatabase = {
     name: 'box'
 };
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    if (gConnectionPool !== null) return callback(null);
+async function initialize() {
+    if (gConnectionPool !== null) return;
 
     if (constants.TEST) {
         // see setupTest script how the mysql-server is run
@@ -66,57 +64,49 @@ function initialize(callback) {
         connection.query('USE ' + gDatabase.name);
         connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
     });
-
-    callback(null);
 }
 
-function uninitialize(callback) {
-    if (!gConnectionPool) return callback(null);
+async function uninitialize() {
+    if (!gConnectionPool) return;
 
-    gConnectionPool.end(callback);
+    gConnectionPool.end();
     gConnectionPool = null;
 }
 
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var cmd = util.format('mysql --host="%s" --user="%s" --password="%s" -Nse "SHOW TABLES" %s | grep -v "^migrations$" | while read table; do mysql --host="%s" --user="%s" --password="%s" -e "SET FOREIGN_KEY_CHECKS = 0; TRUNCATE TABLE $table" %s; done',
+async function clear() {
+    const cmd = util.format('mysql --host="%s" --user="%s" --password="%s" -Nse "SHOW TABLES" %s | grep -v "^migrations$" | while read table; do mysql --host="%s" --user="%s" --password="%s" -e "SET FOREIGN_KEY_CHECKS = 0; TRUNCATE TABLE $table" %s; done',
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name,
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name);
 
-    child_process.exec(cmd, callback);
+    await shell.promises.exec('clear_database', cmd);
 }
 
-function query() {
+async function query() {
     assert.notStrictEqual(gConnectionPool, null);
 
     return new Promise((resolve, reject) => {
         let args = Array.prototype.slice.call(arguments);
-        const callback = typeof args[args.length - 1] === 'function' ? args.pop() : null;
 
         args.push(function queryCallback(error, result) {
-            if (error) return callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+            if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage || null }));
 
-            callback ? callback(null, result) : resolve(result);
+            resolve(result);
         });
 
         gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
     });
 }
 
-function transaction(queries) {
+async function transaction(queries) {
     assert(Array.isArray(queries));
 
-    const args = Array.prototype.slice.call(arguments);
-    const callback = typeof args[args.length - 1] === 'function' ? once(args.pop()) : null;
-
     return new Promise((resolve, reject) => {
         gConnectionPool.getConnection(function (error, connection) {
-            if (error) return callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+            if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
 
             const releaseConnection = (error) => {
                 connection.release();
-                callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+                reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage || null }));
             };
 
             connection.beginTransaction(function (error) {
@@ -132,7 +122,7 @@ function transaction(queries) {
 
                         connection.release();
 
-                        callback ? callback(null, results) : resolve(results);
+                        resolve(results);
                     });
                 });
             });
@@ -140,27 +130,25 @@ function transaction(queries) {
     });
 }
 
-function importFromFile(file, callback) {
+async function importFromFile(file) {
     assert.strictEqual(typeof file, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var cmd = `/usr/bin/mysql -h "${gDatabase.hostname}" -u ${gDatabase.username} -p${gDatabase.password} ${gDatabase.name} < ${file}`;
+    const cmd = `/usr/bin/mysql -h "${gDatabase.hostname}" -u ${gDatabase.username} -p${gDatabase.password} ${gDatabase.name} < ${file}`;
 
-    async.series([
-        query.bind(null, 'CREATE DATABASE IF NOT EXISTS box'),
-        child_process.exec.bind(null, cmd)
-    ], callback);
+    await query('CREATE DATABASE IF NOT EXISTS box');
+    const [error] = await safe(shell.promises.exec('importFromFile', cmd));
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 }
 
-function exportToFile(file, callback) {
+async function exportToFile(file) {
     assert.strictEqual(typeof file, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
     // this option must not be set in production cloudrons which still use the old mysqldump
-    const disableColStats = (constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
+    const colStats = (!constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
 
-    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${disableColStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
+    const cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${colStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
 
-    child_process.exec(cmd, callback);
+    const [error] = await safe(shell.promises.exec('exportToFile', cmd));
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 }

src/datalayout.js

@@ -1,6 +1,6 @@
 'use strict';
 
-let assert = require('assert'),
+const assert = require('assert'),
     path = require('path');
 
 class DataLayout {

src/dig.js

@@ -0,0 +1,40 @@
+'use strict';
+
+exports = module.exports = {
+    resolve,
+};
+
+const assert = require('assert'),
+    constants = require('./constants.js'),
+    dns = require('dns'),
+    safe = require('safetydance'),
+    _ = require('underscore');
+
+// a note on TXT records. It doesn't have quotes ("") at the DNS level. Those quotes
+// are added for DNS server software to enclose spaces. Such quotes may also be returned
+// by the DNS REST API of some providers
+async function resolve(hostname, rrtype, options) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof rrtype, 'string');
+    assert(options && typeof options === 'object');
+
+    const defaultOptions = { server: '127.0.0.1', timeout: 5000 }; // unbound runs on 127.0.0.1
+    const resolver = new dns.promises.Resolver();
+    options = _.extend({ }, defaultOptions, options);
+
+    // Only use unbound on a Cloudron
+    if (constants.CLOUDRON) resolver.setServers([ options.server ]);
+
+    // should callback with ECANCELLED but looks like we might hit https://github.com/nodejs/node/issues/14814
+    const timerId = setTimeout(resolver.cancel.bind(resolver), options.timeout || 5000);
+
+    const [error, result] = await safe(resolver.resolve(hostname, rrtype));
+    clearTimeout(timerId);
+
+    if (error && error.code === 'ECANCELLED') error.code = 'TIMEOUT';
+    if (error) throw error;
+
+    // when you query a random record, it errors with ENOTFOUND. But, if you query a record which has a different type
+    // we sometimes get empty array and sometimes ENODATA. for TXT records, result is 2d array of strings
+    return result;
+}

src/dns.js

@@ -0,0 +1,303 @@
+'use strict';
+
+module.exports = exports = {
+    fqdn,
+    getName,
+
+    getDnsRecords,
+    upsertDnsRecords,
+    removeDnsRecords,
+
+    waitForDnsRecord,
+
+    validateHostname,
+
+    makeWildcard,
+
+    registerLocations,
+    unregisterLocations,
+
+    checkDnsRecords,
+    syncDnsRecords,
+};
+
+const apps = require('./apps.js'),
+    assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:dns'),
+    domains = require('./domains.js'),
+    ipaddr = require('ipaddr.js'),
+    mail = require('./mail.js'),
+    promiseRetry = require('./promise-retry.js'),
+    safe = require('safetydance'),
+    settings = require('./settings.js'),
+    sysinfo = require('./sysinfo.js'),
+    tld = require('tldjs');
+
+// choose which subdomain backend we use for test purpose we use route53
+function api(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
+    switch (provider) {
+    case 'cloudflare': return require('./dns/cloudflare.js');
+    case 'route53': return require('./dns/route53.js');
+    case 'gcdns': return require('./dns/gcdns.js');
+    case 'digitalocean': return require('./dns/digitalocean.js');
+    case 'gandi': return require('./dns/gandi.js');
+    case 'godaddy': return require('./dns/godaddy.js');
+    case 'linode': return require('./dns/linode.js');
+    case 'vultr': return require('./dns/vultr.js');
+    case 'namecom': return require('./dns/namecom.js');
+    case 'namecheap': return require('./dns/namecheap.js');
+    case 'netcup': return require('./dns/netcup.js');
+    case 'noop': return require('./dns/noop.js');
+    case 'manual': return require('./dns/manual.js');
+    case 'wildcard': return require('./dns/wildcard.js');
+    default: return null;
+    }
+}
+
+function fqdn(subdomain, domainObject) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domainObject, 'object');
+
+    return subdomain + (subdomain ? '.' : '') + domainObject.domain;
+}
+
+// Hostname validation comes from RFC 1123 (section 2.1)
+// Domain name validation comes from RFC 2181 (Name syntax)
+// https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
+// We are validating the validity of the location-fqdn as host name (and not dns name)
+function validateHostname(subdomain, domainObject) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domainObject, 'object');
+
+    const hostname = fqdn(subdomain, domainObject);
+
+    const RESERVED_LOCATIONS = [
+        constants.SMTP_LOCATION,
+        constants.IMAP_LOCATION
+    ];
+    if (RESERVED_LOCATIONS.indexOf(subdomain) !== -1) return new BoxError(BoxError.BAD_FIELD, `subdomain '${subdomain}' is reserved`);
+
+    if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, `subdomain '${subdomain}' is reserved`);
+
+    // workaround https://github.com/oncletom/tld.js/issues/73
+    var tmp = hostname.replace('_', '-');
+    if (!tld.isValid(tmp)) return new BoxError(BoxError.BAD_FIELD, 'Hostname is not a valid domain name');
+
+    if (hostname.length > 253) return new BoxError(BoxError.BAD_FIELD, 'Hostname length exceeds 253 characters');
+
+    if (subdomain) {
+        // label validation
+        if (subdomain.split('.').some(function (p) { return p.length > 63 || p.length < 1; })) return new BoxError(BoxError.BAD_FIELD, 'Invalid subdomain length');
+        if (subdomain.match(/^[A-Za-z0-9-.]+$/) === null) return new BoxError(BoxError.BAD_FIELD, 'Subdomain can only contain alphanumeric, hyphen and dot');
+        if (/^[-.]/.test(subdomain)) return new BoxError(BoxError.BAD_FIELD, 'Subdomain cannot start or end with hyphen or dot');
+    }
+
+    return null;
+}
+
+// returns the 'name' that needs to be inserted into zone
+// eslint-disable-next-line no-unused-vars
+function getName(domain, subdomain, type) {
+    const part = domain.domain.slice(0, -domain.zoneName.length - 1);
+
+    if (subdomain === '') return part;
+
+    return part ? `${subdomain}.${part}` : subdomain;
+}
+
+async function getDnsRecords(subdomain, domain, type) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof type, 'string');
+
+    const domainObject = await domains.get(domain);
+    return await api(domainObject.provider).get(domainObject, subdomain, type);
+}
+
+async function checkDnsRecords(subdomain, domain) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domain, 'string');
+
+    const ipv4Records = await getDnsRecords(subdomain, domain, 'A');
+    const ipv4 = await sysinfo.getServerIPv4();
+
+    // if empty OR exactly one record with the ip, we don't need to overwrite
+    if (ipv4Records.length !== 0 && (ipv4Records.length !== 1 || ipv4Records[0] !== ipv4)) return { needsOverwrite: true };
+
+    const ipv6 = await sysinfo.getServerIPv6();
+    if (ipv6) {
+        const ipv6Records = await getDnsRecords(subdomain, domain, 'AAAA');
+
+        // if empty OR exactly one record with the ip, we don't need to overwrite
+        if (ipv6Records.length !== 0 && (ipv6Records.length !== 1 || ipaddr.parse(ipv6Records[0]).toRFC5952String() !== ipv6)) return { needsOverwrite: true };
+    }
+
+    return { needsOverwrite: false }; // one record exists and in sync
+}
+
+// note: for TXT records the values must be quoted
+async function upsertDnsRecords(subdomain, domain, type, values) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    debug(`upsertDNSRecord: location ${subdomain} on domain ${domain} of type ${type} with values ${JSON.stringify(values)}`);
+
+    const domainObject = await domains.get(domain);
+    await api(domainObject.provider).upsert(domainObject, subdomain, type, values);
+}
+
+async function removeDnsRecords(subdomain, domain, type, values) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    debug('removeDNSRecord: %s on %s type %s values', subdomain, domain, type, values);
+
+    const domainObject = await domains.get(domain);
+    const [error] = await safe(api(domainObject.provider).del(domainObject, subdomain, type, values));
+    if (error && error.reason !== BoxError.NOT_FOUND) throw error; // this is never returned afaict
+}
+
+async function waitForDnsRecord(subdomain, domain, type, value, options) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert(type === 'A' || type === 'AAAA' || type === 'TXT');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+
+    const domainObject = await domains.get(domain);
+
+    // linode DNS takes ~15mins
+    if (!options.interval) options.interval = domainObject.provider === 'linode' ? 20000 : 5000;
+
+    await api(domainObject.provider).wait(domainObject, subdomain, type, value, options);
+}
+
+function makeWildcard(vhost) {
+    assert.strictEqual(typeof vhost, 'string');
+
+    // if the vhost is like *.example.com, this function will do nothing
+    let parts = vhost.split('.');
+    parts[0] = '*';
+    return parts.join('.');
+}
+
+async function registerLocation(location, options, recordType, recordValue) {
+    const overwriteDns = options.overwriteDns || false;
+
+    // get the current record before updating it
+    const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, recordType));
+    if (getError) {
+        const retryable = getError.reason !== BoxError.ACCESS_DENIED && getError.reason !== BoxError.NOT_FOUND;
+        debug(`registerLocation: Get error. retryable: ${retryable}. ${getError.message}`);
+        throw new BoxError(getError.reason, getError.message, { domain: location, retryable });
+    }
+
+    if (values.length === 1 && values[0] === recordValue) return; // up-to-date
+
+    // refuse to update any existing DNS record for custom domains that we did not create
+    if (values.length !== 0 && !overwriteDns) throw new BoxError(BoxError.ALREADY_EXISTS, `DNS ${recordType} record already exists`, { domain: location, retryable: false });
+
+    const [upsertError] = await safe(upsertDnsRecords(location.subdomain, location.domain, recordType, [ recordValue ]));
+    if (upsertError) {
+        const retryable = upsertError.reason === BoxError.BUSY || upsertError.reason === BoxError.EXTERNAL_ERROR;
+        debug(`registerLocation: Upsert error. retryable: ${retryable}. ${upsertError.message}`);
+        throw new BoxError(BoxError.EXTERNAL_ERROR, upsertError.message, { domain: location, retryable });
+    }
+}
+
+async function registerLocations(locations, options, progressCallback) {
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
+
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();
+
+    for (const location of locations) {
+        progressCallback({ message: `Registering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+
+        await promiseRetry({ times: 200, interval: 5000, debug, retry: (error) => error.retryable }, async function () {
+            await registerLocation(location, options, 'A', ipv4);
+            if (ipv6) await registerLocation(location, options, 'AAAA', ipv6);
+        });
+    }
+}
+
+async function unregisterLocation(location, recordType, recordValue) {
+    const [error] = await safe(removeDnsRecords(location.subdomain, location.domain, recordType, [ recordValue ]));
+    if (!error || error.reason === BoxError.NOT_FOUND) return;
+
+    const retryable = error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR;
+    debug(`unregisterLocation: Error unregistering location ${recordType}. retryable: ${retryable}. ${error.message}`);
+
+    throw new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location, retryable });
+}
+
+async function unregisterLocations(locations, progressCallback) {
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();
+
+    for (const location of locations) {
+        progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+
+        await promiseRetry({ times: 30, interval: 5000, debug, retry: (error) => error.retryable }, async function () {
+            await unregisterLocation(location, 'A', ipv4);
+            if (ipv6) await unregisterLocation(location, 'AAAA', ipv6);
+        });
+    }
+}
+
+async function syncDnsRecords(options, progressCallback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    if (options.domain && options.type === 'mail') return await mail.setDnsRecords(options.domain);
+
+    let allDomains = await domains.list();
+
+    if (options.domain) allDomains = allDomains.filter(d => d.domain === options.domain);
+
+    const mailSubdomain = settings.mailFqdn().substr(0, settings.mailFqdn().length - settings.mailDomain().length - 1);
+
+    const allApps = await apps.list();
+
+    let progress = 1, errors = [];
+
+    // we sync by domain only to get some nice progress
+    for (const domain of allDomains) {
+        progressCallback({ percent: progress, message: `Updating DNS of ${domain.domain}`});
+        progress += Math.round(100/(1+allDomains.length));
+
+        let locations = [];
+        if (domain.domain === settings.dashboardDomain()) locations.push({ subdomain: constants.DASHBOARD_LOCATION, domain: settings.dashboardDomain() });
+        if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.dashboardFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
+
+        for (const app of allApps) {
+            const appLocations = [{ subdomain: app.subdomain, domain: app.domain }].concat(app.redirectDomains).concat(app.aliasDomains);
+            locations = locations.concat(appLocations.filter(al => al.domain === domain.domain));
+        }
+
+        try {
+            await registerLocations(locations, { overwriteDns: true }, progressCallback);
+            progressCallback({ message: `Updating mail DNS of ${domain.domain}`});
+            await mail.setDnsRecords(domain.domain);
+        } catch (error) {
+            errors.push({ domain: domain.domain, message: error.message });
+        }
+    }
+
+    return { errors };
+}

src/dns/cloudflare.js

@@ -1,29 +1,29 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
-    async = require('async'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/cloudflare'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
 // we are using latest v4 stable API https://api.cloudflare.com/#getting-started-endpoints
-var CLOUDFLARE_ENDPOINT = 'https://api.cloudflare.com/client/v4';
+const CLOUDFLARE_ENDPOINT = 'https://api.cloudflare.com/client/v4';
 
 function removePrivateFields(domainObject) {
     domainObject.config.token = constants.SECRET_PLACEHOLDER;
@@ -34,12 +34,11 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function translateRequestError(result, callback) {
+function translateRequestError(result) {
     assert.strictEqual(typeof result, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, util.format('%s %j', result.statusCode, 'API does not exist')));
-    if (result.statusCode === 422) return callback(new BoxError(BoxError.BAD_FIELD, result.body.message));
+    if (result.statusCode === 404) return new BoxError(BoxError.NOT_FOUND, util.format('%s %j', result.statusCode, 'API does not exist'));
+    if (result.statusCode === 422) return new BoxError(BoxError.BAD_FIELD, result.body.message);
     if (result.statusCode === 400 || result.statusCode === 401 || result.statusCode === 403) {
         let message = 'Unknown error';
         if (typeof result.body.error === 'string') {
@@ -48,288 +47,230 @@ function translateRequestError(result, callback) {
             let error = result.body.errors[0];
             message = `message: ${error.message} statusCode: ${result.statusCode} code:${error.code}`;
         }
-        return callback(new BoxError(BoxError.ACCESS_DENIED, message));
+        return new BoxError(BoxError.ACCESS_DENIED, message);
     }
 
-    callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+    return new BoxError(BoxError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body));
 }
 
-function createRequest(method, url, dnsConfig) {
+function createRequest(method, url, domainConfig) {
     assert.strictEqual(typeof method, 'string');
     assert.strictEqual(typeof url, 'string');
-    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof domainConfig, 'object');
 
-    let request = superagent(method, url)
-        .timeout(30 * 1000);
+    const request = superagent(method, url).timeout(30 * 1000).ok(() => true);
 
-    if (dnsConfig.tokenType === 'GlobalApiKey') {
-        request.set('X-Auth-Key', dnsConfig.token).set('X-Auth-Email', dnsConfig.email);
+    if (domainConfig.tokenType === 'GlobalApiKey') {
+        request.set('X-Auth-Key', domainConfig.token).set('X-Auth-Email', domainConfig.email);
     } else {
-        request.set('Authorization', 'Bearer ' + dnsConfig.token);
+        request.set('Authorization', 'Bearer ' + domainConfig.token);
     }
 
     return request;
 }
 
-function getZoneByName(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneByName(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active', dnsConfig)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(error);
-            if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
-            if (!result.body.result.length) return callback(new BoxError(BoxError.NOT_FOUND, util.format('%s %j', result.statusCode, result.body)));
+    const [error, response] = await safe(createRequest('GET', `${CLOUDFLARE_ENDPOINT}/zones?name=${zoneName}&status=active`, domainConfig));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
+    if (!response.body.result.length) throw new BoxError(BoxError.NOT_FOUND, util.format('%s %j', response.statusCode, response.body));
 
-            callback(null, result.body.result[0]);
-        });
+    return response.body.result[0];
 }
 
 // gets records filtered by zone, type and fqdn
-function getDnsRecords(dnsConfig, zoneId, fqdn, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getDnsRecords(domainConfig, zoneId, fqdn, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneId, 'string');
     assert.strictEqual(typeof fqdn, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
-        .query({ type: type, name: fqdn })
-        .end(function (error, result) {
-            if (error && !error.response) return callback(error);
-            if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
+    const [error, response] = await safe(createRequest('GET', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig)
+        .query({ type: type, name: fqdn }));
 
-            var tmp = result.body.result;
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
 
-            return callback(null, tmp);
-        });
+    return response.body.result;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     debug('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
-    getZoneByName(dnsConfig, zoneName, function(error, result) {
-        if (error) return callback(error);
+    const result = await getZoneByName(domainConfig, zoneName);
+    const zoneId = result.id;
 
-        let zoneId = result.id;
+    const records = await getDnsRecords(domainConfig, zoneId, fqdn, type);
 
-        getDnsRecords(dnsConfig, zoneId, fqdn, type, function (error, dnsRecords) {
-            if (error) return callback(error);
+    let i = 0; // // used to track available records to update instead of create
 
-            let i = 0; // // used to track available records to update instead of create
+    for (let value of values) {
+        let priority = null;
 
-            async.eachSeries(values, function (value, iteratorCallback) {
-                var priority = null;
+        if (type === 'MX') {
+            priority = parseInt(value.split(' ')[0], 10);
+            value = value.split(' ')[1];
+        }
 
-                if (type === 'MX') {
-                    priority = parseInt(value.split(' ')[0], 10);
-                    value = value.split(' ')[1];
-                }
+        const data = {
+            type: type,
+            name: fqdn,
+            content: value,
+            priority: priority,
+            proxied: false,
+            ttl: 120  // 1 means "automatic" (meaning 300ms) and 120 is the lowest supported
+        };
 
-                var data = {
-                    type: type,
-                    name: fqdn,
-                    content: value,
-                    priority: priority,
-                    proxied: false,
-                    ttl: 120  // 1 means "automatic" (meaning 300ms) and 120 is the lowest supported
-                };
+        if (i >= records.length) { // create a new record
+            debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: false`);
 
-                if (i >= dnsRecords.length) { // create a new record
-                    debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: false`);
+            const [error, response] = await safe(createRequest('POST', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig)
+                .send(data));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
+        } else { // replace existing record
+            data.proxied = records[i].proxied; // preserve proxied parameter
 
-                    createRequest('POST', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
-                        .send(data)
-                        .end(function (error, result) {
-                            if (error && !error.response) return iteratorCallback(error);
-                            if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, iteratorCallback);
+            debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
 
-                            iteratorCallback(null);
-                        });
-                } else { // replace existing record
-                    data.proxied = dnsRecords[i].proxied; // preserve proxied parameter
+            const [error, response] = await safe(createRequest('PUT', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[i].id}`, domainConfig)
+                .send(data));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
+            ++i; // increment, as we have consumed the record
+        }
+    }
 
-                    debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
-
-                    createRequest('PUT', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id, dnsConfig)
-                        .send(data)
-                        .end(function (error, result) {
-                            ++i; // increment, as we have consumed the record
-
-                            if (error && !error.response) return iteratorCallback(error);
-                            if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, iteratorCallback);
-
-                            iteratorCallback(null);
-                        });
-                }
-            }, callback);
-        });
-    });
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[j].id}`, domainConfig));
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(dnsConfig, zoneName, function(error, zone) {
-        if (error) return callback(error);
-
-        getDnsRecords(dnsConfig, zone.id, fqdn, type, function (error, result) {
-            if (error) return callback(error);
-
-            var tmp = result.map(function (record) { return record.content; });
-            debug('get: %j', tmp);
-
-            callback(null, tmp);
-        });
-    });
+    const zone = await getZoneByName(domainConfig, zoneName);
+    const result = await getDnsRecords(domainConfig, zone.id, fqdn, type);
+    const tmp = result.map(function (record) { return record.content; });
+    return tmp;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(dnsConfig, zoneName, function(error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        getDnsRecords(dnsConfig, zone.id, fqdn, type, function(error, result) {
-            if (error) return callback(error);
-            if (result.length === 0) return callback(null);
+    const result = await getDnsRecords(domainConfig, zone.id, fqdn, type);
+    if (result.length === 0) return;
 
-            var zoneId = result[0].zone_id;
+    const zoneId = result[0].zone_id;
 
-            var tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); });
-            debug('del: %j', tmp);
+    const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); });
+    debug('del: %j', tmp);
 
-            if (tmp.length === 0) return callback(null);
+    if (tmp.length === 0) return;
 
-            async.eachSeries(tmp, function (record, callback) {
-                createRequest('DELETE', CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id, dnsConfig)
-                    .end(function (error, result) {
-                        if (error && !error.response) return callback(error);
-                        if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
-
-                        debug('del: done');
-
-                        callback(null);
-                    });
-            }, function (error) {
-                if (error) return callback(error);
-
-                callback(null, 'unused');
-            });
-        });
-    });
+    for (const r of tmp) {
+        const [error, response] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${r.id}`, domainConfig));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(subdomain, domainObject);
 
     debug('wait: %s for zone %s of type %s', fqdn, zoneName, type);
 
-    getZoneByName(dnsConfig, zoneName, function(error, result) {
-        if (error) return callback(error);
+    const result = await getZoneByName(domainConfig, zoneName);
+    const zoneId = result.id;
 
-        let zoneId = result.id;
+    const dnsRecords = await getDnsRecords(domainConfig, zoneId, fqdn, type);
+    if (dnsRecords.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
 
-        getDnsRecords(dnsConfig, zoneId, fqdn, type, function (error, dnsRecords) {
-            if (error) return callback(error);
-            if (dnsRecords.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
+    if (!dnsRecords[0].proxied) return await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 
-            if (!dnsRecords[0].proxied) return waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    debug('wait: skipping wait of proxied domain');
 
-            debug('wait: skipping wait of proxied domain');
-
-            callback(null); // maybe we can check for dns to be cloudflare IPs? https://api.cloudflare.com/#cloudflare-ips-cloudflare-ip-details
-        });
-    });
+    // maybe we can check for dns to be cloudflare IPs? https://api.cloudflare.com/#cloudflare-ips-cloudflare-ip-details
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
     // token can be api token or global api key
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
-    if (dnsConfig.tokenType !== 'GlobalApiKey' && dnsConfig.tokenType !== 'ApiToken')  return callback(new BoxError(BoxError.BAD_FIELD, 'tokenType is required', { field: 'tokenType' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
+    if (domainConfig.tokenType !== 'GlobalApiKey' && domainConfig.tokenType !== 'ApiToken') throw new BoxError(BoxError.BAD_FIELD, 'tokenType is required');
 
-    if (dnsConfig.tokenType === 'GlobalApiKey') {
-        if (typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    if (domainConfig.tokenType === 'GlobalApiKey') {
+        if (typeof domainConfig.email !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string');
     }
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        token: dnsConfig.token,
-        tokenType: dnsConfig.tokenType,
-        email: dnsConfig.email || null
+    const credentials = {
+        token: domainConfig.token,
+        tokenType: domainConfig.tokenType,
+        email: domainConfig.email || null
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        getZoneByName(dnsConfig, zoneName, function(error, zone) {
-            if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-            if (!_.isEqual(zone.name_servers.sort(), nameservers.sort())) {
-                debug('verifyDnsConfig: %j and %j do not match', nameservers, zone.name_servers);
-                return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Cloudflare', { field: 'nameservers' }));
-            }
+    if (!_.isEqual(zone.name_servers.sort(), nameservers.sort())) {
+        debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.name_servers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Cloudflare');
+    }
 
-            const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-            upsert(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-                debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-                del(domainObject, location, 'A', [ ip ], function (error) {
-                    if (error) return callback(error);
-
-                    debug('verifyDnsConfig: Test A record removed again');
-
-                    callback(null, credentials);
-                });
-            });
-        });
-    });
+    return credentials;
 }

src/dns/digitalocean.js

@@ -7,16 +7,15 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
-    async = require('async'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/digitalocean'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
@@ -37,244 +36,208 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function getInternal(dnsConfig, zoneName, name, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneRecords(domainConfig, zoneName, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var nextPage = null, matchingRecords = [];
+    let nextPage = null, matchingRecords = [];
 
     debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
 
-    async.doWhilst(function (iteratorDone) {
-        var url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records';
+    do {
+        const url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records';
 
-        superagent.get(url)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
+        const [error, response] = await safe(superagent.get(url)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
             .timeout(30 * 1000)
             .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return iteratorDone(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return iteratorDone(new BoxError(BoxError.NOT_FOUND, formatError(result)));
-                if (result.statusCode === 403 || result.statusCode === 401) return iteratorDone(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return iteratorDone(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            .ok(() => true));
 
-                matchingRecords = matchingRecords.concat(result.body.domain_records.filter(function (record) {
-                    return (record.type === type && record.name === name);
-                }));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                nextPage = (result.body.links && result.body.links.pages) ? result.body.links.pages.next : null;
+        matchingRecords = matchingRecords.concat(response.body.domain_records.filter(function (record) {
+            return (record.type === type && record.name === name);
+        }));
 
-                iteratorDone();
-            });
-    }, function (testDone) { return testDone(null, !!nextPage); }, function (error) {
-        debug('getInternal:', error, JSON.stringify(matchingRecords));
+        nextPage = (response.body.links && response.body.links.pages) ? response.body.links.pages.next : null;
+    } while (nextPage);
 
-        if (error) return callback(error);
-
-        return callback(null, matchingRecords);
-    });
+    return matchingRecords;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const records = await getZoneRecords(domainConfig, zoneName, name, type);
 
-        // used to track available records to update instead of create
-        var i = 0, recordIds = [];
+    // used to track available records to update instead of create
+    let i = 0, recordIds = [];
 
-        async.eachSeries(values, function (value, iteratorCallback) {
-            var priority = null;
+    for (let value of values) {
+        let priority = null;
 
-            if (type === 'MX') {
-                priority = value.split(' ')[0];
-                value = value.split(' ')[1];
-            }
+        if (type === 'MX') {
+            priority = value.split(' ')[0];
+            value = value.split(' ')[1];
+        }
 
-            var data = {
-                type: type,
-                name: name,
-                data: value,
-                priority: priority,
-                ttl: 30 // Recent DO DNS API break means this value must atleast be 30
-            };
+        const data = {
+            type: type,
+            name: name,
+            data: value,
+            priority: priority,
+            ttl: 30 // Recent DO DNS API break means this value must atleast be 30
+        };
 
-            if (i >= result.length) {
-                superagent.post(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records')
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode === 422) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, result.body.message));
-                        if (result.statusCode !== 201) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        if (i >= records.length) {
+            const [error, response] = await safe(superagent.post(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records')
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        recordIds.push(safe.query(result.body, 'domain_record.id'));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
+            if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        return iteratorCallback(null);
-                    });
-            } else {
-                superagent.put(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + result[i].id)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                    // increment, as we have consumed the record
-                        ++i;
+            recordIds.push(safe.query(records.body, 'domain_record.id'));
+        } else {
+            const [error, response] = await safe(superagent.put(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + records[i].id)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode === 422) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, result.body.message));
-                        if (result.statusCode !== 200) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            ++i;
 
-                        recordIds.push(safe.query(result.body, 'domain_record.id'));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        return iteratorCallback(null);
-                    });
-            }
-        }, function (error) {
-            if (error) return callback(error);
+            recordIds.push(safe.query(records.body, 'domain_record.id'));
+        }
+    }
 
-            debug('upsert: completed with recordIds:%j', recordIds);
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] = await safe(superagent.del(`${DIGITALOCEAN_ENDPOINT}/v2/domains/${zoneName}/records/${records[j].id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
 
-            callback();
-        });
-    });
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
+
+    debug('upsert: completed with recordIds:%j', recordIds);
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const result = await getZoneRecords(domainConfig, zoneName, name, type);
 
-        // We only return the value string
-        var tmp = result.map(function (record) { return record.data; });
-
-        debug('get: %j', tmp);
-
-        return callback(null, tmp);
-    });
+    const tmp = result.map(function (record) { return record.data; });
+    return tmp;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const result = await getZoneRecords(domainConfig, zoneName, name, type);
+    if (result.length === 0) return;
 
-        if (result.length === 0) return callback(null);
+    const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
+    if (tmp.length === 0) return;
 
-        var tmp = result.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
-
-        debug('del: %j', tmp);
-
-        if (tmp.length === 0) return callback(null);
-
-        // FIXME we only handle the first one currently
-
-        superagent.del(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + tmp[0].id)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
+    for (const r of tmp) {
+        const [error, response] = await safe(superagent.del(`${DIGITALOCEAN_ENDPOINT}/v2/domains/${zoneName}/records/${r.id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
             .timeout(30 * 1000)
             .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return callback(null);
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            .ok(() => true));
 
-                debug('del: done');
-
-                return callback(null);
-            });
-    });
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) return;
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        token: dnsConfig.token
+    const credentials = {
+        token: domainConfig.token
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.digitalocean.com') === -1) {
-            debug('verifyDnsConfig: %j does not contains DO NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to DigitalOcean', { field: 'nameservers' }));
-        }
+    if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.digitalocean.com') === -1) {
+        debug('verifyDomainConfig: %j does not contains DO NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to DigitalOcean');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/gandi.js

@@ -1,26 +1,27 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/gandi'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
 
-var GANDI_API = 'https://dns.api.gandi.net/api/v5';
+const GANDI_API = 'https://dns.api.gandi.net/api/v5';
 
 function formatError(response) {
     return util.format(`Gandi DNS error [${response.statusCode}] ${response.body.message}`);
@@ -35,146 +36,126 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    var data = {
+    const data = {
         'rrset_ttl': 300, // this is the minimum allowed
         'rrset_values': values // for mx records, value is already of the '<priority> <server>' format
     };
 
-    superagent.put(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
-        .set('X-Api-Key', dnsConfig.token)
+    const [error, response] = await safe(superagent.put(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
+        .set('X-Api-Key', domainConfig.token)
         .timeout(30 * 1000)
         .send(data)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            return callback(null);
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+    if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type}`);
 
-    superagent.get(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
-        .set('X-Api-Key', dnsConfig.token)
+    const [error, response] = await safe(superagent.get(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
+        .set('X-Api-Key', domainConfig.token)
         .timeout(30 * 1000)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode === 404) return callback(null, [ ]);
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            debug('get: %j', result.body);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode === 404) return [];
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-            return callback(null, result.body.rrset_values);
-        });
+    return response.body.rrset_values;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    superagent.del(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
-        .set('X-Api-Key', dnsConfig.token)
+    const [error, response] = await safe(superagent.del(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
+        .set('X-Api-Key', domainConfig.token)
         .timeout(30 * 1000)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 404) return callback(null);
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            debug('del: done');
-
-            return callback(null);
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 404) return;
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
 
-    var credentials = {
-        token: dnsConfig.token
+    const credentials = {
+        token: domainConfig.token
     };
 
     const ip = '127.0.0.1';
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.gandi.net') !== -1; })) {
-            debug('verifyDnsConfig: %j does not contain Gandi NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Gandi', { field: 'nameservers' }));
-        }
+    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.gandi.net') !== -1; })) {
+        debug('verifyDomainConfig: %j does not contain Gandi NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Gandi');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/gcdns.js

@@ -1,23 +1,24 @@
 'use strict';
 
+const safe = require('safetydance');
+
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/gcdns'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
     GCDNS = require('@google-cloud/dns').DNS,
-    util = require('util'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
@@ -30,210 +31,166 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.credentials.private_key === constants.SECRET_PLACEHOLDER && currentConfig.credentials) newConfig.credentials.private_key = currentConfig.credentials.private_key;
 }
 
-function getDnsCredentials(dnsConfig) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+function getDnsCredentials(domainConfig) {
+    assert.strictEqual(typeof domainConfig, 'object');
 
     return {
-        projectId: dnsConfig.projectId,
+        projectId: domainConfig.projectId,
         credentials: {
-            client_email: dnsConfig.credentials.client_email,
-            private_key: dnsConfig.credentials.private_key
+            client_email: domainConfig.credentials.client_email,
+            private_key: domainConfig.credentials.private_key
         }
     };
 }
 
-function getZoneByName(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneByName(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var gcdns = new GCDNS(getDnsCredentials(dnsConfig));
+    const gcdns = new GCDNS(getDnsCredentials(domainConfig));
 
-    gcdns.getZones(function (error, zones) {
-        if (error && error.message === 'invalid_grant') return callback(new BoxError(BoxError.ACCESS_DENIED, 'The key was probably revoked'));
-        if (error && error.reason === 'No such domain') return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-        if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-        if (error && error.code === 404) return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-        if (error) {
-            debug('gcdns.getZones', error);
-            return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-        }
+    const [error, result] = await safe(gcdns.getZones());
+    if (error && error.message === 'invalid_grant') throw new BoxError(BoxError.ACCESS_DENIED, 'The key was probably revoked');
+    if (error && error.reason === 'No such domain') throw new BoxError(BoxError.NOT_FOUND, error.message);
+    if (error && error.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 404) throw new BoxError(BoxError.NOT_FOUND, error.message);
+    if (error) {
+        debug('gcdns.getZones', error);
+        throw new BoxError(BoxError.EXTERNAL_ERROR, error);
+    }
 
-        var zone = zones.filter(function (zone) {
-            return zone.metadata.dnsName.slice(0, -1) === zoneName;     // the zone name contains a '.' at the end
-        })[0];
+    const zone = result[0].filter(function (zone) {
+        return zone.metadata.dnsName.slice(0, -1) === zoneName;     // the zone name contains a '.' at the end
+    })[0];
 
-        if (!zone) return callback(new BoxError(BoxError.NOT_FOUND, 'no such zone'));
+    if (!zone) throw new BoxError(BoxError.NOT_FOUND, 'no such zone');
 
-        callback(null, zone); //zone.metadata ~= {name="", dnsName="", nameServers:[]}
-    });
+    return zone; //zone.metadata ~= {name="", dnsName="", nameServers:[]}
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
-    getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
 
-        zone.getRecords({ type: type, name: fqdn + '.' }, function (error, oldRecords) {
-            if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) {
-                debug('upsert->zone.getRecords', error);
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-            }
+    const [error, result] = await safe(zone.getRecords({ type: type, name: fqdn + '.' }));
+    if (error && error.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 
-            var newRecord = zone.record(type, {
-                name: fqdn + '.',
-                data: values,
-                ttl: 1
-            });
-
-            zone.createChange({ delete: oldRecords, add: newRecord }, function(error /*, change */) {
-                if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-                if (error && error.code === 412) return callback(new BoxError(BoxError.BUSY, error.message));
-                if (error) {
-                    debug('upsert->zone.createChange', error);
-                    return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-                }
-
-                callback(null);
-            });
-        });
+    const newRecord = zone.record(type, {
+        name: fqdn + '.',
+        data: values,
+        ttl: 1
     });
+
+    const [changeError] = await safe(zone.createChange({ delete: result[0], add: newRecord }));
+    if (changeError && changeError.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, changeError.message);
+    if (changeError && changeError.code === 412) throw new BoxError(BoxError.BUSY, changeError.message);
+    if (changeError) throw new BoxError(BoxError.EXTERNAL_ERROR, changeError.message);
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
 
-        var params = {
-            name: fqdn + '.',
-            type: type
-        };
+    const params = {
+        name: fqdn + '.',
+        type: type
+    };
 
-        zone.getRecords(params, function (error, records) {
-            if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-            if (records.length === 0) return callback(null, [ ]);
+    const [error, result] = await safe(zone.getRecords(params));
+    if (error && error.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
+    if (result[0].length === 0) return [];
 
-            return callback(null, records[0].data);
-        });
-    });
+    return result[0][0].data;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
 
-        zone.getRecords({ type: type, name: fqdn + '.' }, function(error, oldRecords) {
-            if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) {
-                debug('del->zone.getRecords', error);
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-            }
+    const [error, result] = await safe(zone.getRecords({ type: type, name: fqdn + '.' }));
+    if (error && error.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 
-            zone.deleteRecords(oldRecords, function (error, change) {
-                if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-                if (error && error.code === 412) return callback(new BoxError(BoxError.BUSY, error.message));
-                if (error) {
-                    debug('del->zone.createChange', error);
-                    return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-                }
-
-                callback(null, change.id);
-            });
-        });
-    });
+    const [delError] = await safe(zone.deleteRecords(result[0]));
+    if (delError && delError.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, delError.message);
+    if (delError && delError.code === 412) throw new BoxError(BoxError.BUSY, delError.message);
+    if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, delError.message);
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (typeof dnsConfig.projectId !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'projectId must be a string', { field: 'projectId' }));
-    if (!dnsConfig.credentials || typeof dnsConfig.credentials !== 'object') return callback(new BoxError(BoxError.BAD_FIELD, 'credentials must be an object', { field: 'credentials' }));
-    if (typeof dnsConfig.credentials.client_email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'credentials.client_email must be a string', { field: 'client_email' }));
-    if (typeof dnsConfig.credentials.private_key !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'credentials.private_key must be a string', { field: 'private_key' }));
+    if (typeof domainConfig.projectId !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'projectId must be a string');
+    if (!domainConfig.credentials || typeof domainConfig.credentials !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'credentials must be an object');
+    if (typeof domainConfig.credentials.client_email !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.client_email must be a string');
+    if (typeof domainConfig.credentials.private_key !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.private_key must be a string');
 
-    var credentials = getDnsCredentials(dnsConfig);
+    const credentials = getDnsCredentials(domainConfig);
 
     const ip = '127.0.0.1';
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        getZoneByName(credentials, zoneName, function (error, zone) {
-            if (error) return callback(error);
+    const zone = await getZoneByName(credentials, zoneName);
 
-            var definedNS = zone.metadata.nameServers.sort().map(function(r) { return r.replace(/\.$/, ''); });
-            if (!_.isEqual(definedNS, nameservers.sort())) {
-                debug('verifyDnsConfig: %j and %j do not match', nameservers, definedNS);
-                return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS', { field: 'nameservers' }));
-            }
+    const definedNS = zone.metadata.nameServers.sort().map(function(r) { return r.replace(/\.$/, ''); });
+    if (!_.isEqual(definedNS, nameservers.sort())) {
+        debug('verifyDomainConfig: %j and %j do not match', nameservers, definedNS);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS');
+    }
 
-            const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            upsert(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-                debug('verifyDnsConfig: Test A record added');
-
-                del(domainObject, location, 'A', [ ip ], function (error) {
-                    if (error) return callback(error);
-
-                    debug('verifyDnsConfig: Test A record removed again');
-
-                    callback(null, credentials);
-                });
-            });
-        });
-    });
+    return credentials;
 }

src/dns/godaddy.js

@@ -1,21 +1,22 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/godaddy'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -26,6 +27,7 @@ const GODADDY_API = 'https://api.godaddy.com/v1/domains';
 // this is a workaround for godaddy not having a delete API
 // https://stackoverflow.com/questions/39347464/delete-record-libcloud-godaddy-api
 const GODADDY_INVALID_IP = '0.0.0.0';
+const GODADDY_INVALID_IPv6 = '0:0:0:0:0:0:0:0';
 const GODADDY_INVALID_TXT = '""';
 
 function formatError(response) {
@@ -41,22 +43,21 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.apiSecret === constants.SECRET_PLACEHOLDER) newConfig.apiSecret = currentConfig.apiSecret;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    var records = [ ];
-    values.forEach(function (value) {
-        var record = { ttl: 600 }; // 600 is the min ttl
+    const records = [];
+    for (const value of values) {
+        const record = { ttl: 600 }; // 600 is the min ttl
 
         if (type === 'MX') {
             record.priority = parseInt(value.split(' ')[0], 10);
@@ -66,152 +67,135 @@ function upsert(domainObject, location, type, values, callback) {
         }
 
         records.push(record);
-    });
+    }
 
-    superagent.put(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
-        .set('Authorization', `sso-key ${dnsConfig.apiKey}:${dnsConfig.apiSecret}`)
+    const [error, response] = await safe(superagent.put(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
+        .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
         .timeout(30 * 1000)
         .send(records)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, formatError(result))); // no such zone
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.BAD_FIELD, formatError(result))); // conflict
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            return callback(null);
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response)); // no such zone
+    if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, formatError(response)); // conflict
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type}`);
 
-    superagent.get(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
-        .set('Authorization', `sso-key ${dnsConfig.apiKey}:${dnsConfig.apiSecret}`)
+    const [error, response] = await safe(superagent.get(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
+        .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
         .timeout(30 * 1000)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode === 404) return callback(null, [ ]);
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            debug('get: %j', result.body);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode === 404) return [];
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-            var values = result.body.map(function (record) { return record.data; });
+    const values = response.body.map(function (record) { return record.data; });
 
-            if (values.length === 1 && values[0] === GODADDY_INVALID_IP) return callback(null, [ ]); // pretend this record doesn't exist
+    if (values.length === 1) {
+        if ((type === 'A' && values[0] === GODADDY_INVALID_IP)
+            || (type === 'AAAA' && values[0] === GODADDY_INVALID_IPv6)
+            || (type === 'TXT' && values[0] === GODADDY_INVALID_TXT)) return []; // pretend this record doesn't exist
+    }
 
-            return callback(null, values);
-        });
+    return values;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Record deletion is not supported by GoDaddy API'));
+    if (type !== 'A' && type !== 'AAAA' && type !== 'TXT') throw new BoxError(BoxError.EXTERNAL_ERROR, 'Record deletion is not supported by GoDaddy API');
 
     // check if the record exists at all so that we don't insert the "Dead" record for no reason
-    get(domainObject, location, type, function (error, values) {
-        if (error) return callback(error);
-        if (values.length === 0) return callback();
+    const existingRecords = await get(domainObject, location, type);
+    if (existingRecords.length === 0) return;
 
-        // godaddy does not have a delete API. so fill it up with an invalid IP that we can ignore in future get()
-        var records = [{
-            ttl: 600,
-            data: type === 'A' ? GODADDY_INVALID_IP : GODADDY_INVALID_TXT
-        }];
+    // godaddy does not have a delete API. so fill it up with an invalid IP that we can ignore in future get()
+    const records = [{
+        ttl: 600,
+        data: type === 'A' ? GODADDY_INVALID_IP : (type === 'AAAA' ? GODADDY_INVALID_IPv6 : GODADDY_INVALID_TXT)
+    }];
 
-        superagent.put(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
-            .set('Authorization', `sso-key ${dnsConfig.apiKey}:${dnsConfig.apiSecret}`)
-            .send(records)
-            .timeout(30 * 1000)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return callback(null);
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+    const [error, response] = await safe(superagent.put(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
+        .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
+        .send(records)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-                debug('del: done');
-
-                return callback(null);
-            });
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 404) return;
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.apiKey || typeof dnsConfig.apiKey !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string', { field: 'apiKey' }));
-    if (!dnsConfig.apiSecret || typeof dnsConfig.apiSecret !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiSecret must be a non-empty string', { field: 'apiSecret' }));
+    if (!domainConfig.apiKey || typeof domainConfig.apiKey !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string');
+    if (!domainConfig.apiSecret || typeof domainConfig.apiSecret !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'apiSecret must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        apiKey: dnsConfig.apiKey,
-        apiSecret: dnsConfig.apiSecret
+    const credentials = {
+        apiKey: domainConfig.apiKey,
+        apiSecret: domainConfig.apiSecret
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1; })) {
-            debug('verifyDnsConfig: %j does not contain GoDaddy NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to GoDaddy', { field: 'nameservers' }));
-        }
+    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1; })) {
+        debug('verifyDomainConfig: %j does not contain GoDaddy NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to GoDaddy');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/interface.js

@@ -7,18 +7,17 @@
 // -------------------------------------------
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
-    BoxError = require('../boxerror.js'),
-    util = require('util');
+const assert = require('assert'),
+    BoxError = require('../boxerror.js');
 
 function removePrivateFields(domainObject) {
     // in-place removal of tokens and api keys with constants.SECRET_PLACEHOLDER
@@ -30,57 +29,50 @@ function injectPrivateFields(newConfig, currentConfig) {
     // in-place injection of tokens and api keys which came in with constants.SECRET_PLACEHOLDER
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, subdomain, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     // Result: none
 
-    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upsert is not implemented'));
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'upsert is not implemented');
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, subdomain, type) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // Result: Array of matching DNS records in string format
 
-    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'get is not implemented'));
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'get is not implemented');
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, subdomain, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     // Result: none
 
-    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'del is not implemented'));
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'del is not implemented');
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
-
-    callback();
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    // Result: dnsConfig object
+    // Result: domainConfig object
 
-    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'verifyDnsConfig is not implemented'));
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'verifyDomainConfig is not implemented');
 }

src/dns/linode.js

@@ -7,16 +7,16 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
-let async = require('async'),
-    assert = require('assert'),
+const assert = require('assert'),
     constants = require('../constants.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/linode'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -36,278 +36,232 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function getZoneId(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneId(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // returns 100 at a time
-    superagent.get(`${LINODE_ENDPOINT}/domains`)
-        .set('Authorization', 'Bearer ' + dnsConfig.token)
+    const [error, response] = await safe(superagent.get(`${LINODE_ENDPOINT}/domains`)
+        .set('Authorization', 'Bearer ' + domainConfig.token)
         .timeout(30 * 1000)
         .retry(5)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            if (!Array.isArray(result.body.data)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-            const zone = result.body.data.find(d => d.domain === zoneName);
+    if (!Array.isArray(response.body.data)) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response');
 
-            if (!zone || !zone.id) return callback(new BoxError(BoxError.NOT_FOUND, 'Zone not found'));
+    const zone = response.body.data.find(d => d.domain === zoneName);
 
-            debug(`getZoneId: zone id of ${zoneName} is ${zone.id}`);
+    if (!zone || !zone.id) throw new BoxError(BoxError.NOT_FOUND, 'Zone not found');
 
-            callback(null, zone.id);
-        });
+    debug(`getZoneId: zone id of ${zoneName} is ${zone.id}`);
+
+    return zone.id;
 }
 
-function getZoneRecords(dnsConfig, zoneName, name, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneRecords(domainConfig, zoneName, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
 
-    getZoneId(dnsConfig, zoneName, function (error, zoneId) {
-        if (error) return callback(error);
+    const zoneId = await getZoneId(domainConfig, zoneName);
 
-        let page = 0, more = false;
-        let records = [];
+    let page = 0, more = false;
+    let records = [];
 
-        async.doWhilst(function (iteratorDone) {
-            const url = `${LINODE_ENDPOINT}/domains/${zoneId}/records?page=${++page}`;
+    do {
+        const url = `${LINODE_ENDPOINT}/domains/${zoneId}/records?page=${++page}`;
 
-            superagent.get(url)
-                .set('Authorization', 'Bearer ' + dnsConfig.token)
-                .timeout(30 * 1000)
-                .retry(5)
-                .end(function (error, result) {
-                    if (error && !error.response) return iteratorDone(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                    if (result.statusCode === 404) return iteratorDone(new BoxError(BoxError.NOT_FOUND, formatError(result)));
-                    if (result.statusCode === 403 || result.statusCode === 401) return iteratorDone(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                    if (result.statusCode !== 200) return iteratorDone(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        const [error, response] = await safe(superagent.get(url)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
 
-                    records = records.concat(result.body.data.filter(function (record) {
-                        return (record.type === type && record.name === name);
-                    }));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                    more = result.body.page !== result.body.pages;
+        records = records.concat(response.body.data.filter(function (record) {
+            return (record.type === type && record.name === name);
+        }));
 
-                    iteratorDone();
-                });
-        }, function (testDone) { return testDone(null, more); }, function (error) {
-            debug('getZoneRecords:', error, JSON.stringify(records));
+        more = response.body.page !== response.body.pages;
+    } while (more);
 
-            if (error) return callback(error);
-
-            callback(null, { zoneId, records });
-        });
-    });
+    return { zoneId, records };
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
-
-        const { records } = result;
-        var tmp = records.map(function (record) { return record.target; });
-
-        debug('get: %j', tmp);
-
-        return callback(null, tmp);
-    });
+    const { records } = await getZoneRecords(domainConfig, zoneName, name, type);
+    const tmp = records.map(function (record) { return record.target; });
+    return tmp;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const { zoneId, records } = await getZoneRecords(domainConfig, zoneName, name, type);
+    let i = 0, recordIds = []; // used to track available records to update instead of create
 
-        const { zoneId, records } = result;
-        let i = 0, recordIds = []; // used to track available records to update instead of create
+    for (const value of values) {
+        const data = {
+            type: type,
+            ttl_sec: 300 // lowest
+        };
 
-        async.eachSeries(values, function (value, iteratorCallback) {
-            let data = {
-                type: type,
-                ttl_sec: 300 // lowest
-            };
+        if (type === 'MX') {
+            data.priority = parseInt(value.split(' ')[0], 10);
+            data.target = value.split(' ')[1];
+        } else if (type === 'TXT') {
+            data.target = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
+        } else {
+            data.target = value;
+        }
 
-            if (type === 'MX') {
-                data.priority = parseInt(value.split(' ')[0], 10);
-                data.target = value.split(' ')[1];
-            } else if (type === 'TXT') {
-                data.target = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
-            } else {
-                data.target = value;
-            }
+        if (i >= records.length) {
+            data.name = name; // only set for new records
 
-            if (i >= records.length) {
-                data.name = name; // only set for new records
+            const [error, response] = await safe(superagent.post(`${LINODE_ENDPOINT}/domains/${zoneId}/records`)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                superagent.post(`${LINODE_ENDPOINT}/domains/${zoneId}/records`)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode !== 200) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        recordIds.push(result.body.id);
+            recordIds.push(response.body.id);
+        } else {
+            const [error, response] = await safe(superagent.put(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${records[i].id}`)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        return iteratorCallback(null);
-                    });
-            } else {
-                superagent.put(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${records[i].id}`)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        // increment, as we have consumed the record
-                        ++i;
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode !== 200) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            ++i;
 
-                        recordIds.push(result.body.id);
+            recordIds.push(response.body.id);
+        }
+    }
 
-                        return iteratorCallback(null);
-                    });
-            }
-        }, function (error) {
-            if (error) return callback(error);
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] =  await safe(superagent.del(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${records[j].id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
 
-            debug('upsert: completed with recordIds:%j', recordIds);
-
-            callback();
-        });
-    });
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const { zoneId, records } = await getZoneRecords(domainConfig, zoneName, name, type);
+    if (records.length === 0) return;
 
-        const { zoneId, records } = result;
-        if (records.length === 0) return callback(null);
+    const tmp = records.filter(function (record) { return values.some(function (value) { return value === record.target; }); });
+    if (tmp.length === 0) return;
 
-        var tmp = records.filter(function (record) { return values.some(function (value) { return value === record.target; }); });
-
-        debug('del: %j', tmp);
-
-        if (tmp.length === 0) return callback(null);
-
-        // FIXME we only handle the first one currently
-
-        superagent.del(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${tmp[0].id}`)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
+    for (const r of tmp) {
+        const [error, response] = await safe(superagent.del(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${r.id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
             .timeout(30 * 1000)
             .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return callback(null);
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                debug('del: done');
-
-                return callback(null);
-            });
-    });
+            .ok(() => true));
+        if (error && !error.response) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) return;
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        token: dnsConfig.token
+    const credentials = {
+        token: domainConfig.token
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) {
-            debug('verifyDnsConfig: %j does not contains linode NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode', { field: 'nameservers' }));
-        }
+    if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) {
+        debug('verifyDomainConfig: %j does not contains linode NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/manual.js

@@ -1,21 +1,21 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/manual'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
-    util = require('util'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
     waitForDns = require('./waitfordns.js');
 
 function removePrivateFields(domainObject) {
@@ -27,61 +27,55 @@ function injectPrivateFields(newConfig, currentConfig) {
 
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
 
-    return callback(null);
+    return;
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    callback(null, [ ]); // returning ip confuses apptask into thinking the entry already exists
+    return []; // returning ip confuses apptask into thinking the entry already exists
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    return callback();
+    return;
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     const zoneName = domainObject.zoneName;
 
     // Very basic check if the nameservers can be fetched
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        callback(null, {});
-    });
+    return {};
 }

src/dns/namecheap.js

@@ -1,25 +1,25 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    verifyDnsConfig: verifyDnsConfig,
-    wait: wait
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    verifyDomainConfig,
+    wait
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecheap'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
-    querystring = require('querystring'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     sysinfo = require('../sysinfo.js'),
+    util = require('util'),
     waitForDns = require('./waitfordns.js'),
     xml2js = require('xml2js');
 
@@ -34,289 +34,247 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function getQuery(dnsConfig, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
+async function getQuery(domainConfig) {
+    assert.strictEqual(typeof domainConfig, 'object');
 
-    sysinfo.getServerIp(function (error, ip) {
-        if (error) return callback(error);
+    const ip = await sysinfo.getServerIPv4(); // only supports ipv4
 
-        callback(null, {
-            ApiUser: dnsConfig.username,
-            ApiKey: dnsConfig.token,
-            UserName: dnsConfig.username,
-            ClientIp: ip
-        });
-    });
+    return {
+        ApiUser: domainConfig.username,
+        ApiKey: domainConfig.token,
+        UserName: domainConfig.username,
+        ClientIp: ip
+    };
 }
 
-function getZone(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZone(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    getQuery(dnsConfig, function (error, query) {
-        if (error) return callback(error);
+    const query = await getQuery(domainConfig);
+    query.Command = 'namecheap.domains.dns.getHosts';
+    query.SLD = zoneName.split('.')[0];
+    query.TLD = zoneName.split('.')[1];
 
-        query.Command = 'namecheap.domains.dns.getHosts';
-        query.SLD = zoneName.split('.')[0];
-        query.TLD = zoneName.split('.')[1];
+    const [error, response] = await safe(superagent.get(ENDPOINT).query(query).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
 
-        superagent.get(ENDPOINT).query(query).end(function (error, result) {
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+    const parser = new xml2js.Parser();
+    const [parserError, result] = await safe(util.promisify(parser.parseString)(response.text));
+    if (parserError) throw new BoxError(BoxError.EXTERNAL_ERROR, parserError);
 
-            var parser = new xml2js.Parser();
-            parser.parseString(result.text, function (error, result) {
-                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+    const tmp = result.ApiResponse;
+    if (tmp['$'].Status !== 'OK') {
+        const errorMessage = safe.query(tmp, 'Errors[0].Error[0]._', 'Invalid response');
+        if (errorMessage === 'API Key is invalid or API access has not been enabled') throw new BoxError(BoxError.ACCESS_DENIED, errorMessage);
 
-                var tmp = result.ApiResponse;
-                if (tmp['$'].Status !== 'OK') {
-                    var errorMessage = safe.query(tmp, 'Errors[0].Error[0]._', 'Invalid response');
-                    if (errorMessage === 'API Key is invalid or API access has not been enabled') return callback(new BoxError(BoxError.ACCESS_DENIED, errorMessage));
+        throw new BoxError(BoxError.EXTERNAL_ERROR, errorMessage);
+    }
+    const host = safe.query(tmp, 'CommandResponse[0].DomainDNSGetHostsResult[0].host');
+    if (!host) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response: ${JSON.stringify(tmp)}`);
+    if (!Array.isArray(host)) throw new BoxError(BoxError.EXTERNAL_ERROR, `host is not an array: ${JSON.stringify(tmp)}`);
 
-                    return callback(new BoxError(BoxError.EXTERNAL_ERROR, errorMessage));
-                }
-                const host = safe.query(tmp, 'CommandResponse[0].DomainDNSGetHostsResult[0].host');
-                if (!host) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response: ${JSON.stringify(tmp)}`));
-                if (!Array.isArray(host)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `host is not an array: ${JSON.stringify(tmp)}`));
-
-                const hosts = host.map(h => h['$']);
-                callback(null, hosts);
-            });
-        });
-    });
+    const hosts = host.map(h => h['$']);
+    return hosts;
 }
 
-function setZone(dnsConfig, zoneName, hosts, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function setZone(domainConfig, zoneName, hosts) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert(Array.isArray(hosts));
-    assert.strictEqual(typeof callback, 'function');
 
-    getQuery(dnsConfig, function (error, query) {
-        if (error) return callback(error);
+    const query = await getQuery(domainConfig);
+    query.Command = 'namecheap.domains.dns.setHosts';
+    query.SLD = zoneName.split('.')[0];
+    query.TLD = zoneName.split('.')[1];
 
-        query.Command = 'namecheap.domains.dns.setHosts';
-        query.SLD = zoneName.split('.')[0];
-        query.TLD = zoneName.split('.')[1];
+    // Map to query params https://www.namecheap.com/support/api/methods/domains-dns/set-hosts.aspx
+    hosts.forEach(function (host, i) {
+        var n = i+1; // api starts with 1 not 0
+        query['TTL' + n] = '300'; // keep it low
+        query['HostName' + n] = host.HostName || host.Name;
+        query['RecordType' + n] = host.RecordType || host.Type;
+        query['Address' + n] = host.Address;
 
-        // Map to query params https://www.namecheap.com/support/api/methods/domains-dns/set-hosts.aspx
-        hosts.forEach(function (host, i) {
-            var n = i+1; // api starts with 1 not 0
-            query['TTL' + n] = '300'; // keep it low
-            query['HostName' + n] = host.HostName || host.Name;
-            query['RecordType' + n] = host.RecordType || host.Type;
-            query['Address' + n] = host.Address;
-
-            if (host.Type === 'MX') {
-                query['EmailType' + n] = 'MX';
-                if (host.MXPref) query['MXPref' + n] = host.MXPref;
-            }
-        });
-
-        // namecheap recommends sending as POSTDATA with > 10 records
-        const qs = querystring.stringify(query);
-
-        superagent.post(ENDPOINT).set('Content-Type', 'application/x-www-form-urlencoded').send(qs).end(function (error, result) {
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-            var parser = new xml2js.Parser();
-            parser.parseString(result.text, function (error, result) {
-                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-                var tmp = result.ApiResponse;
-                if (tmp['$'].Status !== 'OK') {
-                    var errorMessage = safe.query(tmp, 'Errors[0].Error[0]._', 'Invalid response');
-                    if (errorMessage === 'API Key is invalid or API access has not been enabled') return callback(new BoxError(BoxError.ACCESS_DENIED, errorMessage));
-
-                    return callback(new BoxError(BoxError.EXTERNAL_ERROR, errorMessage));
-                }
-                if (!tmp.CommandResponse[0]) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
-                if (!tmp.CommandResponse[0].DomainDNSSetHostsResult[0]) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
-                if (tmp.CommandResponse[0].DomainDNSSetHostsResult[0]['$'].IsSuccess !== 'true') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
-
-                callback(null);
-            });
-        });
+        if (host.Type === 'MX') {
+            query['EmailType' + n] = 'MX';
+            if (host.MXPref) query['MXPref' + n] = host.MXPref;
+        }
     });
+
+    // namecheap recommends sending as POSTDATA with > 10 records
+    const qs = new URLSearchParams(query).toString();
+
+    const [error, response] = await safe(superagent.post(ENDPOINT).set('Content-Type', 'application/x-www-form-urlencoded').send(qs).ok(() => true));
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error);
+
+    const parser = new xml2js.Parser();
+    const [parserError, result] = await safe(util.promisify(parser.parseString)(response.text));
+    if (parserError) throw new BoxError(BoxError.EXTERNAL_ERROR, parserError.message);
+
+    const tmp = result.ApiResponse;
+    if (tmp['$'].Status !== 'OK') {
+        const errorMessage = safe.query(tmp, 'Errors[0].Error[0]._', 'Invalid response');
+        if (errorMessage === 'API Key is invalid or API access has not been enabled') throw new BoxError(BoxError.ACCESS_DENIED, errorMessage);
+
+        throw new BoxError(BoxError.EXTERNAL_ERROR, errorMessage);
+    }
+    if (!tmp.CommandResponse[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response');
+    if (!tmp.CommandResponse[0].DomainDNSSetHostsResult[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response');
+    if (tmp.CommandResponse[0].DomainDNSSetHostsResult[0]['$'].IsSuccess !== 'true') throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response');
 }
 
-function upsert(domainObject, subdomain, type, values, callback) {
+async function upsert(domainObject, subdomain, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config;
+    const domainConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
-    subdomain = domains.getName(domainObject, subdomain, type) || '@';
+    subdomain = dns.getName(domainObject, subdomain, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getZone(dnsConfig, zoneName, function (error, result) {
-        if (error) return callback(error);
+    const result = await getZone(domainConfig, zoneName);
 
-        // Array to keep track of records that need to be inserted
-        let toInsert = [];
+    // Array to keep track of records that need to be inserted
+    let toInsert = [];
 
-        for (let i = 0; i < values.length; i++) {
-            let curValue = values[i];
-            let wasUpdate = false;
+    for (let i = 0; i < values.length; i++) {
+        let curValue = values[i];
+        let wasUpdate = false;
 
-            for (let j = 0; j < result.length; j++) {
-                let curHost = result[j];
+        for (let j = 0; j < result.length; j++) {
+            let curHost = result[j];
 
-                if (curHost.Type === type && curHost.Name === subdomain) {
-                    // Updating an already existing host
-                    wasUpdate = true;
-                    if (type === 'MX') {
-                        curHost.MXPref = curValue.split(' ')[0];
-                        curHost.Address = curValue.split(' ')[1];
-                    } else {
-                        curHost.Address = curValue;
-                    }
-                }
-            }
-
-            // We don't have this host at all yet, let's push to toInsert array
-            if (!wasUpdate) {
-                let newRecord = {
-                    RecordType: type,
-                    HostName: subdomain,
-                    Address: curValue
-                };
-
-                // Special case for MX records
+            if (curHost.Type === type && curHost.Name === subdomain) {
+                // Updating an already existing host
+                wasUpdate = true;
                 if (type === 'MX') {
-                    newRecord.MXPref = curValue.split(' ')[0];
-                    newRecord.Address = curValue.split(' ')[1];
+                    curHost.MXPref = curValue.split(' ')[0];
+                    curHost.Address = curValue.split(' ')[1];
+                } else {
+                    curHost.Address = curValue;
                 }
-
-                toInsert.push(newRecord);
-
             }
         }
 
-        const hosts = result.concat(toInsert);
+        // We don't have this host at all yet, let's push to toInsert array
+        if (!wasUpdate) {
+            let newRecord = {
+                RecordType: type,
+                HostName: subdomain,
+                Address: curValue
+            };
 
-        setZone(dnsConfig, zoneName, hosts, callback);
-    });
+            // Special case for MX records
+            if (type === 'MX') {
+                newRecord.MXPref = curValue.split(' ')[0];
+                newRecord.Address = curValue.split(' ')[1];
+            }
+
+            toInsert.push(newRecord);
+
+        }
+    }
+
+    const hosts = result.concat(toInsert);
+
+    return await setZone(domainConfig, zoneName, hosts);
 }
 
-function get(domainObject, subdomain, type, callback) {
+async function get(domainObject, subdomain, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config;
+    const domainConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
-    subdomain = domains.getName(domainObject, subdomain, type) || '@';
+    subdomain = dns.getName(domainObject, subdomain, type) || '@';
 
-    getZone(dnsConfig, zoneName, function (error, result) {
-        if (error) return callback(error);
+    const result = await getZone(domainConfig, zoneName);
 
-        // We need to filter hosts to ones with this subdomain and type
-        const actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
+    // We need to filter hosts to ones with this subdomain and type
+    const actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
 
-        // We only return the value string
-        const tmp = actualHosts.map(function (record) { return record.Address; });
-
-        debug(`get: subdomain: ${subdomain} type:${type} value:${JSON.stringify(tmp)}`);
-
-        return callback(null, tmp);
-    });
+    const tmp = actualHosts.map(function (record) { return record.Address; });
+    return tmp;
 }
 
-function del(domainObject, subdomain, type, values, callback) {
+async function del(domainObject, subdomain, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config;
+    const domainConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
-    subdomain = domains.getName(domainObject, subdomain, type) || '@';
+    subdomain = dns.getName(domainObject, subdomain, type) || '@';
 
     debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getZone(dnsConfig, zoneName, function (error, result) {
-        if (error) return callback(error);
+    let result = await getZone(domainConfig, zoneName);
+    if (result.length === 0) return;
+    const originalLength = result.length;
 
-        if (result.length === 0) return callback();
-        const originalLength = result.length;
+    for (let i = 0; i < values.length; i++) {
+        let curValue = values[i];
 
-        for (let i = 0; i < values.length; i++) {
-            let curValue = values[i];
+        result = result.filter(curHost => curHost.Type !== type || curHost.Name !== subdomain || curHost.Address !== curValue);
+    }
 
-            result = result.filter(curHost => curHost.Type !== type || curHost.Name !== subdomain || curHost.Address !== curValue);
-        }
-
-        if (result.length !== originalLength) return setZone(dnsConfig, zoneName, result, callback);
-
-        callback();
-    });
+    if (result.length !== originalLength) return await setZone(domainConfig, zoneName, result);
 }
 
-function verifyDnsConfig(domainObject, callback) {
-    assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const dnsConfig = domainObject.config;
-    const zoneName = domainObject.zoneName;
-    const ip = '127.0.0.1';
-
-    if (!dnsConfig.username || typeof dnsConfig.username !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'username must be a non-empty string', { field: 'username' }));
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
-
-    let credentials = {
-        username: dnsConfig.username,
-        token: dnsConfig.token
-    };
-
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
-
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
-
-        if (nameservers.some(function (n) { return n.toLowerCase().indexOf('.registrar-servers.com') === -1; })) {
-            debug('verifyDnsConfig: %j does not contains NC NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to NameCheap', { field: 'nameservers' }));
-        }
-
-        const testSubdomain = 'cloudrontestdns';
-
-        upsert(domainObject, testSubdomain, 'A', [ip], function (error, changeId) {
-            if (error) return callback(error);
-
-            debug('verifyDnsConfig: Test A record added with change id %s', changeId);
-
-            del(domainObject, testSubdomain, 'A', [ip], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
-}
-
-function wait(domainObject, subdomain, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(subdomain, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
+}
+
+async function verifyDomainConfig(domainObject) {
+    assert.strictEqual(typeof domainObject, 'object');
+
+    const domainConfig = domainObject.config;
+    const zoneName = domainObject.zoneName;
+    const ip = '127.0.0.1';
+
+    if (!domainConfig.username || typeof domainConfig.username !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'username must be a non-empty string');
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
+
+    const credentials = {
+        username: domainConfig.username,
+        token: domainConfig.token
+    };
+
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
+
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
+
+    if (nameservers.some(function (n) { return n.toLowerCase().indexOf('.registrar-servers.com') === -1; })) {
+        debug('verifyDomainConfig: %j does not contains NC NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to NameCheap');
+    }
+
+    const testSubdomain = 'cloudrontestdns';
+
+    await upsert(domainObject, testSubdomain, 'A', [ip]);
+    debug('verifyDomainConfig: Test A record added');
+
+    await del(domainObject, testSubdomain, 'A', [ip]);
+    debug('verifyDomainConfig: Test A record removed again');
+
+    return credentials;
 }

src/dns/namecom.js

@@ -1,24 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecom'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
-    util = require('util'),
     waitForDns = require('./waitfordns.js');
 
 const NAMECOM_API = 'https://api.name.com/v4';
@@ -36,17 +35,16 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function addRecord(dnsConfig, zoneName, name, type, values, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function addRecord(domainConfig, zoneName, name, type, values) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`add: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    var data = {
+    const data = {
         host: name,
         type: type,
         ttl: 300    // 300 is the lowest
@@ -63,31 +61,28 @@ function addRecord(dnsConfig, zoneName, name, type, values, callback) {
         data.answer = values[0];
     }
 
-    superagent.post(`${NAMECOM_API}/domains/${zoneName}/records`)
-        .auth(dnsConfig.username, dnsConfig.token)
+    const [error, response] = await safe(superagent.post(`${NAMECOM_API}/domains/${zoneName}/records`)
+        .auth(domainConfig.username, domainConfig.token)
         .timeout(30 * 1000)
         .send(data)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            return callback(null, 'unused-id');
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function updateRecord(dnsConfig, zoneName, recordId, name, type, values, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function updateRecord(domainConfig, zoneName, recordId, name, type, values) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof recordId, 'number');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`update:${recordId} on ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    var data = {
+    const data = {
         host: name,
         type: type,
         ttl: 300    // 300 is the lowest
@@ -104,184 +99,152 @@ function updateRecord(dnsConfig, zoneName, recordId, name, type, values, callbac
         data.answer = values[0];
     }
 
-    superagent.put(`${NAMECOM_API}/domains/${zoneName}/records/${recordId}`)
-        .auth(dnsConfig.username, dnsConfig.token)
+    const [error, response] = await safe(superagent.put(`${NAMECOM_API}/domains/${zoneName}/records/${recordId}`)
+        .auth(domainConfig.username, domainConfig.token)
         .timeout(30 * 1000)
         .send(data)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            return callback(null);
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function getInternal(dnsConfig, zoneName, name, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getInternal(domainConfig, zoneName, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`getInternal: ${name} in zone ${zoneName} of type ${type}`);
 
-    superagent.get(`${NAMECOM_API}/domains/${zoneName}/records`)
-        .auth(dnsConfig.username, dnsConfig.token)
+    const [error, response] = await safe(superagent.get(`${NAMECOM_API}/domains/${zoneName}/records`)
+        .auth(domainConfig.username, domainConfig.token)
         .timeout(30 * 1000)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            // name.com does not return the correct content-type
-            result.body = safe.JSON.parse(result.text);
-            if (!result.body.records) result.body.records = [];
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-            result.body.records.forEach(function (r) {
-                // name.com api simply strips empty properties
-                r.host = r.host || '';
-            });
+    // name.com does not return the correct content-type
+    response.body = safe.JSON.parse(response.text);
+    if (!response.body.records) response.body.records = [];
 
-            var results = result.body.records.filter(function (r) {
-                return (r.host === name && r.type === type);
-            });
+    response.body.records.forEach(function (r) {
+        // name.com api simply strips empty properties
+        r.host = r.host || '';
+    });
 
-            debug('getInternal: %j', results);
+    const results = response.body.records.filter(function (r) {
+        return (r.host === name && r.type === type);
+    });
 
-            return callback(null, results);
-        });
+    return results;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const result = await getInternal(domainConfig, zoneName, name, type);
+    if (result.length === 0) return await addRecord(domainConfig, zoneName, name, type, values);
 
-        if (result.length === 0) return addRecord(dnsConfig, zoneName, name, type, values, callback);
-
-        return updateRecord(dnsConfig, zoneName, result[0].id, name, type, values, callback);
-    });
+    return await updateRecord(domainConfig, zoneName, result[0].id, name, type, values);
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
-
-        var tmp = result.map(function (record) { return record.answer; });
-
-        debug('get: %j', tmp);
-
-        return callback(null, tmp);
-    });
+    const result = await getInternal(domainConfig, zoneName, name, type);
+    const tmp = result.map(function (record) { return record.answer; });
+    return tmp;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const result = await getInternal(domainConfig, zoneName, name, type);
+    if (result.length === 0) return;
 
-        if (result.length === 0) return callback();
-
-        superagent.del(`${NAMECOM_API}/domains/${zoneName}/records/${result[0].id}`)
-            .auth(dnsConfig.username, dnsConfig.token)
-            .timeout(30 * 1000)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                return callback(null);
-            });
-    });
+    const [error, response] = await safe(superagent.del(`${NAMECOM_API}/domains/${zoneName}/records/${result[0].id}`)
+        .auth(domainConfig.username, domainConfig.token)
+        .timeout(30 * 1000)
+        .ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (typeof dnsConfig.username !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'username must be a string', { field: 'username' }));
-    if (typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a string', { field: 'token' }));
+    if (typeof domainConfig.username !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'username must be a string');
+    if (typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a string');
 
-    var credentials = {
-        username: dnsConfig.username,
-        token: dnsConfig.token
+    const credentials = {
+        username: domainConfig.username,
+        token: domainConfig.token
     };
 
     const ip = '127.0.0.1';
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.name.com') !== -1; })) {
-            debug('verifyDnsConfig: %j does not contain Name.com NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to name.com', { field: 'nameservers' }));
-        }
+    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.name.com') !== -1; })) {
+        debug('verifyDomainConfig: %j does not contain Name.com NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to name.com');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/netcup.js

@@ -1,26 +1,27 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/netcup'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
 
-var API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON';
+const API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON';
 
 function formatError(response) {
     if (response.body) return util.format('Netcup DNS error [%s] %s', response.body.statuscode, response.body.longmessage);
@@ -37,267 +38,226 @@ function injectPrivateFields(newConfig, currentConfig) {
 }
 
 // returns a api session id
-function login(dnsConfig, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
+async function login(domainConfig) {
+    assert.strictEqual(typeof domainConfig, 'object');
 
     const data = {
         action: 'login',
         param:{
-            apikey: dnsConfig.apiKey,
-            apipassword: dnsConfig.apiPassword,
-            customernumber: dnsConfig.customerNumber
+            apikey: domainConfig.apiKey,
+            apipassword: domainConfig.apiPassword,
+            customernumber: domainConfig.customerNumber
         }
     };
 
-    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+    const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    if (!response.body.responsedata.apisessionid) throw new BoxError(BoxError.ACCESS_DENIED, 'invalid api password');
 
-        callback(null, result.body.responsedata.apisessionid);
-    });
+    return response.body.responsedata.apisessionid;
 }
 
-function getAllRecords(dnsConfig, apiSessionId, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getAllRecords(domainConfig, apiSessionId, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof apiSessionId, 'string');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`getAllRecords: getting dns records of ${zoneName}`);
 
     const data = {
         action: 'infoDnsRecords',
         param:{
-            apikey: dnsConfig.apiKey,
+            apikey: domainConfig.apiKey,
             apisessionid: apiSessionId,
-            customernumber: dnsConfig.customerNumber,
+            customernumber: domainConfig.customerNumber,
             domainname: zoneName,
         }
     };
 
-    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+    const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-        debug('getAllRecords:', JSON.stringify(result.body.responsedata.dnsrecords || []));
-
-        callback(null, result.body.responsedata.dnsrecords || []);
-    });
+    return response.body.responsedata.dnsrecords || [];
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    login(dnsConfig, function (error, apiSessionId) {
-        if (error) return callback(error);
+    const apiSessionId = await login(domainConfig);
 
-        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
-            if (error) return callback(error);
+    const result = await getAllRecords(domainConfig, apiSessionId, zoneName);
 
-            let records = [];
+    let records = [];
 
-            values.forEach(function (value) {
-                // remove possible quotation
-                if (value.charAt(0) === '"') value = value.slice(1);
-                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
+    values.forEach(function (value) {
+        // remove possible quotation
+        if (value.charAt(0) === '"') value = value.slice(1);
+        if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
 
-                let priority = null;
-                if (type === 'MX') {
-                    priority = parseInt(value.split(' ')[0], 10);
-                    value = value.split(' ')[1];
-                }
+        let priority = null;
+        if (type === 'MX') {
+            priority = parseInt(value.split(' ')[0], 10);
+            value = value.split(' ')[1];
+        }
 
-                let record = result.find(function (r) { return r.hostname === name && r.type === type; });
-                if (!record) record = { hostname: name, type: type, destination: value, deleterecord: false };
-                else record.destination = value;
+        let record = result.find(function (r) { return r.hostname === name && r.type === type; });
+        if (!record) record = { hostname: name, type: type, destination: value, deleterecord: false };
+        else record.destination = value;
 
-                if (priority !== null) record.priority = priority;
+        if (priority !== null) record.priority = priority;
 
-                records.push(record);
-            });
-
-            const data = {
-                action: 'updateDnsRecords',
-                param:{
-                    apikey: dnsConfig.apiKey,
-                    apisessionid: apiSessionId,
-                    customernumber: dnsConfig.customerNumber,
-                    domainname: zoneName,
-                    dnsrecordset: {
-                        dnsrecords: records
-                    }
-                }
-            };
-
-            debug('upserting', JSON.stringify(data));
-
-            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                debug('upsert:', result.body);
-
-                callback(null);
-            });
-        });
+        records.push(record);
     });
+
+    const data = {
+        action: 'updateDnsRecords',
+        param:{
+            apikey: domainConfig.apiKey,
+            apisessionid: apiSessionId,
+            customernumber: domainConfig.customerNumber,
+            domainname: zoneName,
+            dnsrecordset: {
+                dnsrecords: records
+            }
+        }
+    };
+
+    const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    if (response.body.statuscode !== 2000) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug('get: %s for zone %s of type %s', name, zoneName, type);
 
-    login(dnsConfig, function (error, apiSessionId) {
-        if (error) return callback(error);
+    const apiSessionId = await login(domainConfig);
 
-        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
-            if (error) return callback(error);
+    const result = await getAllRecords(domainConfig, apiSessionId, zoneName);
 
-            // We only return the value string
-            callback(null, result.filter(function (r) { return r.hostname === name && r.type === type; }).map(function (r) { return r.destination; }));
-        });
-    });
+    // We only return the value string
+    return result.filter(function (r) { return r.hostname === name && r.type === type; }).map(function (r) { return r.destination; });
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug('del: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    login(dnsConfig, function (error, apiSessionId) {
-        if (error) return callback(error);
+    const apiSessionId = await login(domainConfig);
 
-        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
-            if (error) return callback(error);
+    const result = await getAllRecords(domainConfig, apiSessionId, zoneName);
 
-            let records = [];
+    let records = [];
 
-            values.forEach(function (value) {
-                // remove possible quotation
-                if (value.charAt(0) === '"') value = value.slice(1);
-                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
+    values.forEach(function (value) {
+        // remove possible quotation
+        if (value.charAt(0) === '"') value = value.slice(1);
+        if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
 
-                let record = result.find(function (r) { return r.hostname === name && r.type === type && r.destination === value; });
-                if (!record) return;
+        let record = result.find(function (r) { return r.hostname === name && r.type === type && r.destination === value; });
+        if (!record) return;
 
-                record.deleterecord = true;
+        record.deleterecord = true;
 
-                records.push(record);
-            });
-
-            if (records.length === 0) return callback(null);
-
-            const data = {
-                action: 'updateDnsRecords',
-                param:{
-                    apikey: dnsConfig.apiKey,
-                    apisessionid: apiSessionId,
-                    customernumber: dnsConfig.customerNumber,
-                    domainname: zoneName,
-                    dnsrecordset: {
-                        dnsrecords: records
-                    }
-                }
-            };
-
-            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                debug('del:', result.body.responsedata);
-
-                callback(null);
-            });
-        });
+        records.push(record);
     });
+
+    if (records.length === 0) return;
+
+    const data = {
+        action: 'updateDnsRecords',
+        param:{
+            apikey: domainConfig.apiKey,
+            apisessionid: apiSessionId,
+            customernumber: domainConfig.customerNumber,
+            domainname: zoneName,
+            dnsrecordset: {
+                dnsrecords: records
+            }
+        }
+    };
+
+    const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    if (response.body.statuscode !== 2000) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.customerNumber || typeof dnsConfig.customerNumber !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'customerNumber must be a non-empty string', { field: 'customerNumber' }));
-    if (!dnsConfig.apiKey || typeof dnsConfig.apiKey !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string', { field: 'apiKey' }));
-    if (!dnsConfig.apiPassword || typeof dnsConfig.apiPassword !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiPassword must be a non-empty string', { field: 'apiPassword' }));
+    if (!domainConfig.customerNumber || typeof domainConfig.customerNumber !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'customerNumber must be a non-empty string');
+    if (!domainConfig.apiKey || typeof domainConfig.apiKey !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string');
+    if (!domainConfig.apiPassword || typeof domainConfig.apiPassword !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'apiPassword must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        customerNumber: dnsConfig.customerNumber,
-        apiKey: dnsConfig.apiKey,
-        apiPassword: dnsConfig.apiPassword,
+    const credentials = {
+        customerNumber: domainConfig.customerNumber,
+        apiKey: domainConfig.apiKey,
+        apiPassword: domainConfig.apiPassword,
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) {
-            debug('verifyDnsConfig: %j does not contains Netcup NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup', { field: 'nameservers' }));
-        }
+    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) {
+        debug('verifyDomainConfig: %j does not contains Netcup NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/noop.js

@@ -1,18 +1,17 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
-    debug = require('debug')('box:dns/noop'),
-    util = require('util');
+const assert = require('assert'),
+    debug = require('debug')('box:dns/noop');
 
 function removePrivateFields(domainObject) {
     return domainObject;
@@ -22,51 +21,46 @@ function removePrivateFields(domainObject) {
 function injectPrivateFields(newConfig, currentConfig) {
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
 
-    return callback(null);
+    return;
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    callback(null, [ ]); // returning ip confuses apptask into thinking the entry already exists
+    return []; // returning ip confuses apptask into thinking the entry already exists
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    return callback();
+    return;
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, location, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    callback();
+    // do nothing
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    return callback(null, { });
+    return {};
 }

src/dns/route53.js

@@ -1,23 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     AWS = require('aws-sdk'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/route53'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
-    util = require('util'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
@@ -30,272 +30,236 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.secretAccessKey === constants.SECRET_PLACEHOLDER) newConfig.secretAccessKey = currentConfig.secretAccessKey;
 }
 
-function getDnsCredentials(dnsConfig) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+function getDnsCredentials(domainConfig) {
+    assert.strictEqual(typeof domainConfig, 'object');
 
-    var credentials = {
-        accessKeyId: dnsConfig.accessKeyId,
-        secretAccessKey: dnsConfig.secretAccessKey,
-        region: dnsConfig.region
+    const credentials = {
+        accessKeyId: domainConfig.accessKeyId,
+        secretAccessKey: domainConfig.secretAccessKey,
+        region: domainConfig.region
     };
 
-    if (dnsConfig.endpoint) credentials.endpoint = new AWS.Endpoint(dnsConfig.endpoint);
+    if (domainConfig.endpoint) credentials.endpoint = new AWS.Endpoint(domainConfig.endpoint);
 
     return credentials;
 }
 
-function getZoneByName(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneByName(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
 
     // backward compat for 2.2, where we only required access to "listHostedZones"
     let listHostedZones;
-    if (dnsConfig.listHostedZonesByName) {
-        listHostedZones = route53.listHostedZonesByName.bind(route53, { MaxItems: '1', DNSName: zoneName + '.' });
+    if (domainConfig.listHostedZonesByName) {
+        listHostedZones = route53.listHostedZonesByName({ MaxItems: '1', DNSName: zoneName + '.' }).promise();
     } else {
-        listHostedZones = route53.listHostedZones.bind(route53, {}); // currently, this route does not support > 100 zones
+        listHostedZones = route53.listHostedZones({}).promise(); // currently, this route does not support > 100 zones
     }
 
-    listHostedZones(function (error, result) {
-        if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-        if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    const [error, result] = await safe(listHostedZones);
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 
-        var zone = result.HostedZones.filter(function (zone) {
-            return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
-        })[0];
+    const zone = result.HostedZones.filter(function (zone) {
+        return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
+    })[0];
 
-        if (!zone) return callback(new BoxError(BoxError.NOT_FOUND, 'no such zone'));
+    if (!zone) throw new BoxError(BoxError.NOT_FOUND, 'no such zone');
 
-        callback(null, zone);
-    });
+    return zone;
 }
 
-function getHostedZone(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getHostedZone(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    getZoneByName(dnsConfig, zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
-        route53.getHostedZone({ Id: zone.Id }, function (error, result) {
-            if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
+    const [error, result] = await safe(route53.getHostedZone({ Id: zone.Id }).promise());
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 
-            callback(null, result);
-        });
-    });
+    return result;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
-    getZoneByName(dnsConfig, zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        var records = values.map(function (v) { return { Value: v }; });  // for mx records, value is already of the '<priority> <server>' format
+    const records = values.map(function (v) { return { Value: v }; });  // for mx records, value is already of the '<priority> <server>' format
 
-        var params = {
-            ChangeBatch: {
-                Changes: [{
-                    Action: 'UPSERT',
-                    ResourceRecordSet: {
-                        Type: type,
-                        Name: fqdn,
-                        ResourceRecords: records,
-                        TTL: 1
-                    }
-                }]
-            },
-            HostedZoneId: zone.Id
-        };
+    const params = {
+        ChangeBatch: {
+            Changes: [{
+                Action: 'UPSERT',
+                ResourceRecordSet: {
+                    Type: type,
+                    Name: fqdn,
+                    ResourceRecords: records,
+                    TTL: 1
+                }
+            }]
+        },
+        HostedZoneId: zone.Id
+    };
 
-        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
-        route53.changeResourceRecordSets(params, function(error) {
-            if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'PriorRequestNotComplete') return callback(new BoxError(BoxError.BUSY, error.message));
-            if (error && error.code === 'InvalidChangeBatch') return callback(new BoxError(BoxError.BAD_FIELD, error.message));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-
-            callback(null);
-        });
-    });
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
+    const [error] = await safe(route53.changeResourceRecordSets(params).promise());
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'PriorRequestNotComplete') throw new BoxError(BoxError.BUSY, error.message);
+    if (error && error.code === 'InvalidChangeBatch') throw new BoxError(BoxError.BAD_FIELD, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(dnsConfig, zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        var params = {
-            HostedZoneId: zone.Id,
-            MaxItems: '1',
-            StartRecordName: fqdn + '.',
-            StartRecordType: type
-        };
+    const params = {
+        HostedZoneId: zone.Id,
+        MaxItems: '1',
+        StartRecordName: fqdn + '.',
+        StartRecordType: type
+    };
 
-        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
-        route53.listResourceRecordSets(params, function (error, result) {
-            if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-            if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
-            if (result.ResourceRecordSets[0].Name !== params.StartRecordName || result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
+    const [error, result] = await safe(route53.listResourceRecordSets(params).promise());
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
+    if (result.ResourceRecordSets.length === 0) return [];
+    if (result.ResourceRecordSets[0].Name !== params.StartRecordName || result.ResourceRecordSets[0].Type !== params.StartRecordType) return [];
 
-            var values = result.ResourceRecordSets[0].ResourceRecords.map(function (record) { return record.Value; });
-
-            callback(null, values);
-        });
-    });
+    const values = result.ResourceRecordSets[0].ResourceRecords.map(function (record) { return record.Value; });
+    return values;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(dnsConfig, zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        var records = values.map(function (v) { return { Value: v }; });
+    const records = values.map(function (v) { return { Value: v }; });
 
-        var resourceRecordSet = {
-            Name: fqdn,
-            Type: type,
-            ResourceRecords: records,
-            TTL: 1
-        };
+    const resourceRecordSet = {
+        Name: fqdn,
+        Type: type,
+        ResourceRecords: records,
+        TTL: 1
+    };
 
-        var params = {
-            ChangeBatch: {
-                Changes: [{
-                    Action: 'DELETE',
-                    ResourceRecordSet: resourceRecordSet
-                }]
-            },
-            HostedZoneId: zone.Id
-        };
+    const params = {
+        ChangeBatch: {
+            Changes: [{
+                Action: 'DELETE',
+                ResourceRecordSet: resourceRecordSet
+            }]
+        },
+        HostedZoneId: zone.Id
+    };
 
-        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
-        route53.changeResourceRecordSets(params, function(error) {
-            if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.message && error.message.indexOf('it was not found') !== -1) {
-                debug('del: resource record set not found.', error);
-                return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-            } else if (error && error.code === 'NoSuchHostedZone') {
-                debug('del: hosted zone not found.', error);
-                return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-            } else if (error && error.code === 'PriorRequestNotComplete') {
-                debug('del: resource is still busy', error);
-                return callback(new BoxError(BoxError.BUSY, error.message));
-            } else if (error && error.code === 'InvalidChangeBatch') {
-                debug('del: invalid change batch. No such record to be deleted.');
-                return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-            } else if (error) {
-                debug('del: error', error);
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-            }
-
-            callback(null);
-        });
-    });
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
+    const [error] = await safe(route53.changeResourceRecordSets(params).promise());
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.message && error.message.indexOf('it was not found') !== -1) {
+        throw new BoxError(BoxError.NOT_FOUND, error.message);
+    } else if (error && error.code === 'NoSuchHostedZone') {
+        throw new BoxError(BoxError.NOT_FOUND, error.message);
+    } else if (error && error.code === 'PriorRequestNotComplete') {
+        throw new BoxError(BoxError.BUSY, error.message);
+    } else if (error && error.code === 'InvalidChangeBatch') {
+        throw new BoxError(BoxError.NOT_FOUND, error.message);
+    } else if (error) {
+        throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.accessKeyId || typeof dnsConfig.accessKeyId !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'accessKeyId must be a non-empty string', { field: 'accessKeyId' }));
-    if (!dnsConfig.secretAccessKey || typeof dnsConfig.secretAccessKey !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'secretAccessKey must be a non-empty string', { field: 'secretAccessKey' }));
+    if (!domainConfig.accessKeyId || typeof domainConfig.accessKeyId !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'accessKeyId must be a non-empty string');
+    if (!domainConfig.secretAccessKey || typeof domainConfig.secretAccessKey !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'secretAccessKey must be a non-empty string');
 
-    var credentials = {
-        accessKeyId: dnsConfig.accessKeyId,
-        secretAccessKey: dnsConfig.secretAccessKey,
-        region: dnsConfig.region || 'us-east-1',
-        endpoint: dnsConfig.endpoint || null,
+    const credentials = {
+        accessKeyId: domainConfig.accessKeyId,
+        secretAccessKey: domainConfig.secretAccessKey,
+        region: domainConfig.region || 'us-east-1',
+        endpoint: domainConfig.endpoint || null,
         listHostedZonesByName: true, // new/updated creds require this perm
     };
 
     const ip = '127.0.0.1';
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        getHostedZone(credentials, zoneName, function (error, zone) {
-            if (error) return callback(error);
+    const zone = await getHostedZone(credentials, zoneName);
 
-            if (!_.isEqual(zone.DelegationSet.NameServers.sort(), nameservers.sort())) {
-                debug('verifyDnsConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers);
-                return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Route53', { field: 'nameservers' }));
-            }
+    if (!_.isEqual(zone.DelegationSet.NameServers.sort(), nameservers.sort())) {
+        debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Route53');
+    }
 
-            const location = 'cloudrontestdns';
-            const newDomainObject = Object.assign({ }, domainObject, { config: credentials });
+    const location = 'cloudrontestdns';
+    const newDomainObject = Object.assign({ }, domainObject, { config: credentials });
 
-            upsert(newDomainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
+    await upsert(newDomainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-                debug('verifyDnsConfig: Test A record added');
+    await get(newDomainObject, location, 'A');
+    debug('verifyDomainConfig: Can list record sets');
 
-                del(newDomainObject, location, 'A', [ ip ], function (error) {
-                    if (error) return callback(error);
+    await del(newDomainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-                    debug('verifyDnsConfig: Test A record removed again');
-
-                    callback(null, credentials);
-                });
-            });
-        });
-    });
+    return credentials;
 }

src/dns/vultr.js

@@ -7,16 +7,15 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
-const async = require('async'),
-    assert = require('assert'),
+const assert = require('assert'),
     constants = require('../constants.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/vultr'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
@@ -37,244 +36,202 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function getZoneRecords(dnsConfig, zoneName, name, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneRecords(domainConfig, zoneName, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
 
-    let per_page = 100, cursor= null;
+    let per_page = 100, cursor = null;
     let records = [];
 
-    async.doWhilst(function (iteratorDone) {
+    do {
         const url = `${VULTR_ENDPOINT}/domains/${zoneName}/records?per_page=${per_page}` + (cursor ? `&cursor=${cursor}` : '');
 
-        superagent.get(url)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
-            .timeout(30 * 1000)
-            .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return iteratorDone(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return iteratorDone(new BoxError(BoxError.NOT_FOUND, formatError(result)));
-                if (result.statusCode === 403 || result.statusCode === 401) return iteratorDone(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return iteratorDone(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        const [error, response] = await safe(superagent.get(url).set('Authorization', 'Bearer ' + domainConfig.token).timeout(30 * 1000).retry(5).ok(() => true));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                records = records.concat(result.body.records.filter(function (record) {
-                    return (record.type === type && record.name === name);
-                }));
+        records = records.concat(response.body.records.filter(function (record) {
+            return (record.type === type && record.name === name);
+        }));
 
-                cursor = safe.query(result.body, 'meta.links.next');
+        cursor = safe.query(response.body, 'meta.links.next');
+    } while (cursor);
 
-                iteratorDone();
-            });
-    }, function (testDone) { return testDone(null, !!cursor); }, function (error) {
-        debug('getZoneRecords:', error, JSON.stringify(records));
-
-        if (error) return callback(error);
-
-        callback(null, records);
-    });
+    return records;
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
-        if (error) return callback(error);
-
-        var tmp = records.map(function (record) { return record.data; });
-
-        debug('get: %j', tmp);
-
-        return callback(null, tmp);
-    });
+    const records = await getZoneRecords(domainConfig, zoneName, name, type);
+    const tmp = records.map(function (record) { return record.data; });
+    return tmp;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
-        if (error) return callback(error);
+    const records = await getZoneRecords(domainConfig, zoneName, name, type);
 
-        let i = 0, recordIds = []; // used to track available records to update instead of create
+    let i = 0, recordIds = []; // used to track available records to update instead of create
 
-        async.eachSeries(values, function (value, iteratorCallback) {
-            let data = {
-                type,
-                ttl: 300 // lowest
-            };
+    for (const value of values) {
+        const data = {
+            type,
+            ttl: 300 // lowest
+        };
 
-            if (type === 'MX') {
-                data.priority = parseInt(value.split(' ')[0], 10);
-                data.data = value.split(' ')[1];
-            } else if (type === 'TXT') {
-                data.data = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
-            } else {
-                data.data = value;
-            }
+        if (type === 'MX') {
+            data.priority = parseInt(value.split(' ')[0], 10);
+            data.data = value.split(' ')[1];
+        } else if (type === 'TXT') {
+            data.data = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
+        } else {
+            data.data = value;
+        }
 
-            if (i >= records.length) {
-                data.name = name; // only set for new records
+        if (i >= records.length) {
+            data.name = name; // only set for new records
 
-                superagent.post(`${VULTR_ENDPOINT}/domains/${zoneName}/records`)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode !== 201) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            const [error, response] = await safe(superagent.post(`${VULTR_ENDPOINT}/domains/${zoneName}/records`)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        recordIds.push(result.body.record.id);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        return iteratorCallback(null);
-                    });
-            } else {
-                superagent.patch(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${records[i].id}`)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        // increment, as we have consumed the record
-                        ++i;
+            recordIds.push(response.body.record.id);
+        } else {
+            const [error, response] = await safe(superagent.patch(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${records[i].id}`)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode !== 204) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            ++i;
 
-                        recordIds.push(records[i-1].id);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        return iteratorCallback(null);
-                    });
-            }
-        }, function (error) {
-            if (error) return callback(error);
+            recordIds.push(records[i-1].id);
+        }
+    }
 
-            debug('upsert: completed with recordIds:%j', recordIds);
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] = await safe(superagent.del(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${records[j].id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5));
 
-            callback();
-        });
-    });
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
+
+    debug('upsert: completed with recordIds:%j', recordIds);
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
-        if (error) return callback(error);
+    const records = await getZoneRecords(domainConfig, zoneName, name, type);
+    if (records.length === 0) return;
 
-        if (records.length === 0) return callback(null);
+    const tmp = records.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
+    if (tmp.length === 0) return;
 
-        const tmp = records.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
-
-        debug('del: %j', tmp);
-
-        if (tmp.length === 0) return callback(null);
-
-        // FIXME we only handle the first one currently
-
-        superagent.del(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${tmp[0].id}`)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
+    for (const r of tmp) {
+        const [error, response] = await safe(superagent.del(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${r.id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
             .timeout(30 * 1000)
             .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return callback(null);
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            .ok(() => true));
 
-                debug('del: done');
-
-                return callback(null);
-            });
-    });
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) continue;
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        token: dnsConfig.token
+    const credentials = {
+        token: domainConfig.token
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.vultr.com') === -1) {
-            debug('verifyDnsConfig: %j does not contains vultr NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Vultr', { field: 'nameservers' }));
-        }
+    if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.vultr.com') === -1) {
+        debug('verifyDomainConfig: %j does not contains vultr NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Vultr');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/waitfordns.js

@@ -2,109 +2,100 @@
 
 exports = module.exports = waitForDns;
 
-var assert = require('assert'),
-    async = require('async'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/waitfordns'),
-    dns = require('../native-dns.js');
+    dig = require('../dig.js'),
+    promiseRetry = require('../promise-retry.js'),
+    safe = require('safetydance');
 
-function resolveIp(hostname, options, callback) {
+async function resolveIp(hostname, type, options) {
     assert.strictEqual(typeof hostname, 'string');
+    assert(type === 'A' || type === 'AAAA');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     // try A record at authoritative server
-    debug(`resolveIp: Checking if ${hostname} has A record at ${options.server}`);
-    dns.resolve(hostname, 'A', options, function (error, results) {
-        if (!error && results.length !== 0) return callback(null, results);
+    debug(`resolveIp: Checking if ${hostname} has ${type} record at ${options.server}`);
+    const [error, results] = await safe(dig.resolve(hostname, type, options));
+    if (!error && results.length !== 0) return results;
 
-        // try CNAME record at authoritative server
-        debug(`resolveIp: Checking if ${hostname} has CNAME record at ${options.server}`);
-        dns.resolve(hostname, 'CNAME', options, function (error, results) {
-            if (error || results.length === 0) return callback(error, results);
+    // try CNAME record at authoritative server
+    debug(`resolveIp: Checking if ${hostname} has CNAME record at ${options.server}`);
+    const cnameResults = await dig.resolve(hostname, 'CNAME', options);
+    if (cnameResults.length === 0) return cnameResults;
 
-            // recurse lookup the CNAME record
-            debug(`resolveIp: Resolving ${hostname}'s CNAME record ${results[0]}`);
-            dns.resolve(results[0], 'A', { server: '127.0.0.1', timeout: options.timeout }, callback);
-        });
-    });
+    // recurse lookup the CNAME record
+    debug(`resolveIp: Resolving ${hostname}'s CNAME record ${cnameResults[0]}`);
+    return await dig.resolve(cnameResults[0], type, options);
 }
 
-function isChangeSynced(hostname, type, value, nameserver, callback) {
+async function isChangeSynced(hostname, type, value, nameserver) {
     assert.strictEqual(typeof hostname, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert.strictEqual(typeof nameserver, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // ns records cannot have cname
-    dns.resolve(nameserver, 'A', { timeout: 5000 }, function (error, nsIps) {
-        if (error || !nsIps || nsIps.length === 0) {
-            debug(`isChangeSynced: cannot resolve NS ${nameserver}`); // it's fine if one or more ns are dead
-            return callback(null, true);
+    const [error, nsIps] = await safe(dig.resolve(nameserver, 'A', { timeout: 5000 }));
+    if (error || !nsIps || nsIps.length === 0) {
+        debug(`isChangeSynced: cannot resolve NS ${nameserver}`); // it's fine if one or more ns are dead
+        return true;
+    }
+
+    const status = [];
+    for (let i = 0; i < nsIps.length; i++) {
+        const nsIp = nsIps[i];
+        const resolveOptions = { server: nsIp, timeout: 5000 };
+        const resolver = type === 'A' || type === 'AAAA' ? resolveIp(hostname, type, resolveOptions) : dig.resolve(hostname, 'TXT', resolveOptions);
+
+        const [error, answer] = await safe(resolver);
+        if (error && error.code === 'TIMEOUT') {
+            debug(`isChangeSynced: NS ${nameserver} (${nsIp}) timed out when resolving ${hostname} (${type})`);
+            status[i] = true; // should be ok if dns server is down
+            continue;
         }
 
-        async.every(nsIps, function (nsIp, iteratorCallback) {
-            const resolveOptions = { server: nsIp, timeout: 5000 };
-            const resolver = type === 'A' ? resolveIp.bind(null, hostname) : dns.resolve.bind(null, hostname, 'TXT');
+        if (error) {
+            debug(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`);
+            status[i] = false;
+            continue;
+        }
 
-            resolver(resolveOptions, function (error, answer) {
-                if (error && error.code === 'TIMEOUT') {
-                    debug(`isChangeSynced: NS ${nameserver} (${nsIp}) timed out when resolving ${hostname} (${type})`);
-                    return iteratorCallback(null, true); // should be ok if dns server is down
-                }
+        let match;
+        if (type === 'A' || type === 'AAAA') {
+            match = answer.length === 1 && answer[0] === value;
+        } else if (type === 'TXT') { // answer is a 2d array of strings
+            match = answer.some(function (a) { return value === a.join(''); });
+        }
 
-                if (error) {
-                    debug(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`);
-                    return iteratorCallback(null, false);
-                }
+        debug(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`);
+        status[i] = match;
+    }
 
-                let match;
-                if (type === 'A') {
-                    match = answer.length === 1 && answer[0] === value;
-                } else if (type === 'TXT') { // answer is a 2d array of strings
-                    match = answer.some(function (a) { return value === a.join(''); });
-                }
-
-                debug(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`);
-
-                iteratorCallback(null, match);
-            });
-        }, callback);
-
-    });
+    return status.every(s => s === true);
 }
 
 // check if IP change has propagated to every nameserver
-function waitForDns(hostname, zoneName, type, value, options, callback) {
+async function waitForDns(hostname, zoneName, type, value, options) {
     assert.strictEqual(typeof hostname, 'string');
     assert.strictEqual(typeof zoneName, 'string');
-    assert(type === 'A' || type === 'TXT');
+    assert(type === 'A' || type === 'AAAA' || type === 'TXT');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    debug('waitForDns: hostname %s to be %s in zone %s.', hostname, value, zoneName);
+    debug(`waitForDns: waiting for ${hostname} to be ${value} in zone ${zoneName}`);
 
-    var attempt = 0;
-    async.retry(options, function (retryCallback) {
-        ++attempt;
-        debug(`waitForDns (try ${attempt}): ${hostname} to be ${value} in zone ${zoneName}`);
+    await promiseRetry(Object.assign({ debug }, options), async function () {
+        const nameservers = await dig.resolve(zoneName, 'NS', { timeout: 5000 });
+        if (!nameservers) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to get nameservers');
+        debug(`waitForDns: nameservers are ${JSON.stringify(nameservers)}`);
 
-        dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-            if (error || !nameservers) return retryCallback(error || new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to get nameservers'));
-
-            async.every(nameservers, isChangeSynced.bind(null, hostname, type, value), function (error, synced) {
-                debug('waitForDns: %s %s ns: %j', hostname, synced ? 'done' : 'not done', nameservers);
-
-                retryCallback(synced ? null : new BoxError(BoxError.EXTERNAL_ERROR, 'ETRYAGAIN'));
-            });
-        });
-    }, function retryDone(error) {
-        if (error) return callback(error);
-
-        debug(`waitForDns: ${hostname} has propagated`);
-
-        callback(null);
+        for (const nameserver of nameservers) {
+            const synced = await isChangeSynced(hostname, type, value, nameserver);
+            debug(`waitForDns: ${hostname} at ns ${nameserver}: ${synced ? 'done' : 'not done'} `);
+            if (!synced) throw new BoxError(BoxError.EXTERNAL_ERROR, 'ETRYAGAIN');
+        }
     });
+
+    debug(`waitForDns: ${hostname} has propagated`);
 }

src/dns/wildcard.js

@@ -1,22 +1,22 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/manual'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
     sysinfo = require('../sysinfo.js'),
-    util = require('util'),
     waitForDns = require('./waitfordns.js');
 
 function removePrivateFields(domainObject) {
@@ -27,75 +27,75 @@ function removePrivateFields(domainObject) {
 function injectPrivateFields(newConfig, currentConfig) {
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
 
-    return callback(null);
+    return;
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    callback(null, [ ]); // returning ip confuses apptask into thinking the entry already exists
+    return []; // returning ip confuses apptask into thinking the entry already exists
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    return callback();
+    return;
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     const zoneName = domainObject.zoneName;
 
-    // Very basic check if the nameservers can be fetched
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        const location = 'cloudrontestdns';
-        const fqdn = domains.fqdn(location, domainObject);
+    const location = 'cloudrontestdns';
+    const fqdn = dns.fqdn(location, domainObject);
 
-        dns.resolve(fqdn, 'A', { server: '127.0.0.1', timeout: 5000 }, function (error, result) {
-            if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, `Unable to resolve ${fqdn}`, { field: 'nameservers' }));
-            if (error || !result) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : `Unable to resolve ${fqdn}`, { field: 'nameservers' }));
+    const [ipv4Error, ipv4Result] = await safe(dig.resolve(fqdn, 'A', { server: '127.0.0.1', timeout: 5000 }));
+    if (ipv4Error && (ipv4Error.code === 'ENOTFOUND' || ipv4Error.code === 'ENODATA')) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv4 of ${fqdn}. Please check if you have set up *.${domainObject.domain} to point to this server's IP`);
+    if (ipv4Error) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv4 of ${fqdn}: ${ipv4Error.message}`);
+    if (!ipv4Result) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv4 of ${fqdn}`);
 
-            sysinfo.getServerIp(function (error, ip) {
-                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to detect IP of this server: ${error.message}`));
+    const ipv4 = await sysinfo.getServerIPv4();
+    if (ipv4Result.length !== 1 || ipv4 !== ipv4Result[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(ipv4Result)} instead of IPv4 ${ipv4}`);
 
-                if (result.length !== 1 || ip !== result[0]) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(result)} instead of ${ip}`));
+    const ipv6 = await sysinfo.getServerIPv6(); // both should be RFC 5952 format
+    if (ipv6) {
+        const [ipv6Error, ipv6Result] = await safe(dig.resolve(fqdn, 'AAAA', { server: '127.0.0.1', timeout: 5000 }));
+        if (ipv6Error && (ipv6Error.code === 'ENOTFOUND' || ipv6Error.code === 'ENODATA')) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv6 of ${fqdn}`);
+        if (ipv6Error) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv6 of ${fqdn}: ${ipv6Error.message}`);
+        if (!ipv6Result) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv6 of ${fqdn}`);
 
-                callback(null, {});
-            });
-        });
-    });
+        if (ipv6Result.length !== 1 || ipv6 !== ipv6Result[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(ipv6Result)} instead of IPv6 ${ipv6}`);
+    }
+
+    return {};
 }

src/dockerproxy.js

@@ -5,9 +5,8 @@ exports = module.exports = {
     stop
 };
 
-var apps = require('./apps.js'),
+const apps = require('./apps.js'),
     assert = require('assert'),
-    BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     express = require('express'),
     debug = require('debug')('box:dockerproxy'),
@@ -18,27 +17,27 @@ var apps = require('./apps.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
+    util = require('util'),
     _ = require('underscore');
 
-var gHttpServer = null;
+let gHttpServer = null;
 
-function authorizeApp(req, res, next) {
+async function authorizeApp(req, res, next) {
     // make the tests pass for now
     if (constants.TEST) {
         req.app = { id: 'testappid' };
         return next();
     }
 
-    apps.getByIpAddress(req.connection.remoteAddress, function (error, app) {
-        if (error && error.reason === BoxError.NOT_FOUND) return next(new HttpError(401, 'Unauthorized'));
-        if (error) return next(new HttpError(500, error));
+    const [error, app] = await safe(apps.getByIpAddress(req.connection.remoteAddress));
+    if (error) return next(new HttpError(500, error));
+    if (!app) return next(new HttpError(401, 'Unauthorized'));
 
-        if (!('docker' in app.manifest.addons)) return next(new HttpError(401, 'Unauthorized'));
+    if (!('docker' in app.manifest.addons)) return next(new HttpError(401, 'Unauthorized'));
 
-        req.app = app;
+    req.app = app;
 
-        next();
-    });
+    next();
 }
 
 function attachDockerRequest(req, res, next) {
@@ -108,18 +107,17 @@ function process(req, res, next) {
     }
 }
 
-function start(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function start() {
     assert(gHttpServer === null, 'Already started');
 
-    let json = middleware.json({ strict: true });
+    const json = middleware.json({ strict: true });
 
     // we protect container create as the app/admin can otherwise mount random paths (like the ghost file)
     // protected other paths is done by preventing install/exec access of apps using docker addon
-    let router = new express.Router();
+    const router = new express.Router();
     router.post('/:version/containers/create', containersCreate);
 
-    let proxyServer = express();
+    const proxyServer = express();
 
     if (constants.TEST) {
         proxyServer.use(function (req, res, next) {
@@ -136,7 +134,6 @@ function start(callback) {
         .use(middleware.lastMile());
 
     gHttpServer = http.createServer(proxyServer);
-    gHttpServer.listen(constants.DOCKER_PROXY_PORT, '172.18.0.1', callback);
 
     // Overwrite the default 2min request timeout. This is required for large builds for example
     gHttpServer.setTimeout(60 * 60 * 1000);
@@ -170,14 +167,12 @@ function start(callback) {
             client.pipe(remote).pipe(client);
         });
     });
+
+    await util.promisify(gHttpServer.listen.bind(gHttpServer))(constants.DOCKER_PROXY_PORT, '172.18.0.1');
 }
 
-function stop(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function stop() {
     if (gHttpServer) gHttpServer.close();
 
     gHttpServer = null;
-
-    callback();
 }

src/domaindb.js

@@ -1,141 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-exports = module.exports = {
-    add,
-    get,
-    getAll,
-    update,
-    del,
-    clear
-};
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js'),
-    safe = require('safetydance');
-
-const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(',');
-
-function postProcess(data) {
-    data.config = safe.JSON.parse(data.configJson);
-    delete data.configJson;
-
-    data.tlsConfig = safe.JSON.parse(data.tlsConfigJson);
-    delete data.tlsConfigJson;
-
-    data.wellKnown = safe.JSON.parse(data.wellKnownJson);
-    delete data.wellKnownJson;
-
-    data.fallbackCertificate = safe.JSON.parse(data.fallbackCertificateJson);
-    delete data.fallbackCertificateJson;
-
-    return data;
-}
-
-function get(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`SELECT ${DOMAINS_FIELDS} FROM domains WHERE domain=?`, [ domain ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
-function getAll(callback) {
-    database.query(`SELECT ${DOMAINS_FIELDS} FROM domains ORDER BY domain`, function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        results.forEach(postProcess);
-
-        callback(null, results);
-    });
-}
-
-function add(name, data, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof data.zoneName, 'string');
-    assert.strictEqual(typeof data.provider, 'string');
-    assert.strictEqual(typeof data.config, 'object');
-    assert.strictEqual(typeof data.tlsConfig, 'object');
-    assert.strictEqual(typeof data.fallbackCertificate, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    let queries = [
-        { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson, fallbackCertificateJson) VALUES (?, ?, ?, ?, ?, ?)',
-            args: [ name, data.zoneName, data.provider, JSON.stringify(data.config), JSON.stringify(data.tlsConfig), JSON.stringify(data.fallbackCertificate) ] },
-        { query: 'INSERT INTO mail (domain, dkimSelector) VALUES (?, ?)', args: [ name, data.dkimSelector || 'cloudron' ] },
-    ];
-
-    database.transaction(queries, function (error) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'Domain already exists'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function update(name, domain, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof domain, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var args = [ ], fields = [ ];
-    for (var k in domain) {
-        if (k === 'config' || k === 'tlsConfig' || k === 'wellKnown' || k === 'fallbackCertificate') { // json fields
-            fields.push(`${k}Json = ?`);
-            args.push(JSON.stringify(domain[k]));
-        } else {
-            fields.push(k + ' = ?');
-            args.push(domain[k]);
-        }
-    }
-    args.push(name);
-
-    database.query('UPDATE domains SET ' + fields.join(', ') + ' WHERE domain=?', args, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function del(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    let queries = [
-        { query: 'DELETE FROM mail WHERE domain = ?', args: [ domain ] },
-        { query: 'DELETE FROM domains WHERE domain = ?', args: [ domain ] },
-    ];
-
-    database.transaction(queries, function (error, results) {
-        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') {
-            if (error.message.indexOf('apps_mailDomain_constraint') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by an app or the mailbox of an app. Check the domains of apps and the Email section of each app.'));
-            if (error.message.indexOf('subdomains') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more app(s).'));
-            if (error.message.indexOf('mail') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more mailboxes. Delete them first in the Email view.'));
-
-            return callback(new BoxError(BoxError.CONFLICT, error.message));
-        }
-
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    database.query('DELETE FROM domains', function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(error);
-    });
-}

src/domains.js

@@ -3,53 +3,45 @@
 module.exports = exports = {
     add,
     get,
-    getAll,
-    update,
+    list,
+    setConfig,
+    setWellKnown,
     del,
     clear,
 
-    fqdn,
-    getName,
-
-    getDnsRecords,
-    upsertDnsRecords,
-    removeDnsRecords,
-
-    waitForDnsRecord,
-
     removePrivateFields,
     removeRestrictedFields,
-
-    validateHostname,
-
-    makeWildcard,
-
-    parentDomain,
-
-    registerLocations,
-    unregisterLocations,
-
-    checkDnsRecords,
-    syncDnsRecords
 };
 
-const apps = require('./apps.js'),
-    assert = require('assert'),
-    async = require('async'),
+const assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    constants = require('./constants.js'),
     crypto = require('crypto'),
-    debug = require('debug')('box:domains'),
-    domaindb = require('./domaindb.js'),
+    database = require('./database.js'),
     eventlog = require('./eventlog.js'),
     mail = require('./mail.js'),
     reverseProxy = require('./reverseproxy.js'),
+    safe = require('safetydance'),
     settings = require('./settings.js'),
-    sysinfo = require('./sysinfo.js'),
     tld = require('tldjs'),
     _ = require('underscore');
 
-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
+const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(',');
+
+function postProcess(data) {
+    data.config = safe.JSON.parse(data.configJson);
+    delete data.configJson;
+
+    data.tlsConfig = safe.JSON.parse(data.tlsConfigJson);
+    delete data.tlsConfigJson;
+
+    data.wellKnown = safe.JSON.parse(data.wellKnownJson);
+    delete data.wellKnownJson;
+
+    data.fallbackCertificate = safe.JSON.parse(data.fallbackCertificateJson);
+    delete data.fallbackCertificateJson;
+
+    return data;
+}
 
 // choose which subdomain backend we use for test purpose we use route53
 function api(provider) {
@@ -74,68 +66,23 @@ function api(provider) {
     }
 }
 
-function parentDomain(domain) {
-    assert.strictEqual(typeof domain, 'string');
-    return domain.replace(/^\S+?\./, ''); // +? means non-greedy
-}
-
-function verifyDnsConfig(dnsConfig, domain, zoneName, provider, callback) {
-    assert(dnsConfig && typeof dnsConfig === 'object'); // the dns config to test with
+async function verifyDomainConfig(domainConfig, domain, zoneName, provider) {
+    assert(domainConfig && typeof domainConfig === 'object'); // the dns config to test with
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof provider, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var backend = api(provider);
-    if (!backend) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid provider', { field: 'provider' }));
+    const backend = api(provider);
+    if (!backend) throw new BoxError(BoxError.BAD_FIELD, 'Invalid provider');
 
-    const domainObject = { config: dnsConfig, domain: domain, zoneName: zoneName };
-    api(provider).verifyDnsConfig(domainObject, function (error, result) {
-        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`));
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`));
-        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`));
-        if (error) return callback(error);
+    const domainObject = { config: domainConfig, domain: domain, zoneName: zoneName };
+    const [error, result] = await safe(api(provider).verifyDomainConfig(domainObject));
+    if (error && error.reason === BoxError.ACCESS_DENIED) return { error: new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`) };
+    if (error && error.reason === BoxError.NOT_FOUND) return { error: new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`) };
+    if (error && error.reason === BoxError.EXTERNAL_ERROR) return { error: new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`) };
+    if (error) return { error };
 
-        callback(null, result);
-    });
-}
-
-function fqdn(location, domainObject) {
-    return location + (location ? '.' : '') + domainObject.domain;
-}
-
-// Hostname validation comes from RFC 1123 (section 2.1)
-// Domain name validation comes from RFC 2181 (Name syntax)
-// https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
-// We are validating the validity of the location-fqdn as host name (and not dns name)
-function validateHostname(location, domainObject) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domainObject, 'object');
-
-    const hostname = fqdn(location, domainObject);
-
-    const RESERVED_LOCATIONS = [
-        constants.SMTP_LOCATION,
-        constants.IMAP_LOCATION
-    ];
-    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
-
-    if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
-
-    // workaround https://github.com/oncletom/tld.js/issues/73
-    var tmp = hostname.replace('_', '-');
-    if (!tld.isValid(tmp)) return new BoxError(BoxError.BAD_FIELD, 'Hostname is not a valid domain name', { field: 'location' });
-
-    if (hostname.length > 253) return new BoxError(BoxError.BAD_FIELD, 'Hostname length exceeds 253 characters', { field: 'location' });
-
-    if (location) {
-        // label validation
-        if (location.split('.').some(function (p) { return p.length > 63 || p.length < 1; })) return new BoxError(BoxError.BAD_FIELD, 'Invalid subdomain length', { field: 'location' });
-        if (location.match(/^[A-Za-z0-9-.]+$/) === null) return new BoxError(BoxError.BAD_FIELD, 'Subdomain can only contain alphanumeric, hyphen and dot', { field: 'location' });
-        if (/^[-.]/.test(location)) return new BoxError(BoxError.BAD_FIELD, 'Subdomain cannot start or end with hyphen or dot', { field: 'location' });
-    }
-
-    return null;
+    return { error: null, sanitizedConfig: result };
 }
 
 function validateTlsConfig(tlsConfig, dnsProvider) {
@@ -148,12 +95,12 @@ function validateTlsConfig(tlsConfig, dnsProvider) {
     case 'fallback':
         break;
     default:
-        return new BoxError(BoxError.BAD_FIELD, 'tlsConfig.provider must be fallback, letsencrypt-prod/staging', { field: 'tlsProvider' });
+        return new BoxError(BoxError.BAD_FIELD, 'tlsConfig.provider must be fallback, letsencrypt-prod/staging');
     }
 
     if (tlsConfig.wildcard) {
-        if (!tlsConfig.provider.startsWith('letsencrypt')) return new BoxError(BoxError.BAD_FIELD, 'wildcard can only be set with letsencrypt', { field: 'wildcard' });
-        if (dnsProvider === 'manual' || dnsProvider === 'noop' || dnsProvider === 'wildcard') return new BoxError(BoxError.BAD_FIELD, 'wildcard cert requires a programmable DNS backend', { field: 'tlsProvider' });
+        if (!tlsConfig.provider.startsWith('letsencrypt')) return new BoxError(BoxError.BAD_FIELD, 'wildcard can only be set with letsencrypt');
+        if (dnsProvider === 'manual' || dnsProvider === 'noop' || dnsProvider === 'wildcard') return new BoxError(BoxError.BAD_FIELD, 'wildcard cert requires a programmable DNS backend');
     }
 
     return null;
@@ -165,37 +112,37 @@ function validateWellKnown(wellKnown) {
     return null;
 }
 
-function add(domain, data, auditSource, callback) {
+async function add(domain, data, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof data.zoneName, 'string');
     assert.strictEqual(typeof data.provider, 'string');
     assert.strictEqual(typeof data.config, 'object');
     assert.strictEqual(typeof data.fallbackCertificate, 'object');
     assert.strictEqual(typeof data.tlsConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     let { zoneName, provider, config, fallbackCertificate, tlsConfig, dkimSelector } = data;
 
-    if (!tld.isValid(domain)) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
-    if (domain.endsWith('.')) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
+    if (!tld.isValid(domain)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid domain');
+    if (domain.endsWith('.')) throw new BoxError(BoxError.BAD_FIELD, 'Invalid domain');
 
     if (zoneName) {
-        if (!tld.isValid(zoneName)) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' }));
-        if (zoneName.endsWith('.')) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' }));
+        if (!tld.isValid(zoneName)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName');
+        if (zoneName.endsWith('.')) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName');
     } else {
         zoneName = tld.getDomain(domain) || domain;
     }
 
     if (fallbackCertificate) {
         let error = reverseProxy.validateCertificate('test', { domain, config }, fallbackCertificate);
-        if (error) return callback(error);
+        if (error) throw error;
     } else {
-        fallbackCertificate = reverseProxy.generateFallbackCertificateSync(domain);
-        if (fallbackCertificate.error) return callback(fallbackCertificate.error);
+        fallbackCertificate = await reverseProxy.generateFallbackCertificate(domain);
     }
 
     let error = validateTlsConfig(tlsConfig, provider);
-    if (error) return callback(error);
+    if (error) throw error;
+
+    const dkimKey = await mail.generateDkimKey();
 
     if (!dkimSelector) {
         // create a unique suffix. this lets one add this domain can be added in another cloudron instance and not have their dkim selector conflict
@@ -203,47 +150,41 @@ function add(domain, data, auditSource, callback) {
         dkimSelector = `cloudron-${suffix}`;
     }
 
-    verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
-        if (error) return callback(error);
+    const result = await verifyDomainConfig(config, domain, zoneName, provider);
+    if (result.error) throw result.error;
 
-        domaindb.add(domain, { zoneName, provider, config: sanitizedConfig, tlsConfig, dkimSelector, fallbackCertificate }, function (error) {
-            if (error) return callback(error);
+    let queries = [
+        { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson, fallbackCertificateJson) VALUES (?, ?, ?, ?, ?, ?)',
+            args: [ domain, zoneName, provider, JSON.stringify(result.sanitizedConfig), JSON.stringify(tlsConfig), JSON.stringify(fallbackCertificate) ] },
+        { query: 'INSERT INTO mail (domain, dkimKeyJson, dkimSelector) VALUES (?, ?, ?)', args: [ domain, JSON.stringify(dkimKey), dkimSelector || 'cloudron' ] },
+    ];
 
-            reverseProxy.setFallbackCertificate(domain, fallbackCertificate, function (error) {
-                if (error) return callback(error);
+    [error] = await safe(database.transaction(queries));
+    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Domain already exists');
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 
-                eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
+    await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
 
-                mail.onDomainAdded(domain, NOOP_CALLBACK);
+    await eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
 
-                callback();
-            });
-        });
-    });
+    safe(mail.onDomainAdded(domain)); // background
 }
 
-function get(domain, callback) {
+async function get(domain) {
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    domaindb.get(domain, function (error, result) {
-        if (error) return callback(error);
-
-        return callback(null, result);
-    });
+    const result = await database.query(`SELECT ${DOMAINS_FIELDS} FROM domains WHERE domain=?`, [ domain ]);
+    if (result.length === 0) return null;
+    return postProcess(result[0]);
 }
 
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    domaindb.getAll(function (error, result) {
-        if (error) return callback(error);
-
-        return callback(null, result);
-    });
+async function list() {
+    const results = await database.query(`SELECT ${DOMAINS_FIELDS} FROM domains ORDER BY domain`);
+    results.forEach(postProcess);
+    return results;
 }
 
-function update(domain, data, auditSource, callback) {
+async function setConfig(domain, data, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof data.zoneName, 'string');
     assert.strictEqual(typeof data.provider, 'string');
@@ -251,196 +192,108 @@ function update(domain, data, auditSource, callback) {
     assert.strictEqual(typeof data.fallbackCertificate, 'object');
     assert.strictEqual(typeof data.tlsConfig, 'object');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    let { zoneName, provider, config, fallbackCertificate, tlsConfig, wellKnown } = data;
+    let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
+    let error;
 
-    if (settings.isDemo() && (domain === settings.dashboardDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+    if (settings.isDemo() && (domain === settings.dashboardDomain())) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
 
-    domaindb.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await get(domain);
+    if (zoneName) {
+        if (!tld.isValid(zoneName)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName');
+    } else {
+        zoneName = domainObject.zoneName;
+    }
 
-        if (zoneName) {
-            if (!tld.isValid(zoneName)) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' }));
+    if (fallbackCertificate) {
+        let error = reverseProxy.validateCertificate('test', domainObject, fallbackCertificate);
+        if (error) throw error;
+    }
+
+    error = validateTlsConfig(tlsConfig, provider);
+    if (error) throw error;
+
+    if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config);
+
+    const result = await verifyDomainConfig(config, domain, zoneName, provider);
+    if (result.error) throw result.error;
+
+    const newData = {
+        config: result.sanitizedConfig,
+        zoneName,
+        provider,
+        tlsConfig,
+    };
+
+    if (fallbackCertificate) newData.fallbackCertificate = fallbackCertificate;
+
+    let args = [ ], fields = [ ];
+    for (const k in newData) {
+        if (k === 'config' || k === 'tlsConfig' || k === 'fallbackCertificate') { // json fields
+            fields.push(`${k}Json = ?`);
+            args.push(JSON.stringify(newData[k]));
         } else {
-            zoneName = domainObject.zoneName;
+            fields.push(k + ' = ?');
+            args.push(newData[k]);
         }
+    }
+    args.push(domain);
 
-        if (fallbackCertificate) {
-            let error = reverseProxy.validateCertificate('test', domainObject, fallbackCertificate);
-            if (error) return callback(error);
-        }
+    [error] = await safe(database.query('UPDATE domains SET ' + fields.join(', ') + ' WHERE domain=?', args));
+    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 
-        error = validateTlsConfig(tlsConfig, provider);
-        if (error) return callback(error);
+    if (!fallbackCertificate) return;
 
-        error = validateWellKnown(wellKnown, provider);
-        if (error) return callback(error);
+    await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
 
-        if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config);
-
-        verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
-            if (error) return callback(error);
-
-            let newData = {
-                config: sanitizedConfig,
-                zoneName,
-                provider,
-                tlsConfig,
-                wellKnown,
-            };
-
-            if (fallbackCertificate) newData.fallbackCertificate = fallbackCertificate;
-
-            domaindb.update(domain, newData, function (error) {
-                if (error) return callback(error);
-
-                if (!fallbackCertificate) return callback();
-
-                reverseProxy.setFallbackCertificate(domain, fallbackCertificate, function (error) {
-                    if (error) return callback(error);
-
-                    eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, zoneName, provider });
-
-                    callback();
-                });
-            });
-        });
-    });
+    await eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, zoneName, provider });
 }
 
-function del(domain, auditSource, callback) {
+async function setWellKnown(domain, wellKnown, auditSource) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof wellKnown, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
+
+    let error = validateWellKnown(wellKnown);
+    if (error) throw error;
+
+    [error] = await safe(database.query('UPDATE domains SET wellKnownJson = ? WHERE domain=?', [ JSON.stringify(wellKnown), domain ]));
+    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
+
+    await eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, wellKnown });
+}
+
+async function del(domain, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    if (domain === settings.dashboardDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain'));
-    if (domain === settings.mailDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove mail domain. Change the mail server location first'));
+    if (domain === settings.dashboardDomain()) throw new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain');
+    if (domain === settings.mailDomain()) throw new BoxError(BoxError.CONFLICT, 'Cannot remove mail domain. Change the mail server location first');
 
-    domaindb.del(domain, function (error) {
-        if (error) return callback(error);
+    let queries = [
+        { query: 'DELETE FROM mail WHERE domain = ?', args: [ domain ] },
+        { query: 'DELETE FROM domains WHERE domain = ?', args: [ domain ] },
+    ];
 
-        eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
+    const [error, results] = await safe(database.transaction(queries));
+    if (error && error.code === 'ER_ROW_IS_REFERENCED_2') {
+        if (error.message.indexOf('apps_mailDomain_constraint') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by an app or the mailbox of an app. Check the domains of apps and the Email section of each app.');
+        if (error.message.indexOf('locations') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more app(s).');
+        if (error.message.indexOf('mail') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more mailboxes. Delete them first in the Email view.');
+        throw new BoxError(BoxError.CONFLICT, error.message);
+    }
+    if (error) throw error;
+    if (results[1].affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
 
-        mail.onDomainRemoved(domain, NOOP_CALLBACK);
+    await eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
 
-        return callback(null);
-    });
+    safe(mail.onDomainRemoved(domain));
 }
 
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    domaindb.clear(function (error) {
-        if (error) return callback(error);
-
-        return callback(null);
-    });
-}
-
-// returns the 'name' that needs to be inserted into zone
-// eslint-disable-next-line no-unused-vars
-function getName(domain, location, type) {
-    const part = domain.domain.slice(0, -domain.zoneName.length - 1);
-
-    if (location === '') return part;
-
-    return part ? `${location}.${part}` : location;
-}
-
-function getDnsRecords(location, domain, type, callback) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    get(domain, function (error, domainObject) {
-        if (error) return callback(error);
-
-        api(domainObject.provider).get(domainObject, location, type, function (error, values) {
-            if (error) return callback(error);
-
-            callback(null, values);
-        });
-    });
-}
-
-function checkDnsRecords(location, domain, callback) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    getDnsRecords(location, domain, 'A', function (error, values) {
-        if (error) return callback(error);
-
-        sysinfo.getServerIp(function (error, ip) {
-            if (error) return callback(error);
-
-            if (values.length === 0) return callback(null, { needsOverwrite: false }); // does not exist
-            if (values[0] === ip) return callback(null, { needsOverwrite: false }); // exists but in sync
-
-            callback(null, { needsOverwrite: true });
-        });
-    });
-}
-
-// note: for TXT records the values must be quoted
-function upsertDnsRecords(location, domain, type, values, callback) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('upsertDNSRecord: %s on %s type %s values', location, domain, type, values);
-
-    get(domain, function (error, domainObject) {
-        if (error) return callback(error);
-
-        api(domainObject.provider).upsert(domainObject, location, type, values, function (error) {
-            if (error) return callback(error);
-
-            callback(null);
-        });
-    });
-}
-
-function removeDnsRecords(location, domain, type, values, callback) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('removeDNSRecord: %s on %s type %s values', location, domain, type, values);
-
-    get(domain, function (error, domainObject) {
-        if (error) return callback(error);
-
-        api(domainObject.provider).del(domainObject, location, type, values, function (error) {
-            if (error && error.reason !== BoxError.NOT_FOUND) return callback(error);
-
-            callback(null);
-        });
-    });
-}
-
-function waitForDnsRecord(location, domain, type, value, options, callback) {
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert(type === 'A' || type === 'TXT');
-    assert.strictEqual(typeof value, 'string');
-    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
-
-    get(domain, function (error, domainObject) {
-        if (error) return callback(error);
-
-        // linode DNS takes ~15mins
-        if (!options.interval) options.interval = domainObject.provider === 'linode' ? 20000 : 5000;
-
-        api(domainObject.provider).wait(domainObject, location, type, value, options, callback);
-    });
+async function clear() {
+    await database.query('DELETE FROM domains');
 }
 
 // removes all fields that are strictly private and should never be returned by API calls
@@ -457,135 +310,3 @@ function removeRestrictedFields(domain) {
 
     return result;
 }
-
-function makeWildcard(vhost) {
-    assert.strictEqual(typeof vhost, 'string');
-
-    // if the vhost is like *.example.com, this function will do nothing
-    let parts = vhost.split('.');
-    parts[0] = '*';
-    return parts.join('.');
-}
-
-function registerLocations(locations, options, progressCallback, callback) {
-    assert(Array.isArray(locations));
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
-
-    const overwriteDns = options.overwriteDns || false;
-
-    sysinfo.getServerIp(function (error, ip) {
-        if (error) return callback(error);
-
-        async.eachSeries(locations, function (location, iteratorDone) {
-            async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
-                progressCallback({ message: `Registering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
-
-                // get the current record before updating it
-                getDnsRecords(location.subdomain, location.domain, 'A', function (error, values) {
-                    if (error && error.reason === BoxError.EXTERNAL_ERROR) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location })); // try again
-                    if (error && error.reason === BoxError.ACCESS_DENIED) return retryCallback(null, new BoxError(BoxError.ACCESS_DENIED, error.message, { domain: location }));
-                    if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, new BoxError(BoxError.NOT_FOUND, error.message, { domain: location }));
-                    if (error) return retryCallback(null, new BoxError(BoxError.EXTERNAL_ERROR, error.message, location)); // give up for other errors
-
-                    if (values.length !== 0 && values[0] === ip) return retryCallback(null); // up-to-date
-
-                    // refuse to update any existing DNS record for custom domains that we did not create
-                    if (values.length !== 0 && !overwriteDns) return retryCallback(null, new BoxError(BoxError.ALREADY_EXISTS, 'DNS Record already exists', { domain: location }));
-
-                    upsertDnsRecords(location.subdomain, location.domain, 'A', [ ip ], function (error) {
-                        if (error && (error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                            progressCallback({ message: `registerSubdomains: Upsert error. Will retry. ${error.message}` });
-                            return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location })); // try again
-                        }
-
-                        retryCallback(null, error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, location) : null);
-                    });
-                });
-            }, function (error, result) {
-                if (error || result) return iteratorDone(error || result);
-
-                iteratorDone(null);
-            });
-        }, callback);
-    });
-}
-
-function unregisterLocations(locations, progressCallback, callback) {
-    assert(Array.isArray(locations));
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    sysinfo.getServerIp(function (error, ip) {
-        if (error) return callback(error);
-
-        async.eachSeries(locations, function (location, iteratorDone) {
-            async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
-                progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
-
-                removeDnsRecords(location.subdomain, location.domain, 'A', [ ip ], function (error) {
-                    if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, null);
-                    if (error && (error.reason === BoxError.SBUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                        progressCallback({ message: `Error unregistering location. Will retry. ${error.message}`});
-                        return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location })); // try again
-                    }
-
-                    retryCallback(null, error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }) : null);
-                });
-            }, function (error, result) {
-                if (error || result) return iteratorDone(error || result);
-
-                iteratorDone();
-            });
-        }, callback);
-    });
-}
-
-function syncDnsRecords(options, progressCallback, callback) {
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (options.domain && options.type === 'mail') return mail.setDnsRecords(options.domain, callback);
-
-    getAll(function (error, domains) {
-        if (error) return callback(error);
-
-        if (options.domain) domains = domains.filter(d => d.domain === options.domain);
-
-        const mailSubdomain = settings.mailFqdn().substr(0, settings.mailFqdn().length - settings.mailDomain().length - 1);
-
-        apps.getAll(function (error, allApps) {
-            if (error) return callback(error);
-
-            let progress = 1, errors = [];
-
-            // we sync by domain only to get some nice progress
-            async.eachSeries(domains, function (domain, iteratorDone) {
-                progressCallback({ percent: progress, message: `Updating DNS of ${domain.domain}`});
-                progress += Math.round(100/(1+domains.length));
-
-                let locations = [];
-                if (domain.domain === settings.dashboardDomain()) locations.push({ subdomain: constants.DASHBOARD_LOCATION, domain: settings.dashboardDomain() });
-                if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.dashboardFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
-
-                allApps.forEach(function (app) {
-                    const appLocations = [{ subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains);
-                    locations = locations.concat(appLocations.filter(al => al.domain === domain.domain));
-                });
-
-                async.series([
-                    registerLocations.bind(null, locations, { overwriteDns: true }, progressCallback),
-                    progressCallback.bind(null, { message: `Updating mail DNS of ${domain.domain}`}),
-                    mail.setDnsRecords.bind(null, domain.domain)
-                ], function (error) {
-                    if (error) errors.push({ domain: domain.domain, message: error.message });
-                    iteratorDone();
-                });
-            }, () => callback(null, { errors }));
-        });
-    });
-}

src/dyndns.js

@@ -4,12 +4,11 @@ exports = module.exports = {
     sync
 };
 
-let apps = require('./apps.js'),
+const apps = require('./apps.js'),
     assert = require('assert'),
-    async = require('async'),
     constants = require('./constants.js'),
     debug = require('debug')('box:dyndns'),
-    domains = require('./domains.js'),
+    dns = require('./dns.js'),
     eventlog = require('./eventlog.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -17,46 +16,42 @@ let apps = require('./apps.js'),
     sysinfo = require('./sysinfo.js');
 
 // called for dynamic dns setups where we have to update the IP
-function sync(auditSource, callback) {
+async function sync(auditSource) {
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    sysinfo.getServerIp(function (error, ip) {
-        if (error) return callback(error);
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();
 
-        let info = safe.JSON.parse(safe.fs.readFileSync(paths.DYNDNS_INFO_FILE, 'utf8')) || { ip: null };
-        if (info.ip === ip) {
-            debug(`refreshDNS: no change in IP ${ip}`);
-            return callback();
-        }
+    const info = safe.JSON.parse(safe.fs.readFileSync(paths.DYNDNS_INFO_FILE, 'utf8')) || { ipv4: null, ipv6: null };
+    if (info.ip) { // legacy cache file
+        info.ipv4 = info.ip;
+        delete info.ip;
+    }
+    const ipv4Changed = info.ipv4 !== ipv4;
+    const ipv6Changed = ipv6 && info.ipv6 !== ipv6; // both should be RFC 5952 format
 
-        debug(`refreshDNS: updating ip from ${info.ip} to ${ip}`);
+    if (!ipv4Changed && !ipv6Changed) {
+        debug(`refreshDNS: no change in IP ipv4: ${ipv4} ipv6: ${ipv6}`);
+        return;
+    }
 
-        domains.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    debug(`refreshDNS: updating IP from ${info.ipv4} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`);
+    if (ipv4Changed) await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'A', [ ipv4 ]);
+    if (ipv6Changed) await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'AAAA', [ ipv6 ]);
 
-            debug('refreshDNS: updated admin location');
+    const result = await apps.list();
+    for (const app of result) {
+        // do not change state of installing apps since apptask will error if dns record already exists
+        if (app.installationState !== apps.ISTATE_INSTALLED) continue;
 
-            apps.getAll(function (error, result) {
-                if (error) return callback(error);
+        if (ipv4Changed) await dns.upsertDnsRecords(app.subdomain, app.domain, 'A', [ ipv4 ]);
+        if (ipv6Changed) await dns.upsertDnsRecords(app.subdomain, app.domain, 'AAAA', [ ipv6 ]);
+    }
 
-                async.each(result, function (app, callback) {
-                    // do not change state of installing apps since apptask will error if dns record already exists
-                    if (app.installationState !== apps.ISTATE_INSTALLED) return callback();
+    debug('refreshDNS: updated apps');
 
-                    domains.upsertDnsRecords(app.location, app.domain, 'A', [ ip ], callback);
-                }, function (error) {
-                    if (error) return callback(error);
-
-                    debug('refreshDNS: updated apps');
-
-                    eventlog.add(eventlog.ACTION_DYNDNS_UPDATE, auditSource, { fromIp: info.ip, toIp: ip });
-                    info.ip = ip;
-                    safe.fs.writeFileSync(paths.DYNDNS_INFO_FILE, JSON.stringify(info), 'utf8');
-
-                    callback();
-                });
-            });
-        });
-    });
+    await eventlog.add(eventlog.ACTION_DYNDNS_UPDATE, auditSource, { fromIpv4: info.ipv4, fromIpv6: info.ipv6, toIpv4: ipv4, toIpv6: ipv6 });
+    info.ipv4 = ipv4;
+    info.ipv6 = ipv6;
+    safe.fs.writeFileSync(paths.DYNDNS_INFO_FILE, JSON.stringify(info), 'utf8');
 }

src/eventlog.js

@@ -4,7 +4,7 @@ exports = module.exports = {
     add,
     upsertLoginEvent,
     get,
-    getAllPaged,
+    listPaged,
     cleanup,
     _clear: clear,
 
@@ -19,6 +19,8 @@ exports = module.exports = {
     ACTION_APP_UNINSTALL: 'app.uninstall',
     ACTION_APP_UPDATE: 'app.update',
     ACTION_APP_UPDATE_FINISH: 'app.update.finish',
+    ACTION_APP_BACKUP: 'app.backup',
+    ACTION_APP_BACKUP_FINISH: 'app.backup.finish',
     ACTION_APP_LOGIN: 'app.login',
     ACTION_APP_OOM: 'app.oom',
     ACTION_APP_UP: 'app.up',
@@ -34,6 +36,7 @@ exports = module.exports = {
 
     ACTION_CERTIFICATE_NEW: 'certificate.new',
     ACTION_CERTIFICATE_RENEWAL: 'certificate.renew',
+    ACTION_CERTIFICATE_CLEANUP: 'certificate.cleanup',
 
     ACTION_DASHBOARD_DOMAIN_UPDATE: 'dashboard.domain.update',
 
@@ -54,6 +57,11 @@ exports = module.exports = {
     ACTION_PROVISION: 'cloudron.provision',
     ACTION_RESTORE: 'cloudron.restore', // unused
     ACTION_START: 'cloudron.start',
+
+    ACTION_SERVICE_CONFIGURE: 'service.configure',
+    ACTION_SERVICE_REBUILD: 'service.rebuild',
+    ACTION_SERVICE_RESTART: 'service.restart',
+
     ACTION_UPDATE: 'cloudron.update',
     ACTION_UPDATE_FINISH: 'cloudron.update.finish',
 
@@ -66,6 +74,7 @@ exports = module.exports = {
 
     ACTION_VOLUME_ADD: 'volume.add',
     ACTION_VOLUME_UPDATE: 'volume.update',
+    ACTION_VOLUME_REMOUNT: 'volume.remount',
     ACTION_VOLUME_REMOVE: 'volume.remove',
 
     ACTION_DYNDNS_UPDATE: 'dyndns.update',
@@ -78,18 +87,19 @@ exports = module.exports = {
 
 const assert = require('assert'),
     database = require('./database.js'),
-    debug = require('debug')('box:eventlog'),
     mysql = require('mysql'),
     notifications = require('./notifications.js'),
     safe = require('safetydance'),
     uuid = require('uuid');
 
-const EVENTLOG_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
+const EVENTLOG_FIELDS = [ 'id', 'action', 'sourceJson', 'dataJson', 'creationTime' ].join(',');
 
 function postProcess(record) {
     // usually we have sourceJson and dataJson, however since this used to be the JSON data type, we don't
-    record.source = safe.JSON.parse(record.source);
-    record.data = safe.JSON.parse(record.data);
+    record.source = safe.JSON.parse(record.sourceJson);
+    delete record.sourceJson;
+    record.data = safe.JSON.parse(record.dataJson);
+    delete record.dataJson;
 
     return record;
 }
@@ -101,14 +111,9 @@ async function add(action, source, data) {
     assert.strictEqual(typeof data, 'object');
 
     const id = uuid.v4();
-    try {
-        await database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ]);
-        await notifications.onEvent(id, action, source, data);
-        return id;
-    } catch (error) {
-        debug('add: error adding event', error);
-        return null;
-    }
+    await database.query('INSERT INTO eventlog (id, action, sourceJson, dataJson) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ]);
+    await notifications.onEvent(id, action, source, data);
+    return id;
 }
 
 // never throws, only logs because previously code did not take a callback
@@ -119,23 +124,18 @@ async function upsertLoginEvent(action, source, data) {
 
     // can't do a real sql upsert, for frequent eventlog entries we only have to do 2 queries once a day
     const queries = [{
-        query: 'UPDATE eventlog SET creationTime=NOW(), data=? WHERE action = ? AND source LIKE ? AND DATE(creationTime)=CURDATE()',
+        query: 'UPDATE eventlog SET creationTime=NOW(), dataJson=? WHERE action = ? AND sourceJson LIKE ? AND DATE(creationTime)=CURDATE()',
         args: [ JSON.stringify(data), action, JSON.stringify(source) ]
     }, {
-        query: 'SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog WHERE action = ? AND source LIKE ? AND DATE(creationTime)=CURDATE()',
+        query: 'SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog WHERE action = ? AND sourceJson LIKE ? AND DATE(creationTime)=CURDATE()',
         args: [ action, JSON.stringify(source) ]
     }];
 
-    try {
-        const result = await database.transaction(queries);
-        if (result[0].affectedRows >= 1) return result[1][0].id;
+    const result = await database.transaction(queries);
+    if (result[0].affectedRows >= 1) return result[1][0].id;
 
-        // no existing eventlog found, create one
-        return await add(action, source, data);
-    } catch (error) {
-        debug('add: error adding event', error);
-        return null;
-    }
+    // no existing eventlog found, create one
+    return await add(action, source, data);
 }
 
 async function get(id) {
@@ -147,7 +147,7 @@ async function get(id) {
     return postProcess(result[0]);
 }
 
-async function getAllPaged(actions, search, page, perPage) {
+async function listPaged(actions, search, page, perPage) {
     assert(Array.isArray(actions));
     assert(typeof search === 'string' || search === null);
     assert.strictEqual(typeof page, 'number');
@@ -157,7 +157,7 @@ async function getAllPaged(actions, search, page, perPage) {
     let query = `SELECT ${EVENTLOG_FIELDS} FROM eventlog`;
 
     if (actions.length || search) query += ' WHERE';
-    if (search) query += ' (source LIKE ' + mysql.escape('%' + search + '%') + ' OR data LIKE ' + mysql.escape('%' + search + '%') + ')';
+    if (search) query += ' (sourceJson LIKE ' + mysql.escape('%' + search + '%') + ' OR dataJson LIKE ' + mysql.escape('%' + search + '%') + ')';
 
     if (actions.length && search) query += ' AND ( ';
     actions.forEach(function (action, i) {

src/externalldap.js

@@ -1,9 +1,8 @@
 'use strict';
 
 exports = module.exports = {
-    search,
     verifyPassword,
-    createAndVerifyUserIfNotExist,
+    maybeCreateUser,
 
     testConfig,
     startSyncer,
@@ -14,18 +13,19 @@ exports = module.exports = {
     sync
 };
 
-var assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
+const assert = require('assert'),
+    AuditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:externalldap'),
     groups = require('./groups.js'),
     ldap = require('ldapjs'),
     once = require('once'),
+    safe = require('safetydance'),
     settings = require('./settings.js'),
     tasks = require('./tasks.js'),
-    users = require('./users.js');
+    users = require('./users.js'),
+    util = require('util');
 
 function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.bindPassword === constants.SECRET_PLACEHOLDER) newConfig.bindPassword = currentConfig.bindPassword;
@@ -40,10 +40,11 @@ function removePrivateFields(ldapConfig) {
 function translateUser(ldapConfig, ldapUser) {
     assert.strictEqual(typeof ldapConfig, 'object');
 
+    // RFC: https://datatracker.ietf.org/doc/html/rfc2798
     return {
         username: ldapUser[ldapConfig.usernameField],
         email: ldapUser.mail || ldapUser.mailPrimaryAddress,
-        displayName: ldapUser.cn // user.giveName + ' ' + user.sn
+        displayName: ldapUser.displayName || ldapUser.cn // user.giveName + ' ' + user.sn
     };
 }
 
@@ -57,532 +58,410 @@ function validUserRequirements(user) {
 }
 
 // performs service bind if required
-function getClient(externalLdapConfig, doBindAuth, callback) {
+async function getClient(externalLdapConfig, options) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof doBindAuth, 'boolean');
-    assert.strictEqual(typeof callback, 'function');
-
-    // ensure we only callback once since we also have to listen to client.error events
-    callback = once(callback);
+    assert.strictEqual(typeof options, 'object');
 
     // basic validation to not crash
-    try { ldap.parseDN(externalLdapConfig.baseDn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid baseDn')); }
-    try { ldap.parseFilter(externalLdapConfig.filter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid filter')); }
+    try { ldap.parseDN(externalLdapConfig.baseDn); } catch (e) { throw new BoxError(BoxError.BAD_FIELD, 'invalid baseDn'); }
+    try { ldap.parseFilter(externalLdapConfig.filter); } catch (e) { throw new BoxError(BoxError.BAD_FIELD, 'invalid filter'); }
 
-    var config = {
+    const config = {
         url: externalLdapConfig.url,
         tlsOptions: {
             rejectUnauthorized: externalLdapConfig.acceptSelfSignedCerts ? false : true
         }
     };
 
-    var client;
+    let client;
     try {
         client = ldap.createClient(config);
     } catch (e) {
-        if (e instanceof ldap.ProtocolError) return callback(new BoxError(BoxError.BAD_FIELD, 'url protocol is invalid'));
-        return callback(new BoxError(BoxError.INTERNAL_ERROR, e));
+        if (e instanceof ldap.ProtocolError) throw new BoxError(BoxError.BAD_FIELD, 'url protocol is invalid');
+        throw new BoxError(BoxError.INTERNAL_ERROR, e);
     }
 
-    // ensure we don't just crash
-    client.on('error', function (error) {
-        callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-    });
-
     // skip bind auth if none exist or if not wanted
-    if (!externalLdapConfig.bindDn || !doBindAuth) return callback(null, client);
+    if (!externalLdapConfig.bindDn || !options.bind) return client;
 
-    client.bind(externalLdapConfig.bindDn, externalLdapConfig.bindPassword, function (error) {
-        if (error instanceof ldap.InvalidCredentialsError) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+    return await new Promise((resolve, reject) => {
+        reject = once(reject);
 
-        callback(null, client);
+        // ensure we don't just crash
+        client.on('error', function (error) {
+            reject(new BoxError(BoxError.EXTERNAL_ERROR, error));
+        });
+
+        client.bind(externalLdapConfig.bindDn, externalLdapConfig.bindPassword, function (error) {
+            if (error instanceof ldap.InvalidCredentialsError) return reject(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (error) return reject(new BoxError(BoxError.EXTERNAL_ERROR, error));
+
+            resolve(client);
+        });
     });
 }
 
-function ldapGetByDN(externalLdapConfig, dn, callback) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
+async function clientSearch(client, dn, searchOptions) {
+    assert.strictEqual(typeof client, 'object');
     assert.strictEqual(typeof dn, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert.strictEqual(typeof searchOptions, 'object');
 
-    getClient(externalLdapConfig, true, function (error, client) {
-        if (error) return callback(error);
-
-        let searchOptions = {
-            paged: true,
-            scope: 'sub'    // We may have to make this configurable
-        };
-
-        debug(`Get object at ${dn}`);
-
-        // basic validation to not crash
-        try { ldap.parseDN(dn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid DN')); }
+    // basic validation to not crash
+    try { ldap.parseDN(dn); } catch (e) { throw new BoxError(BoxError.BAD_FIELD, 'invalid DN'); }
 
+    return await new Promise((resolve, reject) => {
         client.search(dn, searchOptions, function (error, result) {
-            if (error instanceof ldap.NoSuchObjectError) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+            if (error instanceof ldap.NoSuchObjectError) return reject(new BoxError(BoxError.NOT_FOUND));
+            if (error) return reject(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
             let ldapObjects = [];
 
             result.on('searchEntry', entry => ldapObjects.push(entry.object));
-            result.on('error', error => callback(new BoxError(BoxError.EXTERNAL_ERROR, error)));
+            result.on('error', error => reject(new BoxError(BoxError.EXTERNAL_ERROR, error)));
 
             result.on('end', function (result) {
-                client.unbind();
+                if (result.status !== 0) return reject(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
 
-                if (result.status !== 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
-                if (ldapObjects.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
-
-                callback(null, ldapObjects[0]);
+                resolve(ldapObjects);
             });
         });
     });
 }
 
+async function ldapGetByDN(externalLdapConfig, dn) {
+    assert.strictEqual(typeof externalLdapConfig, 'object');
+    assert.strictEqual(typeof dn, 'string');
+
+    const searchOptions = {
+        paged: true,
+        scope: 'sub'    // We may have to make this configurable
+    };
+
+    debug(`ldapGetByDN: Get object at ${dn}`);
+
+    const client = await getClient(externalLdapConfig, { bind: true });
+    const result = await clientSearch(client, dn, searchOptions);
+    client.unbind();
+    if (result.length === 0) throw new BoxError(BoxError.NOT_FOUND);
+    return result[0];
+}
+
 // TODO support search by email
-function ldapUserSearch(externalLdapConfig, options, callback) {
+async function ldapUserSearch(externalLdapConfig, options) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    getClient(externalLdapConfig, true, function (error, client) {
-        if (error) return callback(error);
+    const searchOptions = {
+        paged: true,
+        filter: ldap.parseFilter(externalLdapConfig.filter),
+        scope: 'sub'    // We may have to make this configurable
+    };
 
-        let searchOptions = {
-            paged: true,
-            filter: ldap.parseFilter(externalLdapConfig.filter),
-            scope: 'sub'    // We may have to make this configurable
-        };
+    if (options.filter) { // https://github.com/ldapjs/node-ldapjs/blob/master/docs/filters.md
+        const extraFilter = ldap.parseFilter(options.filter);
+        searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
+    }
 
-        if (options.filter) { // https://github.com/ldapjs/node-ldapjs/blob/master/docs/filters.md
-            let extraFilter = ldap.parseFilter(options.filter);
-            searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
-        }
-
-        debug(`Listing users at ${externalLdapConfig.baseDn} with filter ${searchOptions.filter.toString()}`);
-
-        client.search(externalLdapConfig.baseDn, searchOptions, function (error, result) {
-            if (error instanceof ldap.NoSuchObjectError) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-            let ldapUsers = [];
-
-            result.on('searchEntry', entry => ldapUsers.push(entry.object));
-            result.on('error', error => callback(new BoxError(BoxError.EXTERNAL_ERROR, error)));
-
-            result.on('end', function (result) {
-                client.unbind();
-
-                if (result.status !== 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
-
-                callback(null, ldapUsers);
-            });
-        });
-    });
+    const client = await getClient(externalLdapConfig, { bind: true });
+    const result = await clientSearch(client, externalLdapConfig.baseDn, searchOptions);
+    client.unbind();
+    return result;
 }
 
-function ldapGroupSearch(externalLdapConfig, options, callback) {
+async function ldapGroupSearch(externalLdapConfig, options) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    getClient(externalLdapConfig, true, function (error, client) {
-        if (error) return callback(error);
+    const searchOptions = {
+        paged: true,
+        scope: 'sub'    // We may have to make this configurable
+    };
 
-        let searchOptions = {
-            paged: true,
-            scope: 'sub'    // We may have to make this configurable
-        };
+    if (externalLdapConfig.groupFilter) searchOptions.filter = ldap.parseFilter(externalLdapConfig.groupFilter);
 
-        if (externalLdapConfig.groupFilter) searchOptions.filter = ldap.parseFilter(externalLdapConfig.groupFilter);
+    if (options.filter) { // https://github.com/ldapjs/node-ldapjs/blob/master/docs/filters.md
+        const extraFilter = ldap.parseFilter(options.filter);
+        searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
+    }
 
-        if (options.filter) { // https://github.com/ldapjs/node-ldapjs/blob/master/docs/filters.md
-            let extraFilter = ldap.parseFilter(options.filter);
-            searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
-        }
-
-        debug(`Listing groups at ${externalLdapConfig.groupBaseDn} with filter ${searchOptions.filter.toString()}`);
-
-        client.search(externalLdapConfig.groupBaseDn, searchOptions, function (error, result) {
-            if (error instanceof ldap.NoSuchObjectError) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-            let ldapGroups = [];
-
-            result.on('searchEntry', entry => ldapGroups.push(entry.object));
-            result.on('error', error => callback(new BoxError(BoxError.EXTERNAL_ERROR, error)));
-
-            result.on('end', function (result) {
-                client.unbind();
-
-                if (result.status !== 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
-
-                callback(null, ldapGroups);
-            });
-        });
-    });
+    const client = await getClient(externalLdapConfig, { bind: true });
+    const result = await clientSearch(client, externalLdapConfig.groupBaseDn, searchOptions);
+    client.unbind();
+    return result;
 }
 
-function testConfig(config, callback) {
+async function testConfig(config) {
     assert.strictEqual(typeof config, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    if (config.provider === 'noop') return callback();
+    if (config.provider === 'noop') return null;
 
-    if (!config.url) return callback(new BoxError(BoxError.BAD_FIELD, 'url must not be empty'));
-    if (!config.url.startsWith('ldap://') && !config.url.startsWith('ldaps://')) return callback(new BoxError(BoxError.BAD_FIELD, 'url is missing ldap:// or ldaps:// prefix'));
+    if (!config.url) return new BoxError(BoxError.BAD_FIELD, 'url must not be empty');
+    if (!config.url.startsWith('ldap://') && !config.url.startsWith('ldaps://')) return new BoxError(BoxError.BAD_FIELD, 'url is missing ldap:// or ldaps:// prefix');
     if (!config.usernameField) config.usernameField = 'uid';
 
     // bindDn may not be a dn!
-    if (!config.baseDn) return callback(new BoxError(BoxError.BAD_FIELD, 'basedn must not be empty'));
-    try { ldap.parseDN(config.baseDn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid baseDn')); }
+    if (!config.baseDn) return new BoxError(BoxError.BAD_FIELD, 'basedn must not be empty');
+    try { ldap.parseDN(config.baseDn); } catch (e) { return new BoxError(BoxError.BAD_FIELD, 'invalid baseDn'); }
 
-    if (!config.filter) return callback(new BoxError(BoxError.BAD_FIELD, 'filter must not be empty'));
-    try { ldap.parseFilter(config.filter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid filter')); }
+    if (!config.filter) return new BoxError(BoxError.BAD_FIELD, 'filter must not be empty');
+    try { ldap.parseFilter(config.filter); } catch (e) { return new BoxError(BoxError.BAD_FIELD, 'invalid filter'); }
 
-    if ('syncGroups' in config && typeof config.syncGroups !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'syncGroups must be a boolean'));
-    if ('acceptSelfSignedCerts' in config && typeof config.acceptSelfSignedCerts !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'acceptSelfSignedCerts must be a boolean'));
+    if ('syncGroups' in config && typeof config.syncGroups !== 'boolean') return new BoxError(BoxError.BAD_FIELD, 'syncGroups must be a boolean');
+    if ('acceptSelfSignedCerts' in config && typeof config.acceptSelfSignedCerts !== 'boolean') return new BoxError(BoxError.BAD_FIELD, 'acceptSelfSignedCerts must be a boolean');
 
     if (config.syncGroups) {
-        if (!config.groupBaseDn) return callback(new BoxError(BoxError.BAD_FIELD, 'groupBaseDn must not be empty'));
-        try { ldap.parseDN(config.groupBaseDn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid groupBaseDn')); }
+        if (!config.groupBaseDn) return new BoxError(BoxError.BAD_FIELD, 'groupBaseDn must not be empty');
+        try { ldap.parseDN(config.groupBaseDn); } catch (e) { return new BoxError(BoxError.BAD_FIELD, 'invalid groupBaseDn'); }
 
-        if (!config.groupFilter) return callback(new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty'));
-        try { ldap.parseFilter(config.groupFilter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid groupFilter')); }
+        if (!config.groupFilter) return new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty');
+        try { ldap.parseFilter(config.groupFilter); } catch (e) { return new BoxError(BoxError.BAD_FIELD, 'invalid groupFilter'); }
 
-        if (!config.groupnameField || typeof config.groupnameField !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty'));
+        if (!config.groupnameField || typeof config.groupnameField !== 'string') return new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty');
     }
 
-    getClient(config, true, function (error, client) {
-        if (error) return callback(error);
+    const [error, client] = await safe(getClient(config, { bind: true }));
+    if (error) return error;
 
-        var opts = {
-            filter: config.filter,
-            scope: 'sub'
-        };
+    const opts = {
+        filter: config.filter,
+        scope: 'sub'
+    };
 
-        client.search(config.baseDn, opts, function (error, result) {
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+    const [searchError, ] = await safe(clientSearch(client, config.baseDn, opts));
+    client.unbind();
+    if (searchError) return searchError;
 
-            result.on('searchEntry', function (/* entry */) {});
-            result.on('error', function (error) { client.unbind(); callback(new BoxError(BoxError.BAD_FIELD, `Unable to search directory: ${error.message}`)); });
-            result.on('end', function (/* result */) { client.unbind(); callback(); });
-        });
-    });
+    return null;
 }
 
-function search(identifier, callback) {
+async function maybeCreateUser(identifier) {
     assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
-        if (error) return callback(error);
-        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+    const externalLdapConfig = await settings.getExternalLdapConfig();
+    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
+    if (!externalLdapConfig.autoCreate) throw new BoxError(BoxError.BAD_STATE, 'auto create not enabled');
 
-        ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
-            if (error) return callback(error);
+    const ldapUsers = await ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` });
+    if (ldapUsers.length === 0) throw new BoxError(BoxError.NOT_FOUND);
+    if (ldapUsers.length > 1) throw new BoxError(BoxError.CONFLICT);
 
-            // translate ldap properties to ours
-            let users = ldapUsers.map(function (u) { return translateUser(externalLdapConfig, u); });
+    const user = translateUser(externalLdapConfig, ldapUsers[0]);
+    if (!validUserRequirements(user)) throw new BoxError(BoxError.BAD_FIELD);
 
-            callback(null, users);
-        });
-    });
+    const [error, userId] = await safe(users.add(user.email, { username: user.username, password: null, displayName: user.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP_AUTO_CREATE));
+    if (error) {
+        debug(`maybeCreateUser: failed to auto create user ${user.username}`, error);
+        throw error;
+    }
+
+    // fetch the full record
+    return await users.get(userId);
 }
 
-function createAndVerifyUserIfNotExist(identifier, password, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof password, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
-        if (error) return callback(error);
-        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
-        if (!externalLdapConfig.autoCreate) return callback(new BoxError(BoxError.BAD_STATE, 'auto create not enabled'));
-
-        ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
-            if (error) return callback(error);
-            if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
-
-            let user = translateUser(externalLdapConfig, ldapUsers[0]);
-            if (!validUserRequirements(user)) return callback(new BoxError(BoxError.BAD_FIELD));
-
-            users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_AUTO_CREATE, function (error, user) {
-                if (error) {
-                    debug(`createAndVerifyUserIfNotExist: Failed to auto create user ${user.username}`, error);
-                    return callback(new BoxError(BoxError.INTERNAL_ERROR));
-                }
-
-                verifyPassword(user, password, function (error) {
-                    if (error) return callback(error);
-                    callback(null, user);
-                });
-            });
-        });
-    });
-}
-
-function verifyPassword(user, password, callback) {
+async function verifyPassword(user, password) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof password, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
-        if (error) return callback(error);
-        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+    const externalLdapConfig = await settings.getExternalLdapConfig();
+    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
 
-        ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${user.username}` }, function (error, ldapUsers) {
-            if (error) return callback(error);
-            if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
+    const ldapUsers = await ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${user.username}` });
+    if (ldapUsers.length === 0) throw new BoxError(BoxError.NOT_FOUND);
+    if (ldapUsers.length > 1) throw new BoxError(BoxError.CONFLICT);
 
-            getClient(externalLdapConfig, false, function (error, client) {
-                if (error) return callback(error);
+    const client = await getClient(externalLdapConfig, { bind: false });
 
-                client.bind(ldapUsers[0].dn, password, function (error) {
-                    if (error instanceof ldap.InvalidCredentialsError) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                    if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+    const [error] = await safe(util.promisify(client.bind.bind(client))(ldapUsers[0].dn, password));
+    client.unbind();
+    if (error instanceof ldap.InvalidCredentialsError) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error);
 
-                    callback(null, translateUser(externalLdapConfig, ldapUsers[0]));
-                });
-            });
-        });
-    });
+    return translateUser(externalLdapConfig, ldapUsers[0]);
 }
 
-function startSyncer(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function startSyncer() {
+    const externalLdapConfig = await settings.getExternalLdapConfig();
+    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
 
-    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
-        if (error) return callback(error);
-        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+    const taskId = await tasks.add(tasks.TASK_SYNC_EXTERNAL_LDAP, []);
 
-        tasks.add(tasks.TASK_SYNC_EXTERNAL_LDAP, [], function (error, taskId) {
-            if (error) return callback(error);
-
-            tasks.startTask(taskId, {}, function (error, result) {
-                debug('sync: done', error, result);
-            });
-
-            callback(null, taskId);
-        });
+    tasks.startTask(taskId, {}, function (error, result) {
+        debug('sync: done', error, result);
     });
+
+    return taskId;
 }
 
-function syncUsers(externalLdapConfig, progressCallback, callback) {
+async function syncUsers(externalLdapConfig, progressCallback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
 
-    ldapUserSearch(externalLdapConfig, {}, function (error, ldapUsers) {
-        if (error) return callback(error);
+    const ldapUsers = await ldapUserSearch(externalLdapConfig, {});
 
-        debug(`Found ${ldapUsers.length} users`);
+    debug(`syncUsers: Found ${ldapUsers.length} users`);
 
-        let percent = 10;
-        let step = 30/(ldapUsers.length+1); // ensure no divide by 0
+    let percent = 10;
+    let step = 30/(ldapUsers.length+1); // ensure no divide by 0
 
-        // we ignore all errors here and just log them for now
-        async.eachSeries(ldapUsers, function (user, iteratorCallback) {
-            user = translateUser(externalLdapConfig, user);
+    // we ignore all errors here and just log them for now
+    for (let i = 0; i < ldapUsers.length; i++) {
+        let ldapUser = translateUser(externalLdapConfig, ldapUsers[i]);
+        if (!validUserRequirements(ldapUser)) continue;
 
-            if (!validUserRequirements(user)) return iteratorCallback();
+        percent += step;
+        progressCallback({ percent, message: `Syncing... ${ldapUser.username}` });
 
-            percent += step;
-            progressCallback({ percent, message: `Syncing... ${user.username}` });
+        const user = await users.getByUsername(ldapUser.username);
 
-            users.getByUsername(user.username, function (error, result) {
-                if (error && error.reason !== BoxError.NOT_FOUND) return iteratorCallback(error);
+        if (!user) {
+            debug(`syncUsers: [adding user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
 
-                if (!result) {
-                    debug(`[adding user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+            const [userAddError] = await safe(users.add(ldapUser.email, { username: ldapUser.username, password: null, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP_TASK));
+            if (userAddError) debug('syncUsers: Failed to create user', ldapUser, userAddError);
+        } else if (user.source !== 'ldap') {
+            debug(`syncUsers: [mapping user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
 
-                    users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
-                        if (error) debug('syncUsers: Failed to create user', user, error.message);
-                        iteratorCallback();
-                    });
-                } else if (result.source !== 'ldap') {
-                    debug(`[conflicting user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+            const [userMappingError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName, source: 'ldap' }, AuditSource.EXTERNAL_LDAP_TASK));
+            if (userMappingError) debug('Failed to map user', ldapUser, userMappingError);
+        } else if (user.email !== ldapUser.email || user.displayName !== ldapUser.displayName) {
+            debug(`syncUsers: [updating user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
 
-                    iteratorCallback();
-                } else if (result.email !== user.email || result.displayName !== user.displayName) {
-                    debug(`[updating user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+            const [userUpdateError] = await safe(users.update(user, { email: ldapUser.email, fallbackEmail: ldapUser.email, displayName: ldapUser.displayName }, AuditSource.EXTERNAL_LDAP_TASK));
+            if (userUpdateError) debug('Failed to update user', ldapUser, userUpdateError);
+        } else {
+            // user known and up-to-date
+            debug(`syncUsers: [up-to-date user] username=${ldapUser.username} email=${ldapUser.email} displayName=${ldapUser.displayName}`);
+        }
+    }
 
-                    users.update(result, { email: user.email, fallbackEmail: user.email, displayName: user.displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
-                        if (error) debug('Failed to update user', user, error);
-
-                        iteratorCallback();
-                    });
-                } else {
-                    // user known and up-to-date
-                    debug(`[up-to-date user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
-
-                    iteratorCallback();
-                }
-            });
-        }, callback);
-    });
+    debug('syncUsers: done');
 }
 
-function syncGroups(externalLdapConfig, progressCallback, callback) {
+async function syncGroups(externalLdapConfig, progressCallback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
 
     if (!externalLdapConfig.syncGroups) {
-        debug('Group sync is disabled');
+        debug('syncGroups: Group sync is disabled');
         progressCallback({ percent: 70, message: 'Skipping group sync...' });
-        return callback(null, []);
+        return [];
     }
 
-    ldapGroupSearch(externalLdapConfig, {}, function (error, ldapGroups) {
-        if (error) return callback(error);
+    const ldapGroups = await ldapGroupSearch(externalLdapConfig, {});
 
-        debug(`Found ${ldapGroups.length} groups`);
+    debug(`syncGroups: Found ${ldapGroups.length} groups`);
 
-        let percent = 40;
-        let step = 30/(ldapGroups.length+1); // ensure no divide by 0
+    let percent = 40;
+    let step = 30/(ldapGroups.length+1); // ensure no divide by 0
 
-        // we ignore all non internal errors here and just log them for now
-        async.eachSeries(ldapGroups, function (ldapGroup, iteratorCallback) {
-            var groupName = ldapGroup[externalLdapConfig.groupnameField];
-            if (!groupName) return iteratorCallback();
-            // some servers return empty array for unknown properties :-/
-            if (typeof groupName !== 'string') return iteratorCallback();
+    // we ignore all non internal errors here and just log them for now
+    for (const ldapGroup of ldapGroups) {
+        let groupName = ldapGroup[externalLdapConfig.groupnameField];
+        if (!groupName) return;
+        // some servers return empty array for unknown properties :-/
+        if (typeof groupName !== 'string') return;
 
-            // groups are lowercase
-            groupName = groupName.toLowerCase();
+        // groups are lowercase
+        groupName = groupName.toLowerCase();
 
-            percent += step;
-            progressCallback({ percent, message: `Syncing... ${groupName}` });
+        percent += step;
+        progressCallback({ percent, message: `Syncing... ${groupName}` });
 
-            groups.getByName(groupName, function (error, result) {
-                if (error && error.reason !== BoxError.NOT_FOUND) return iteratorCallback(error);
+        const result = await groups.getByName(groupName);
 
-                if (!result) {
-                    debug(`[adding group] groupname=${groupName}`);
+        if (!result) {
+            debug(`syncGroups: [adding group] groupname=${groupName}`);
 
-                    groups.create(groupName, 'ldap', function (error) {
-                        if (error) debug('syncGroups: Failed to create group', groupName, error);
-                        iteratorCallback();
-                    });
-                } else {
-                    debug(`[up-to-date group] groupname=${groupName}`);
+            const [error] = await safe(groups.add({ name: groupName, source: 'ldap' }));
+            if (error) debug('syncGroups: Failed to create group', groupName, error);
+        } else {
+            debug(`syncGroups: [up-to-date group] groupname=${groupName}`);
+        }
+    }
 
-                    iteratorCallback();
-                }
-            });
-        }, function (error) {
-            if (error) return callback(error);
-
-            debug('sync: ldap sync is done', error);
-
-            callback(error);
-        });
-    });
+    debug('syncGroups: sync done');
 }
 
-function syncGroupUsers(externalLdapConfig, progressCallback, callback) {
+async function syncGroupUsers(externalLdapConfig, progressCallback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
 
     if (!externalLdapConfig.syncGroups) {
-        debug('Group users sync is disabled');
+        debug('syncGroupUsers: Group users sync is disabled');
         progressCallback({ percent: 99, message: 'Skipping group users sync...' });
-        return callback(null, []);
+        return [];
     }
 
-    groups.getAll(function (error, result) {
-        if (error) return callback(error);
+    const allGroups = await groups.list();
+    const ldapGroups = allGroups.filter(function (g) { return g.source === 'ldap'; });
+    debug(`syncGroupUsers: Found ${ldapGroups.length} groups to sync users`);
 
-        var ldapGroups = result.filter(function (g) { return g.source === 'ldap'; });
-        debug(`Found ${ldapGroups.length} groups to sync users`);
+    for (const group of ldapGroups) {
+        debug(`syncGroupUsers: Sync users for group ${group.name}`);
 
-        async.eachSeries(ldapGroups, function (group, iteratorCallback) {
-            debug(`Sync users for group ${group.name}`);
+        const result = await ldapGroupSearch(externalLdapConfig, {});
+        if (!result || result.length === 0) {
+            debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
+            continue;
+        }
 
-            ldapGroupSearch(externalLdapConfig, {}, function (error, result) {
-                if (error) return callback(error);
-                if (!result || result.length === 0) {
-                    debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
-                    return callback();
-                }
+        // since our group names are lowercase we cannot use potentially case matching ldap filters
+        let found = result.find(function (r) {
+            if (!r[externalLdapConfig.groupnameField]) return false;
+            return r[externalLdapConfig.groupnameField].toLowerCase() === group.name;
+        });
 
-                // since our group names are lowercase we cannot use potentially case matching ldap filters
-                let found = result.find(function (r) {
-                    if (!r[externalLdapConfig.groupnameField]) return false;
-                    return r[externalLdapConfig.groupnameField].toLowerCase() === group.name;
-                });
+        if (!found) {
+            debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
+            continue;
+        }
 
-                if (!found) {
-                    debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
-                    return callback();
-                }
+        let ldapGroupMembers = found.member || found.uniqueMember || [];
 
-                var ldapGroupMembers = found.member || found.uniqueMember || [];
+        // if only one entry is in the group ldap returns a string, not an array!
+        if (typeof ldapGroupMembers === 'string') ldapGroupMembers = [ ldapGroupMembers ];
 
-                // if only one entry is in the group ldap returns a string, not an array!
-                if (typeof ldapGroupMembers === 'string') ldapGroupMembers = [ ldapGroupMembers ];
+        debug(`syncGroupUsers: Group ${group.name} has ${ldapGroupMembers.length} members.`);
 
-                debug(`Group ${group.name} has ${ldapGroupMembers.length} members.`);
+        for (const memberDn of ldapGroupMembers) {
+            const [ldapError, result] = await safe(ldapGetByDN(externalLdapConfig, memberDn));
+            if (ldapError) {
+                debug(`syncGroupUsers: Failed to get ${memberDn}:`, ldapError);
+                continue;
+            }
 
-                async.eachSeries(ldapGroupMembers, function (memberDn, iteratorCallback) {
-                    ldapGetByDN(externalLdapConfig, memberDn, function (error, result) {
-                        if (error) {
-                            debug(`Failed to get ${memberDn}:`, error);
-                            return iteratorCallback();
-                        }
+            debug(`syncGroupUsers: Found member object at ${memberDn} adding to group ${group.name}`);
 
-                        debug(`Found member object at ${memberDn} adding to group ${group.name}`);
+            const username = result[externalLdapConfig.usernameField];
+            if (!username) continue;
 
-                        const username = result[externalLdapConfig.usernameField];
-                        if (!username) return iteratorCallback();
+            const [getError, userObject] = await safe(users.getByUsername(username));
+            if (getError || !userObject) {
+                debug(`syncGroupUsers: Failed to get user by username ${username}`, getError ? getError : 'User not found');
+                continue;
+            }
 
-                        users.getByUsername(username, function (error, result) {
-                            if (error) {
-                                debug(`syncGroupUsers: Failed to get user by username ${username}`, error);
-                                return iteratorCallback();
-                            }
+            const [addError] = await safe(groups.addMember(group.id, userObject.id));
+            if (addError && addError.reason !== BoxError.ALREADY_EXISTS) debug('syncGroupUsers: Failed to add member', addError);
+        }
+    }
 
-                            groups.addMember(group.id, result.id, function (error) {
-                                if (error && error.reason !== BoxError.ALREADY_EXISTS) debug('syncGroupUsers: Failed to add member', error);
-                                iteratorCallback();
-                            });
-                        });
-                    });
-                }, function (error) {
-                    if (error) debug('syncGroupUsers: ', error);
-                    iteratorCallback();
-                });
-            });
-        }, callback);
-    });
+    debug('syncGroupUsers: done');
 }
 
-function sync(progressCallback, callback) {
+async function sync(progressCallback) {
     assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
 
     progressCallback({ percent: 10, message: 'Starting ldap user sync' });
 
-    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
-        if (error) return callback(error);
-        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+    const externalLdapConfig = await settings.getExternalLdapConfig();
+    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
 
-        async.series([
-            syncUsers.bind(null, externalLdapConfig, progressCallback),
-            syncGroups.bind(null, externalLdapConfig, progressCallback),
-            syncGroupUsers.bind(null, externalLdapConfig, progressCallback)
-        ], function (error) {
-            if (error) return callback(error);
+    await syncUsers(externalLdapConfig, progressCallback);
+    await syncGroups(externalLdapConfig, progressCallback);
+    await syncGroupUsers(externalLdapConfig, progressCallback);
 
-            progressCallback({ percent: 100, message: 'Done' });
+    progressCallback({ percent: 100, message: 'Done' });
 
-            debug('sync: ldap sync is done', error);
-
-            callback(error);
-        });
-    });
+    debug('sync: done');
 }

src/groupdb.js

@@ -1,275 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get,
-    getByName,
-    getWithMembers,
-    getAll,
-    getAllWithMembers,
-    add,
-    update,
-    del,
-    count,
-
-    getMembers,
-    addMember,
-    removeMember,
-    setMembers,
-    isMember,
-
-    getMembership,
-    setMembership,
-
-    _clear: clear
-};
-
-var assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js');
-
-var GROUPS_FIELDS = [ 'id', 'name', 'source' ].join(',');
-
-function get(groupId, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups WHERE id = ? ORDER BY name', [ groupId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function getByName(name, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups WHERE name = ?', [ name ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function getWithMembers(groupId, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
-                    ' FROM userGroups LEFT OUTER JOIN groupMembers ON userGroups.id = groupMembers.groupId ' +
-                    ' WHERE userGroups.id = ? ' +
-                    ' GROUP BY userGroups.id', [ groupId ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
-
-        var result = results[0];
-        result.userIds = result.userIds ? result.userIds.split(',') : [ ];
-
-        callback(null, result);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups ORDER BY name', function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAllWithMembers(callback) {
-    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
-                    ' FROM userGroups LEFT OUTER JOIN groupMembers ON userGroups.id = groupMembers.groupId ' +
-                    ' GROUP BY userGroups.id ORDER BY name', function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
-
-        callback(null, results);
-    });
-}
-
-function add(id, name, source, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof source, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('INSERT INTO userGroups (id, name, source) VALUES (?, ?, ?)', [ id, name, source ], function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
-        if (error || result.affectedRows !== 1) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function update(id, data, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert(data && typeof data === 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var args = [ ];
-    var fields = [ ];
-    for (var k in data) {
-        if (k === 'name') {
-            assert.strictEqual(typeof data.name, 'string');
-            fields.push(k + ' = ?');
-            args.push(data.name);
-        }
-    }
-    args.push(id);
-
-    database.query('UPDATE userGroups SET ' + fields.join(', ') + ' WHERE id = ?', args, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('userGroups_name') !== -1) return callback(new BoxError(BoxError.ALREADY_EXISTS, 'name already exists'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
-
-        return callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    // also cleanup the groupMembers table
-    var queries = [];
-    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ id ] });
-    queries.push({ query: 'DELETE FROM userGroups WHERE id = ?', args: [ id ] });
-
-    database.transaction(queries, function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result[1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
-
-        callback(error);
-    });
-}
-
-function count(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT COUNT(*) AS total FROM userGroups', function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        return callback(null, result[0].total);
-    });
-}
-
-function clear(callback) {
-    database.query('DELETE FROM groupMembers', function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        database.query('DELETE FROM userGroups', function (error) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            callback(error);
-        });
-    });
-}
-
-function getMembers(groupId, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT userId FROM groupMembers WHERE groupId=?', [ groupId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        // if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found')); // need to differentiate group with no members and invalid groupId
-
-        callback(error, result.map(function (r) { return r.userId; }));
-    });
-}
-
-function setMembers(groupId, userIds, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert(Array.isArray(userIds));
-    assert.strictEqual(typeof callback, 'function');
-
-    var queries = [];
-    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ groupId ] });
-    for (var i = 0; i < userIds.length; i++) {
-        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', args: [ groupId, userIds[i] ] });
-    }
-
-    database.transaction(queries, function (error) {
-        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.CONFLICT, 'Duplicate member in list'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(error);
-    });
-}
-
-function getMembership(userId, callback) {
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT groupId FROM groupMembers WHERE userId=? ORDER BY groupId', [ userId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        // if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found')); // need to differentiate group with no members and invalid groupId
-
-        callback(error, result.map(function (r) { return r.groupId; }));
-    });
-}
-
-function setMembership(userId, groupIds, callback) {
-    assert.strictEqual(typeof userId, 'string');
-    assert(Array.isArray(groupIds));
-    assert.strictEqual(typeof callback, 'function');
-
-    var queries = [ ];
-    queries.push({ query: 'DELETE from groupMembers WHERE userId = ?', args: [ userId ] });
-    groupIds.forEach(function (gid) {
-        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (? , ?)', args: [ gid, userId ] });
-    });
-
-    database.transaction(queries, function (error) {
-        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.CONFLICT, 'Already member'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function addMember(groupId, userId, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ groupId, userId ], function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
-        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
-        if (error || result.affectedRows !== 1) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function removeMember(groupId, userId, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM groupMembers WHERE groupId = ? AND userId = ?', [ groupId, userId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
-
-        callback(null);
-    });
-}
-
-function isMember(groupId, userId, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT 1 FROM groupMembers WHERE groupId=? AND userId=?', [ groupId, userId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, result.length !== 0);
-    });
-}

src/groups.js

@@ -1,14 +1,14 @@
 'use strict';
 
 exports = module.exports = {
-    create,
+    add,
     remove,
     get,
     getByName,
     update,
     getWithMembers,
-    getAll,
-    getAllWithMembers,
+    list,
+    listWithMembers,
 
     getMembers,
     addMember,
@@ -18,28 +18,28 @@ exports = module.exports = {
 
     setMembership,
     getMembership,
-
-    count
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    groupdb = require('./groupdb.js'),
-    uuid = require('uuid'),
-    _ = require('underscore');
+    database = require('./database.js'),
+    safe = require('safetydance'),
+    uuid = require('uuid');
+
+const GROUPS_FIELDS = [ 'id', 'name', 'source' ].join(',');
 
 // keep this in sync with validateUsername
 function validateGroupname(name) {
     assert.strictEqual(typeof name, 'string');
 
-    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 char', { field: 'name' });
-    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'name too long', { field: 'name' });
+    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 char');
+    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'name too long');
 
-    if (constants.RESERVED_NAMES.indexOf(name) !== -1) return new BoxError(BoxError.BAD_FIELD, 'name is reserved', { field: name });
+    if (constants.RESERVED_NAMES.indexOf(name) !== -1) return new BoxError(BoxError.BAD_FIELD, 'name is reserved');
 
     // need to consider valid LDAP characters here (e.g '+' is reserved)
-    if (/[^a-zA-Z0-9.-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'name can only contain alphanumerals, hyphen and dot', { field: 'name' });
+    if (/[^a-zA-Z0-9.-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'name can only contain alphanumerals, hyphen and dot');
 
     return null;
 }
@@ -47,204 +47,192 @@ function validateGroupname(name) {
 function validateGroupSource(source) {
     assert.strictEqual(typeof source, 'string');
 
-    if (source !== '' && source !== 'ldap') return new BoxError(BoxError.BAD_FIELD, 'source must be "" or "ldap"', { field: source });
+    if (source !== '' && source !== 'ldap') return new BoxError(BoxError.BAD_FIELD, 'source must be "" or "ldap"');
 
     return null;
 }
 
-function create(name, source, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof source, 'string');
-    assert.strictEqual(typeof callback, 'function');
+async function add(group) {
+    assert.strictEqual(typeof group, 'object');
 
-    // we store names in lowercase
-    name = name.toLowerCase();
+    let { name, source } = group;
 
-    var error = validateGroupname(name);
-    if (error) return callback(error);
+    name = name.toLowerCase(); // we store names in lowercase
+    source = source || '';
+
+    let error = validateGroupname(name);
+    if (error) throw error;
 
     error = validateGroupSource(source);
-    if (error) return callback(error);
+    if (error) throw error;
 
-    var id = 'gid-' + uuid.v4();
-    groupdb.add(id, name, source, function (error) {
-        if (error) return callback(error);
+    const id = `gid-${uuid.v4()}`;
 
-        callback(null, { id: id, name: name });
-    });
+    [error] = await safe(database.query('INSERT INTO userGroups (id, name, source) VALUES (?, ?, ?)', [ id, name, source ]));
+    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, error);
+    if (error) throw error;
+
+    return { id, name };
 }
 
-function remove(id, callback) {
+async function remove(id) {
     assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    groupdb.del(id, function (error) {
-        if (error) return callback(error);
+    // also cleanup the groupMembers table
+    let queries = [];
+    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ id ] });
+    queries.push({ query: 'DELETE FROM userGroups WHERE id = ?', args: [ id ] });
 
-        callback(null);
-    });
+    const result = await database.transaction(queries);
+    if (result[1].affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
 }
 
-function get(id, callback) {
+async function get(id) {
     assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    groupdb.get(id, function (error, result) {
-        if (error) return callback(error);
+    const result = await database.query(`SELECT ${GROUPS_FIELDS} FROM userGroups WHERE id = ? ORDER BY name`, [ id ]);
+    if (result.length === 0) return null;
 
-        return callback(null, result);
-    });
+    return result[0];
 }
 
-function getByName(name, callback) {
+async function getByName(name) {
     assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    groupdb.getByName(name, function (error, result) {
-        if (error) return callback(error);
+    const result = await database.query(`SELECT ${GROUPS_FIELDS} FROM userGroups WHERE name = ?`, [ name ]);
+    if (result.length === 0) return null;
 
-        return callback(null, result);
-    });
+    return result[0];
 }
 
-function getWithMembers(id, callback) {
+async function getWithMembers(id) {
     assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    groupdb.getWithMembers(id, function (error, result) {
-        if (error) return callback(error);
+    const results = await database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+        ' FROM userGroups LEFT OUTER JOIN groupMembers ON userGroups.id = groupMembers.groupId ' +
+        ' WHERE userGroups.id = ? ' +
+        ' GROUP BY userGroups.id', [ id ]);
 
-        return callback(null, result);
-    });
+    if (results.length === 0) return null;
+
+    const result = results[0];
+    result.userIds = result.userIds ? result.userIds.split(',') : [ ];
+
+    return result;
 }
 
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    groupdb.getAll(function (error, result) {
-        if (error) return callback(error);
-
-        return callback(null, result);
-    });
+async function list() {
+    const results = await database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups ORDER BY name');
+    return results;
 }
 
-function getAllWithMembers(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function listWithMembers() {
+    const results = await database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+        ' FROM userGroups LEFT OUTER JOIN groupMembers ON userGroups.id = groupMembers.groupId ' +
+        ' GROUP BY userGroups.id ORDER BY name');
 
-    groupdb.getAllWithMembers(function (error, result) {
-        if (error) return callback(error);
-
-        return callback(null, result);
-    });
+    results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
+    return results;
 }
 
-function getMembers(groupId, callback) {
+async function getMembers(groupId) {
     assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    groupdb.getMembers(groupId, function (error, result) {
-        if (error) return callback(error);
+    const result = await database.query('SELECT userId FROM groupMembers WHERE groupId=?', [ groupId ]);
 
-        return callback(null, result);
-    });
+    return result.map(function (r) { return r.userId; });
 }
 
-function getMembership(userId, callback) {
+async function getMembership(userId) {
     assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    groupdb.getMembership(userId, function (error, result) {
-        if (error) return callback(error);
+    const result = await database.query('SELECT groupId FROM groupMembers WHERE userId=? ORDER BY groupId', [ userId ]);
 
-        return callback(null, result);
-    });
+    return result.map(function (r) { return r.groupId; });
 }
 
-function setMembership(userId, groupIds, callback) {
+async function setMembership(userId, groupIds) {
     assert.strictEqual(typeof userId, 'string');
     assert(Array.isArray(groupIds));
-    assert.strictEqual(typeof callback, 'function');
 
-    groupdb.setMembership(userId, groupIds, function (error) {
-        if (error) return callback(error);
-
-        return callback(null);
+    let queries = [ ];
+    queries.push({ query: 'DELETE from groupMembers WHERE userId = ?', args: [ userId ] });
+    groupIds.forEach(function (gid) {
+        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (? , ?)', args: [ gid, userId ] });
     });
+
+    const [error] = await safe(database.transaction(queries));
+    if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, error.message);
+    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.CONFLICT, 'Already member');
+    if (error) throw error;
 }
 
-function addMember(groupId, userId, callback) {
+async function addMember(groupId, userId) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    groupdb.addMember(groupId, userId, function (error) {
-        if (error) return callback(error);
-
-        return callback(null);
-    });
+    const [error] = await safe(database.query('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ groupId, userId ]));
+    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, error);
+    if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.includes('userId')) throw new BoxError(BoxError.NOT_FOUND, 'User not found');
+    if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
+    if (error) throw error;
 }
 
-function setMembers(groupId, userIds, callback) {
+async function setMembers(groupId, userIds) {
     assert.strictEqual(typeof groupId, 'string');
     assert(Array.isArray(userIds));
-    assert.strictEqual(typeof callback, 'function');
 
-    groupdb.setMembers(groupId, userIds, function (error) {
-        if (error) return callback(error);
-
-        return callback(null);
-    });
-}
-
-function removeMember(groupId, userId, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    groupdb.removeMember(groupId, userId, function (error) {
-        if (error) return callback(error);
-
-        return callback(null);
-    });
-}
-
-function isMember(groupId, userId, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    groupdb.isMember(groupId, userId, function (error, result) {
-        if (error) return callback(error);
-
-        return callback(null, result);
-    });
-}
-
-function update(groupId, data, callback) {
-    assert.strictEqual(typeof groupId, 'string');
-    assert(data && typeof data === 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    let error;
-    if ('name' in data) {
-        assert.strictEqual(typeof data.name, 'string');
-        error = validateGroupname(data.name);
-        if (error) return callback(error);
+    let queries = [];
+    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ groupId ] });
+    for (let i = 0; i < userIds.length; i++) {
+        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', args: [ groupId, userIds[i] ] });
     }
 
-    groupdb.update(groupId, _.pick(data, 'name'), function (error) {
-        if (error) return callback(error);
-
-        callback(null);
-    });
+    const [error] = await safe(database.transaction(queries));
+    if (error && error.code === 'ER_NO_REFERENCED_ROW_2') throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
+    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.CONFLICT, 'Duplicate member in list');
+    if (error) throw error;
 }
 
-function count(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function removeMember(groupId, userId) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
 
-    groupdb.count(function (error, count) {
-        if (error) return callback(error);
-
-        callback(null, count);
-    });
+    const result = await database.query('DELETE FROM groupMembers WHERE groupId = ? AND userId = ?', [ groupId, userId ]);
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
+}
+
+async function isMember(groupId, userId) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+
+    const result = await database.query('SELECT 1 FROM groupMembers WHERE groupId=? AND userId=?', [ groupId, userId ]);
+    return result.length !== 0;
+}
+
+async function update(id, data) {
+    assert.strictEqual(typeof id, 'string');
+    assert(data && typeof data === 'object');
+
+    if ('name' in data) {
+        assert.strictEqual(typeof data.name, 'string');
+        const error = validateGroupname(data.name);
+        if (error) throw error;
+    }
+
+    const args = [];
+    const fields = [];
+    for (const k in data) {
+        if (k === 'name') {
+            assert.strictEqual(typeof data.name, 'string');
+            fields.push(k + ' = ?');
+            args.push(data.name);
+        }
+    }
+    args.push(id);
+
+    const [updateError, result] = await safe(database.query('UPDATE userGroups SET ' + fields.join(', ') + ' WHERE id = ?', args));
+    if (updateError && updateError.code === 'ER_DUP_ENTRY' && updateError.sqlMessage.indexOf('userGroups_name') !== -1) throw new BoxError(BoxError.ALREADY_EXISTS, 'name already exists');
+    if (updateError) throw updateError;
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Group not found');
 }

src/hat.js

@@ -2,8 +2,8 @@
 
 exports = module.exports = hat;
 
-var crypto = require('crypto');
+const crypto = require('crypto');
 
-function hat (bits) {
+function hat(bits) {
     return crypto.randomBytes(bits / 8).toString('hex');
 }

src/infra_version.js

@@ -6,22 +6,22 @@
 
 exports = module.exports = {
     // a version change recreates all containers with latest docker config
-    'version': '48.20.0',
+    'version': '49.0.0',
 
     'baseImages': [
-        { repo: 'cloudron/base', tag: 'cloudron/base:3.0.0@sha256:455c70428723e3a823198c57472785437eb6eab082e79b3ff04ea584faf46e92' }
+        { repo: 'cloudron/base', tag: 'cloudron/base:3.2.0@sha256:ba1d566164a67c266782545ea9809dc611c4152e27686fd14060332dd88263ea' }
     ],
 
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.3.1@sha256:759cafab7625ff538418a1f2ed5558b1d5bff08c576bba577d865d6d02b49091' },
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.0.7@sha256:6679c2fb96f8d6d62349b607748570640a90fc46b50aad80ca2c0161655d07f4' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.0.6@sha256:e583082e15e8e41b0e3b80c3efc917ec429f19fa08a19e14fc27144a8bfe446a' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.0.2@sha256:9df297ccc3370f38c54f8d614e214e082b363777cd1c6c9522e29663cc8f5362' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.0.3@sha256:37e5222e01ae89bc5a742ce12030631de25a127b5deec8a0e992c68df0fdec10' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.3.3@sha256:b1093e6f38bebf4a9ae903ca385aea3a32e7cccae5ede7f2e01a34681e361a5f' },
-        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.0.1@sha256:bed9f6b5d06fe2c5289e895e806cfa5b74ad62993d705be55d4554a67d128029' },
-        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.3.0@sha256:183c11150d5a681cb02f7d2bd542ddb8a8f097422feafb7fac8fdbca0ca55d47' }
+        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.4.0@sha256:45817f1631992391d585f171498d257487d872480fd5646723a2b956cc4ef15d' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.2.1@sha256:75cef64ba4917ba9ec68bc0c9d9ba3a9eeae00a70173cd6d81cc6118038737d9' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.3.0@sha256:9f0cfe83310a6f67885c21804c01e73c8d256606217f44dcefb3e05a5402b2c9' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.0@sha256:c8ebdbe2663b26fcd58b1e6b97906b62565adbe4a06256ba0f86114f78b37e6b' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.2.1@sha256:30a5550c41c3be3a01dbf457497ba2f3c05f3121c595a17ef1aacbef931b6114' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.6.1@sha256:b8b93f007105080d4812a05648e6bc5e15c95c63f511c829cbc14a163d9ea029' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.1.0@sha256:30ec3a01964a1e01396acf265183997c3e17fb07eac1a82b979292cc7719ff4b' },
+        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.6.0@sha256:9c686b10c1a3ba344a743f399d08b4da5426e111f455114980f0ae0229c1ab23' }
     }
 };

src/iputils.js

@@ -7,6 +7,7 @@ exports = module.exports = {
 
 const assert = require('assert');
 
+// this code is used in migrations - 20201120212726-apps-add-containerIp.js
 function intFromIp(address) {
     assert.strictEqual(typeof address, 'string');
 
@@ -20,6 +21,7 @@ function intFromIp(address) {
            (parseInt(parts[3], 10) << (8*0)) & 0x000000FF;
 }
 
+// this code is used in migrations - 20201120212726-apps-add-containerIp.js
 function ipFromInt(input) {
     assert.strictEqual(typeof input, 'number');
 

src/janitor.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const assert = require('assert'),
-    async = require('async'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:janitor'),
     Docker = require('dockerode'),
@@ -13,8 +12,6 @@ exports = module.exports = {
     cleanupDockerVolumes
 };
 
-const NOOP_CALLBACK = function () { };
-
 const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });
 
 async function cleanupTokens() {
@@ -26,44 +23,34 @@ async function cleanupTokens() {
     debug(`Cleaned up ${result} expired tokens`,);
 }
 
-function cleanupTmpVolume(containerInfo, callback) {
+async function cleanupTmpVolume(containerInfo) {
     assert.strictEqual(typeof containerInfo, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    var cmd = 'find /tmp -type f -mtime +10 -exec rm -rf {} +'.split(' '); // 10 day old files
+    const cmd = 'find /tmp -type f -mtime +10 -exec rm -rf {} +'.split(' '); // 10 day old files
 
     debug('cleanupTmpVolume %j', containerInfo.Names);
 
-    gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
-        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`));
+    const [error, execContainer] = await safe(gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }));
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`);
 
-        execContainer.start({ hijack: true }, function (error, stream) {
-            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to start exec container: ${error.message}`));
+    const [startError, stream] = await safe(execContainer.start({ hijack: true }));
+    if (startError) throw new BoxError(BoxError.DOCKER_ERROR, `Failed to start exec container: ${startError.message}`);
 
-            stream.on('error', callback);
-            stream.on('end', callback);
+    gConnection.modem.demuxStream(stream, process.stdout, process.stderr);
 
-            gConnection.modem.demuxStream(stream, process.stdout, process.stderr);
-        });
+    return new Promise((resolve, reject) => {
+        stream.on('error', (error) => reject(new BoxError(BoxError.DOCKER_ERROR, `Failed to cleanup in exec container: ${error.message}`)));
+        stream.on('end', resolve);
     });
 }
 
-function cleanupDockerVolumes(callback) {
-    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
-
-    callback = callback || NOOP_CALLBACK;
-
+async function cleanupDockerVolumes() {
     debug('Cleaning up docker volumes');
 
-    gConnection.listContainers({ all: 0 }, function (error, containers) {
-        if (error) return callback(error);
+    const [error, containers] = await safe(gConnection.listContainers({ all: 0 }));
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
 
-        async.eachSeries(containers, function (container, iteratorDone) {
-            cleanupTmpVolume(container, function (error) {
-                if (error) debug('Error cleaning tmp: %s', error);
-
-                iteratorDone(); // intentionally ignore error
-            });
-        }, callback);
-    });
+    for (const container of containers) {
+        await safe(cleanupTmpVolume(container), { debug }); // intentionally ignore error
+    }
 }

src/locker.js

@@ -55,7 +55,7 @@ Locker.prototype.recursiveLock = function (operation) {
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
-    if (this._operation !== operation) throw BoxError(BoxError.BAD_STATE, 'Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
+    if (this._operation !== operation) throw new BoxError(BoxError.BAD_STATE, 'Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
     if (--this._lockDepth === 0) {
         debug('Released : %s', this._operation);