backups: fix incorrect mountpoint check with managed mounts

it failed with:
Feb 22 08:52:30 strawberry cloudron-firewall.sh[14300]: /home/yellowtent/box/setup/start/cloudron-firewall.sh: line 14: iptables --wait 120 --wait-interval 1: command not found

the root cause was that IFS was getting set but not getting reset later.
the IFS=xx line is not line local as it seems to appear (just a bash statement)

.jshintrc

@@ -1,5 +1,5 @@
 {
     "node": true,
     "unused": true,
-    "esversion": 8
+    "esversion": 11
 }

CHANGES

@@ -2399,3 +2399,39 @@
 * backups: fix issue where mail backups where not cleaned up
 * notifications: fix automatic app update notifications
 
+[7.1.0]
+* Add mail manager role
+* mailbox: app can be set as owner when recvmail addon enabled
+* domains: add well known config UI (for jitsi configuration)
+* Prefix email addon variables with CLOUDRON_EMAIL instead of CLOUDRON_MAIL
+* remove support for manifest version 1
+* Add option to enable/disable mailbox sharing
+* base image 3.2.0
+* Update node to 16.13.1
+* mongodb: update to 4.4
+* Add `upstreamVersion` to manifest
+* Add `logPaths` to manifest
+* Add cifs seal support for backup and volume mounts
+* add a way for admins to set username when profiles are locked
+* Add support for secondary domains
+* postgresql: enable postgis
+* remove nginx config of stopped apps
+* mail: use port25check.cloudron.io to check outbound port 25 connectivity
+* Add import/export of mailboxes and users
+* LDAP server can now be exposed
+* Update monaco-editor to 0.32.1
+* Update xterm.js to 4.17.0
+* Update docker to 20.10.12
+* IPv6 support
+
+[7.1.1]
+* Fix issue where dkimKey of a mail domain is sometimes null
+* firewall: add retry for xtables lock
+* redis: fix issue where protected mode was enabled with no password
+
+[7.1.2]
+* Fix crash in cloudron-firewall when ports are whitelisted
+* eventlog: add event for certificate cleanup
+* eventlog: log event for mailbox alias update
+* backups: fix incorrect mountpoint check with managed mounts
+

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2021 Cloudron UG
+Copyright (c) 2022 Cloudron UG
 
 With regard to the Cloudron Software:
 

baseimage/initializeBaseUbuntuImage.sh

@@ -76,7 +76,7 @@ apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends
 cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
 echo "==> Installing node.js"
-readonly node_version=14.17.6
+readonly node_version=16.13.1
 mkdir -p /usr/local/node-${node_version}
 curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-${node_version}
 ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
@@ -89,11 +89,11 @@ echo "==> Installing Docker"
 
 # create systemd drop-in file. if you channge options here, be sure to fixup installer.sh as well
 mkdir -p /etc/systemd/system/docker.service.d
-echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
+echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
 
 # there are 3 packages for docker - containerd, CLI and the daemon
-readonly docker_version=20.10.7
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.6-1_amd64.deb" -o /tmp/containerd.deb
+readonly docker_version=20.10.12
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.9-1_amd64.deb" -o /tmp/containerd.deb
 curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
 curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 # apt install with install deps (as opposed to dpkg -i)
@@ -157,6 +157,17 @@ if [ -f "/etc/default/motd-news" ]; then
     sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
 fi
 
+# If privacy extensions are not disabled on server, this breaks IPv6 detection
+# https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756
+if [[ ! -f /etc/sysctl.d/99-cloudimg-ipv6.conf ]]; then
+    echo "==> Disable temporary address (IPv6)"
+    echo -e "# See https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756\nnet.ipv6.conf.all.use_tempaddr = 0\nnet.ipv6.conf.default.use_tempaddr = 0\n\n" > /etc/sysctl.d/99-cloudimg-ipv6.conf
+fi
+
+# Disable exim4 (1blu.de)
+systemctl stop exim4 || true
+systemctl disable exim4 || true
+
 # Disable bind for good measure (on online.net, kimsufi servers these are pre-installed)
 systemctl stop bind9 || true
 systemctl disable bind9 || true
@@ -173,8 +184,9 @@ systemctl disable postfix || true
 systemctl stop systemd-resolved || true
 systemctl disable systemd-resolved || true
 
-# ubuntu's default config for unbound does not work if ipv6 is disabled. this config is overwritten in start.sh
+# on vultr, ufw is enabled by default. we have our own firewall
+ufw disable
+
 # we need unbound to work as this is required for installer.sh to do any DNS requests
-ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
-echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: ${ip6}" > /etc/unbound/unbound.conf.d/cloudron-network.conf
+echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: no" > /etc/unbound/unbound.conf.d/cloudron-network.conf
 systemctl restart unbound

box.js

@@ -2,13 +2,14 @@
 
 'use strict';
 
-const dockerProxy = require('./src/dockerproxy.js'),
-    fs = require('fs'),
+const fs = require('fs'),
     ldap = require('./src/ldap.js'),
     paths = require('./src/paths.js'),
     proxyAuth = require('./src/proxyauth.js'),
     safe = require('safetydance'),
-    server = require('./src/server.js');
+    server = require('./src/server.js'),
+    settings = require('./src/settings.js'),
+    userdirectory = require('./src/userdirectory.js');
 
 let logFd;
 
@@ -36,15 +37,16 @@ async function startServers() {
     await server.start(); // do this first since it also inits the database
     await proxyAuth.start();
     await ldap.start();
-    await dockerProxy.start();
+
+    const conf = await settings.getUserDirectoryConfig();
+    if (conf.enabled) await userdirectory.start();
 }
 
 async function main() {
     const [error] = await safe(startServers());
     if (error) return exitSync({ error: new Error(`Error starting server: ${JSON.stringify(error)}`), code: 1 });
 
-    // require those here so that logging handler is already setup
-    require('supererror');
+    // require this here so that logging handler is already setup
     const debug = require('debug')('box:box');
 
     process.on('SIGINT', async function () {
@@ -52,8 +54,8 @@ async function main() {
 
         await proxyAuth.stop();
         await server.stop();
+        await userdirectory.stop();
         await ldap.stop();
-        await dockerProxy.stop();
         setTimeout(process.exit.bind(process), 3000);
     });
 
@@ -62,8 +64,8 @@ async function main() {
 
         await proxyAuth.stop();
         await server.stop();
+        await userdirectory.stop();
         await ldap.stop();
-        await dockerProxy.stop();
         setTimeout(process.exit.bind(process), 3000);
     });
 

migrations/20211006200150-domains-ensure-fallbackCertificate.js

@@ -12,6 +12,8 @@ exports.up = function(db, callback) {
     db.all('SELECT * FROM domains', [ ], function (error, domains) {
         if (error) return callback(error);
 
+        // this code is br0ken since async 3.x since async functions won't get iteratorDone anymore
+        // no point fixing this migration though since it won't run again in old cloudrons. and in new cloudron domains will be empty
         async.eachSeries(domains, async function (domain, iteratorDone) {
             let fallbackCertificate = safe.JSON.parse(domain.fallbackCertificateJson);
             if (!fallbackCertificate || !fallbackCertificate.cert || !fallbackCertificate.key) {

migrations/20211117201805-eventlog-rename-source-to-sourceJson.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE source sourceJson TEXT', []),
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE data dataJson TEXT', []),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE sourceJson source TEXT', []),
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE dataJson data TEXT', []),
+    ], callback);
+};

migrations/20220107015249-settings-sysinfo-ip-to-ipv4.js

@@ -0,0 +1,17 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('SELECT value FROM settings WHERE name=?', [ 'sysinfo_config' ], function (error, result) {
+        if (error || result.length === 0) return callback(error);
+        const sysinfoConfig = JSON.parse(result[0].value);
+        if (sysinfoConfig.provider !== 'fixed' || !sysinfoConfig.ip) return callback();
+        sysinfoConfig.ipv4 = sysinfoConfig.ip;
+        delete sysinfoConfig.ip;
+
+        db.runSql('REPLACE INTO settings (name, value) VALUES(?, ?)', [ 'sysinfo_config', JSON.stringify(sysinfoConfig) ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220113223425-settings-rename-directory-config-to-profile-config.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'directory_config', 'profile_config' ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'profile_config', 'directory_config' ], callback);
+};

migrations/20220115070646-subdomains-add-environmentVariable.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains ADD COLUMN environmentVariable VARCHAR(128)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains DROP COLUMN environmentVariable', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20220202011148-blobs-migrate-token-secret.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const safe = require('safetydance');
+
+const PROXY_AUTH_TOKEN_SECRET_FILE = '/home/yellowtent/platformdata/proxy-auth-token-secret';
+
+exports.up = function (db, callback) {
+    const token = safe.fs.readFileSync(PROXY_AUTH_TOKEN_SECRET_FILE);
+    if (!token) return callback();
+    db.runSql('INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'proxy_auth_token_secret', token ], function (error) {
+        if (error) return callback(error);
+        safe.fs.unlinkSync(PROXY_AUTH_TOKEN_SECRET_FILE);
+        callback();
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220207214956-rename-subdomains-to-locations.js

@@ -0,0 +1,12 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('RENAME TABLE subdomains TO locations', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220219220751-mail-ensure-dkimKey.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const async = require('async'),
+    mail = require('../src/mail.js'),
+    safe = require('safetydance'),
+    util = require('util');
+
+// it seems some mail domains do not have dkimKey in the database for some reason because of some previous bad migration
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM mail', [ ], function (error, mailDomains) {
+        if (error) return callback(error);
+
+        async.eachSeries(mailDomains, function (mailDomain, iteratorDone) {
+            let dkimKey = safe.JSON.parse(mailDomain.dkimKeyJson);
+            if (dkimKey && dkimKey.publicKey && dkimKey.privateKey) return iteratorDone();
+            console.log(`${mailDomain.domain} has no dkim key in the database. generating a new one`);
+            util.callbackify(mail.generateDkimKey)(function (error, dkimKey) {
+                if (error) return iteratorDone(error);
+                db.runSql('UPDATE mail SET dkimKeyJson=? WHERE domain=?', [ JSON.stringify(dkimKey), mailDomain.domain ], iteratorDone);
+            });
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/schema.sql

@@ -98,6 +98,7 @@ CREATE TABLE IF NOT EXISTS apps(
     containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair'
     appStoreIcon MEDIUMBLOB,
     icon MEDIUMBLOB,
+    crontab TEXT,
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
@@ -149,8 +150,8 @@ CREATE TABLE IF NOT EXISTS backups(
 CREATE TABLE IF NOT EXISTS eventlog(
     id VARCHAR(128) NOT NULL,
     action VARCHAR(128) NOT NULL,
-    source TEXT, /* { userId, username, ip }. userId can be null for cron,sysadmin */
-    data TEXT, /* free flowing json based on action */
+    sourceJson TEXT, /* { userId, username, ip }. userId can be null for cron,sysadmin */
+    dataJson TEXT, /* free flowing json based on action */
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
 
     INDEX creationTime_index (creationTime),
@@ -213,11 +214,12 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
     UNIQUE (name, domain));
 
-CREATE TABLE IF NOT EXISTS subdomains(
+CREATE TABLE IF NOT EXISTS locations(
     appId VARCHAR(128) NOT NULL,
     domain VARCHAR(128) NOT NULL,
     subdomain VARCHAR(128) NOT NULL,
-    type VARCHAR(128) NOT NULL, /* primary or redirect */
+    type VARCHAR(128) NOT NULL, /* primary, secondary, redirect, alias */
+    environmentVariable VARCHAR(128), /* only set for secondary */
 
     certificateJson MEDIUMTEXT,
 
@@ -285,7 +287,7 @@ CREATE TABLE IF NOT EXISTS appMounts(
 
 CREATE TABLE IF NOT EXISTS blobs(
     id VARCHAR(128) NOT NULL UNIQUE,
-    value TEXT,
+    value MEDIUMBLOB,
     PRIMARY KEY(id));
 
 CHARACTER SET utf8 COLLATE utf8_bin;

package.json

@@ -11,76 +11,72 @@
     "url": "https://git.cloudron.io/cloudron/box.git"
   },
   "dependencies": {
-    "@google-cloud/dns": "^2.2.0",
-    "@google-cloud/storage": "^5.8.5",
+    "@google-cloud/dns": "^2.2.4",
+    "@google-cloud/storage": "^5.16.1",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^3.2.0",
-    "aws-sdk": "^2.936.0",
+    "async": "^3.2.2",
+    "aws-sdk": "^2.1053.0",
     "basic-auth": "^2.0.1",
-    "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.11.0",
+    "body-parser": "^1.19.1",
+    "cloudron-manifestformat": "^5.15.0",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.1.1",
     "connect-timeout": "^1.9.0",
-    "cookie-parser": "^1.4.5",
-    "cookie-session": "^1.4.0",
+    "cookie-parser": "^1.4.6",
+    "cookie-session": "^2.0.0",
     "cron": "^1.8.2",
-    "db-migrate": "^0.11.12",
-    "db-migrate-mysql": "^2.1.2",
-    "debug": "^4.3.1",
+    "db-migrate": "^0.11.13",
+    "db-migrate-mysql": "^2.2.0",
+    "debug": "^4.3.3",
     "delay": "^5.0.0",
-    "dockerode": "^3.3.0",
+    "dockerode": "^3.3.1",
     "ejs": "^3.1.6",
-    "ejs-cli": "^2.2.1",
-    "express": "^4.17.1",
+    "ejs-cli": "^2.2.3",
+    "express": "^4.17.2",
     "ipaddr.js": "^2.0.1",
     "js-yaml": "^4.1.0",
     "json": "^11.0.0",
     "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.3.0",
+    "ldapjs": "^2.3.1",
     "lodash": "^4.17.21",
     "lodash.chunk": "^4.2.0",
-    "mime": "^2.5.2",
     "moment": "^2.29.1",
-    "moment-timezone": "^0.5.33",
+    "moment-timezone": "^0.5.34",
     "morgan": "^1.10.0",
     "multiparty": "^4.2.2",
-    "mustache-express": "^1.3.1",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.6.2",
+    "nodemailer": "^6.7.2",
     "nodemailer-smtp-transport": "^2.7.4",
     "once": "^1.4.0",
     "pretty-bytes": "^5.6.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
-    "qrcode": "^1.4.4",
+    "qrcode": "^1.5.0",
     "readdirp": "^3.6.0",
-    "request": "^2.88.2",
     "s3-block-read-stream": "^0.5.0",
     "safetydance": "^2.2.0",
     "semver": "^7.3.5",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^6.1.0",
-    "supererror": "^0.7.2",
+    "superagent": "^7.0.1",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
-    "ua-parser-js": "^0.7.28",
-    "underscore": "^1.13.1",
+    "ua-parser-js": "^1.0.2",
+    "underscore": "^1.13.2",
     "uuid": "^8.3.2",
-    "validator": "^13.6.0",
-    "ws": "^7.5.1",
+    "validator": "^13.7.0",
+    "ws": "^8.4.0",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.4.1",
-    "js2xmlparser": "^4.0.1",
-    "mocha": "^9.0.1",
+    "js2xmlparser": "^4.0.2",
+    "mocha": "^9.1.3",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^13.1.0",
-    "node-sass": "^6.0.1",
+    "nock": "^13.2.1",
+    "node-sass": "^7.0.1",
     "recursive-readdir": "^2.2.2"
   },
   "scripts": {

runTests

@@ -45,7 +45,7 @@ else
 fi
 
 # create docker network (while the infra code does this, most tests skip infra setup)
-docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 --gateway 172.18.0.1 cloudron || true
+docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 --gateway 172.18.0.1 cloudron --ipv6 --subnet=fd00:c107:d509::/64 || true
 
 # create the same mysql server version to test with
 OUT=`docker inspect mysql-server` || true

scripts/cloudron-setup

@@ -173,7 +173,7 @@ if ! sourceTarballUrl=$(echo "${releaseJson}" | python3 -c 'import json,sys;obj=
     exit 1
 fi
 
-echo "=> Downloading version ${version} ..."
+echo "=> Downloading Cloudron version ${version} ..."
 box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)
 
 if ! $curl -sL "${sourceTarballUrl}" | tar -zxf - -C "${box_src_tmp_dir}"; then
@@ -192,7 +192,7 @@ if [[ "${initBaseImage}" == "true" ]]; then
 fi
 
 # The provider flag is still used for marketplace images
-echo "=> Installing version ${version} (this takes some time) ..."
+echo "=> Installing Cloudron version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
 echo "${provider}" > /etc/cloudron/PROVIDER
 [[ ! -z "${setupToken}" ]] && echo "${setupToken}" > /etc/cloudron/SETUP_TOKEN
@@ -214,7 +214,7 @@ while true; do
     sleep 10
 done
 
-if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
     ip='<IP>'
 fi
 if [[ -z "${setupToken}" ]]; then

scripts/createReleaseTarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v14.17.6" ]]; then
-    echo "This script requires node 14.17.6"
+if [[ "$(node --version)" != "v16.13.1" ]]; then
+    echo "This script requires node 16.13.1"
     exit 1
 fi
 

scripts/installer.sh

@@ -73,20 +73,16 @@ log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION
 
 log "updating docker"
 
-readonly docker_version=20.10.7
+readonly docker_version=20.10.12
 if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
     # there are 3 packages for docker - containerd, CLI and the daemon
-    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.6-1_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.9-1_amd64.deb" -o /tmp/containerd.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
+    log "installing docker"
     prepare_apt_once
-
-    while ! apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb; do
-        log "Failed to install docker. Retry"
-        sleep 1
-    done
-
+    apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
     rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
 fi
 
@@ -115,13 +111,13 @@ if ! which sshfs; then
 fi
 
 log "updating node"
-readonly node_version=14.17.6
+readonly node_version=16.13.1
 if [[ "$(node --version)" != "v${node_version}" ]]; then
     mkdir -p /usr/local/node-${node_version}
     $curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-${node_version}
     ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
     ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-14.15.4
+    rm -rf /usr/local/node-14.17.6
 fi
 
 # this is here (and not in updater.js) because rebuild requires the above node
@@ -159,7 +155,7 @@ done
 log "update cloudron-syslog"
 CLOUDRON_SYSLOG_DIR=/usr/local/cloudron-syslog
 CLOUDRON_SYSLOG="${CLOUDRON_SYSLOG_DIR}/bin/cloudron-syslog"
-CLOUDRON_SYSLOG_VERSION="1.0.3"
+CLOUDRON_SYSLOG_VERSION="1.1.0"
 while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLOUDRON_SYSLOG_VERSION} ]]; do
     rm -rf "${CLOUDRON_SYSLOG_DIR}"
     mkdir -p "${CLOUDRON_SYSLOG_DIR}"

setup/start.sh

@@ -37,8 +37,13 @@ systemctl enable apparmor
 systemctl restart apparmor
 
 usermod ${USER} -a -G docker
-# unbound (which starts after box code) relies on this interface to exist. dockerproxy also relies on this.
-docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 --gateway 172.18.0.1 cloudron || true
+
+if ! grep -q ip6tables /etc/systemd/system/docker.service.d/cloudron.conf; then
+    log "Adding ip6tables flag to docker" # https://github.com/moby/moby/pull/41622
+    echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
+    systemctl daemon-reload
+    systemctl restart docker
+fi
 
 mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
@@ -94,11 +99,6 @@ systemctl restart systemd-journald
 usermod -a -G adm ${USER}
 
 log "Setting up unbound"
-# DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org)
-# We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
-# We listen on 0.0.0.0 because there is no way control ordering of docker (which creates the 172.18.0.0/16) and unbound
-# If IP6 is not enabled, dns queries seem to fail on some hosts. -s returns false if file missing or 0 size
-ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
 cp -f "${script_dir}/start/unbound.conf" /etc/unbound/unbound.conf.d/cloudron-network.conf
 # update the root anchor after a out-of-disk-space situation (see #269)
 unbound-anchor -a /var/lib/unbound/root.key
@@ -114,7 +114,7 @@ systemctl enable box
 systemctl enable cloudron-firewall
 systemctl enable --now cloudron-disable-thp
 
-# update firewall rules
+# update firewall rules. this must be done after docker created it's rules
 systemctl restart cloudron-firewall
 
 # For logrotate
@@ -142,6 +142,15 @@ if [[ "${ubuntu_version}" == "20.04" ]]; then
 fi
 systemctl restart collectd
 
+log "Configuring sysctl"
+# If privacy extensions are not disabled on server, this breaks IPv6 detection
+# https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756
+if [[ ! -f /etc/sysctl.d/99-cloudimg-ipv6.conf ]]; then
+    echo "==> Disable temporary address (IPv6)"
+    echo -e "# See https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756\nnet.ipv6.conf.all.use_tempaddr = 0\nnet.ipv6.conf.default.use_tempaddr = 0\n\n" > /etc/sysctl.d/99-cloudimg-ipv6.conf
+    sysctl -p
+fi
+
 log "Configuring logrotate"
 if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then
     echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf

setup/start/cloudron-firewall.sh

@@ -3,100 +3,148 @@
 set -eu -o pipefail
 
 echo "==> Setting up firewall"
-iptables -t filter -N CLOUDRON || true
-iptables -t filter -F CLOUDRON # empty any existing rules
+
+has_ipv6=$(cat /proc/net/if_inet6 >/dev/null 2>&1 && echo "yes" || echo "no")
+
+# wait for 120 seconds for xtables lock, checking every 1 second
+readonly iptables="iptables --wait 120 --wait-interval 1"
+readonly ip6tables="ip6tables --wait 120 --wait-interval 1"
+
+function ipxtables() {
+    $iptables "$@"
+    [[ "${has_ipv6}" == "yes" ]] && $ip6tables "$@"
+}
+
+ipxtables -t filter -N CLOUDRON || true
+ipxtables -t filter -F CLOUDRON # empty any existing rules
 
 # first setup any user IP block lists
 ipset create cloudron_blocklist hash:net || true
+ipset create cloudron_blocklist6 hash:net family inet6 || true
+
 /home/yellowtent/box/src/scripts/setblocklist.sh
 
-iptables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist src -j DROP
+$iptables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist src -j DROP
 # the DOCKER-USER chain is not cleared on docker restart
-if ! iptables -t filter -C DOCKER-USER -m set --match-set cloudron_blocklist src -j DROP; then
-    iptables -t filter -I DOCKER-USER 1 -m set --match-set cloudron_blocklist src -j DROP
+if ! $iptables -t filter -C DOCKER-USER -m set --match-set cloudron_blocklist src -j DROP; then
+    $iptables -t filter -I DOCKER-USER 1 -m set --match-set cloudron_blocklist src -j DROP
 fi
 
+$ip6tables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist6 src -j DROP
+# there is no DOCKER-USER chain in ip6tables, bug?
+$ip6tables -D FORWARD -m set --match-set cloudron_blocklist6 src -j DROP || true
+$ip6tables -I FORWARD 1 -m set --match-set cloudron_blocklist6 src -j DROP
+
 # allow related and establisted connections
-iptables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
-iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,443 -j ACCEPT # 202 is the alternate ssh port
+ipxtables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,443 -j ACCEPT # 202 is the alternate ssh port
 
 # whitelist any user ports. we used to use --dports but it has a 15 port limit (XT_MULTI_PORTS)
 ports_json="/home/yellowtent/platformdata/firewall/ports.json"
-if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(','))" 2>/dev/null); then
-    IFS=',' arr=(${allowed_tcp_ports})
-    for p in "${arr[@]}"; do
-        iptables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
+if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(' '))" 2>/dev/null); then
+    for p in $allowed_tcp_ports; do
+        ipxtables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
     done
 fi
 
-if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(','))" 2>/dev/null); then
-    IFS=',' arr=(${allowed_udp_ports})
-    for p in "${arr[@]}"; do
-        iptables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
+if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(' '))" 2>/dev/null); then
+    for p in $allowed_udp_ports; do
+        ipxtables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
     done
 fi
 
+# LDAP user directory allow list
+ipset create cloudron_ldap_allowlist hash:net || true
+ipset flush cloudron_ldap_allowlist
+
+ipset create cloudron_ldap_allowlist6 hash:net family inet6 || true
+ipset flush cloudron_ldap_allowlist6
+
+ldap_allowlist_json="/home/yellowtent/platformdata/firewall/ldap_allowlist.txt"
+if [[ -f "${ldap_allowlist_json}" ]]; then
+    # without the -n block, any last line without a new line won't be read it!
+    while read -r line || [[ -n "$line" ]]; do
+        [[ -z "${line}" ]] && continue # ignore empty lines
+        [[ "$line" =~ ^#.*$ ]] && continue # ignore lines starting with #
+        if [[ "$line" == *":"* ]]; then
+            ipset add -! cloudron_ldap_allowlist6 "${line}" # the -! ignore duplicates
+        else
+            ipset add -! cloudron_ldap_allowlist "${line}" # the -! ignore duplicates
+        fi
+    done < "${ldap_allowlist_json}"
+
+    # ldap server we expose 3004 and also redirect from standard ldaps port 636
+    ipxtables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
+    $iptables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist src -p tcp --dport 3004 -j ACCEPT
+    $ip6tables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist6 src -p tcp --dport 3004 -j ACCEPT
+fi
+
 # turn and stun service
-iptables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
 
-iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
-iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT
-iptables -t filter -A CLOUDRON -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
-iptables -t filter -A CLOUDRON -i lo -j ACCEPT # required for localhost connections (mysql)
+# ICMPv6 is very fundamental to IPv6 connectivity unlike ICMPv4
+$iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
+$iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
+$ip6tables -t filter -A CLOUDRON -p ipv6-icmp -j ACCEPT
+
+ipxtables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT
+$iptables -t filter -A CLOUDRON -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+ipxtables -t filter -A CLOUDRON -i lo -j ACCEPT # required for localhost connections (mysql)
 
 # log dropped incoming. keep this at the end of all the rules
-iptables -t filter -A CLOUDRON -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
-iptables -t filter -A CLOUDRON -j DROP
+ipxtables -t filter -A CLOUDRON -m limit --limit 2/min -j LOG --log-prefix "Packet dropped: " --log-level 7
+ipxtables -t filter -A CLOUDRON -j DROP
 
-if ! iptables -t filter -C INPUT -j CLOUDRON 2>/dev/null; then
-    iptables -t filter -I INPUT -j CLOUDRON
-fi
+# prepend our chain to the filter table
+$iptables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $iptables -t filter -I INPUT -j CLOUDRON
+$ip6tables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $ip6tables -t filter -I INPUT -j CLOUDRON
 
 # Setup rate limit chain (the recent info is at /proc/net/xt_recent)
-iptables -t filter -N CLOUDRON_RATELIMIT || true
-iptables -t filter -F CLOUDRON_RATELIMIT # empty any existing rules
+ipxtables -t filter -N CLOUDRON_RATELIMIT || true
+ipxtables -t filter -F CLOUDRON_RATELIMIT # empty any existing rules
 
 # log dropped incoming. keep this at the end of all the rules
-iptables -t filter -N CLOUDRON_RATELIMIT_LOG || true
-iptables -t filter -F CLOUDRON_RATELIMIT_LOG # empty any existing rules
-iptables -t filter -A CLOUDRON_RATELIMIT_LOG -m limit --limit 2/min -j LOG --log-prefix "IPTables RateLimit: " --log-level 7
-iptables -t filter -A CLOUDRON_RATELIMIT_LOG -j DROP
+ipxtables -t filter -N CLOUDRON_RATELIMIT_LOG || true
+ipxtables -t filter -F CLOUDRON_RATELIMIT_LOG # empty any existing rules
+ipxtables -t filter -A CLOUDRON_RATELIMIT_LOG -m limit --limit 2/min -j LOG --log-prefix "IPTables RateLimit: " --log-level 7
+ipxtables -t filter -A CLOUDRON_RATELIMIT_LOG -j DROP
 
 # http https
 for port in 80 443; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
 # ssh
 for port in 22 202; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
-    iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
+done
+
+# ldaps
+for port in 636 3004; do
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
 # docker translates (dnat) 25, 587, 993, 4190 in the PREROUTING step
 for port in 2525 4190 9993; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
+    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
 done
 
 # msa, ldap, imap, sieve, pop3
 for port in 2525 3002 4190 9993 9995; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
+    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
 done
 
 # cloudron docker network: mysql postgresql redis mongodb
 for port in 3306 5432 6379 27017; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
+    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
-if ! iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null; then
-    iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
-fi
+$iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
+$ip6tables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $ip6tables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
 
 # Workaround issue where Docker insists on adding itself first in FORWARD table
-iptables -D FORWARD -j CLOUDRON_RATELIMIT || true
-iptables -I FORWARD 1 -j CLOUDRON_RATELIMIT
-
-echo "==> Setting up firewall done"
+ipxtables -D FORWARD -j CLOUDRON_RATELIMIT || true
+ipxtables -I FORWARD 1 -j CLOUDRON_RATELIMIT

setup/start/cloudron-motd

@@ -4,13 +4,17 @@
 
 printf "**********************************************************************\n\n"
 
+cache_file="/var/cache/cloudron-motd-cache"
+
 if [[ -z "$(ls -A /home/yellowtent/platformdata/addons/mail/dkim)" ]]; then
-    if [[ -f /tmp/.cloudron-motd-cache ]]; then
-        ip=$(cat /tmp/.cloudron-motd-cache)
-    elif ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+    if [[ ! -f "${cache_file}" ]]; then
+        curl --fail --connect-timeout 2 --max-time 2 -q https://ipv4.api.cloudron.io/api/v1/helper/public_ip --output "${cache_file}" || true
+    fi
+    if [[ -f "${cache_file}" ]]; then
+        ip=$(sed -n -e 's/.*"ip": "\(.*\)"/\1/p' /var/cache/cloudron-motd-cache)
+    else
         ip='<IP>'
     fi
-    echo "${ip}" > /tmp/.cloudron-motd-cache
 
     if [[ ! -f /etc/cloudron/SETUP_TOKEN ]]; then
         url="https://${ip}"

setup/start/nginx/nginx.conf

@@ -19,15 +19,10 @@ http {
     include       mime.types;
     default_type  application/octet-stream;
 
-    # the collectd config depends on this log format
-    log_format combined2 '$remote_addr - [$time_local] '
-        '"$request" $status $body_bytes_sent $request_time '
-        '"$http_referer" "$host" "$http_user_agent"';
-
     # required for long host names
     server_names_hash_bucket_size 128;
 
-    access_log /var/log/nginx/access.log combined2;
+    access_log /var/log/nginx/access.log combined;
 
     sendfile        on;
 

setup/start/sudoers

@@ -53,6 +53,9 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/stoptask.sh
 Defaults!/home/yellowtent/box/src/scripts/setblocklist.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setblocklist.sh
 
+Defaults!/home/yellowtent/box/src/scripts/setldapallowlist.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setldapallowlist.sh
+
 Defaults!/home/yellowtent/box/src/scripts/addmount.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/addmount.sh
 

setup/start/systemd/unbound.service

@@ -2,7 +2,7 @@
 
 [Unit]
 Description=Unbound DNS Resolver
-After=network-online.target docker.service
+After=network-online.target
 Before=nss-lookup.target
 Wants=network-online.target nss-lookup.target
 

setup/start/unbound.conf

@@ -1,7 +1,11 @@
+# Unbound is used primarily for RBL queries (host 2.0.0.127.zen.spamhaus.org)
+# We cannot use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
+
 server:
         port: 53
         interface: 127.0.0.1
         interface: 172.18.0.1
+        ip-freebind: yes
         do-ip6: no
         access-control: 127.0.0.1 allow
         access-control: 172.18.0.1/16 allow
@@ -10,4 +14,3 @@ server:
         # enable below for logging to journalctl -u unbound
         # verbosity: 5
         # log-queries: yes
-

src/acme2.js

@@ -204,7 +204,7 @@ Acme2.prototype.waitForOrder = async function (orderUrl) {
 
     debug(`waitForOrder: ${orderUrl}`);
 
-    return await promiseRetry({ times: 15, interval: 20000 }, async () => {
+    return await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
         debug('waitForOrder: getting status');
 
         const result = await this.postAsGet(orderUrl);
@@ -257,7 +257,7 @@ Acme2.prototype.waitForChallenge = async function (challenge) {
 
     debug('waitingForChallenge: %j', challenge);
 
-    await promiseRetry({ times: 15, interval: 20000 }, async () => {
+    await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
         debug('waitingForChallenge: getting status');
 
         const result = await this.postAsGet(challenge.url);
@@ -335,8 +335,8 @@ Acme2.prototype.downloadCertificate = async function (hostname, certUrl, certFil
     assert.strictEqual(typeof hostname, 'string');
     assert.strictEqual(typeof certUrl, 'string');
 
-    await promiseRetry({ times: 5, interval: 20000 }, async () => {
-        debug('downloadCertificate: downloading certificate');
+    await promiseRetry({ times: 5, interval: 20000, debug }, async () => {
+        debug(`downloadCertificate: downloading certificate of ${hostname}`);
 
         const result = await this.postAsGet(certUrl);
         if (result.statusCode === 202) throw new BoxError(BoxError.ACME_ERROR, 'Retry downloading certificate');
@@ -509,7 +509,7 @@ Acme2.prototype.acmeFlow = async function (hostname, domain, paths) {
 };
 
 Acme2.prototype.loadDirectory = async function () {
-    await promiseRetry({ times: 3, interval: 20000 }, async () => {
+    await promiseRetry({ times: 3, interval: 20000, debug }, async () => {
         const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true);
 
         if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching directory : ${response.status}`);
@@ -544,9 +544,8 @@ async function getCertificate(vhost, domain, paths, options) {
     assert.strictEqual(typeof paths, 'object');
     assert.strictEqual(typeof options, 'object');
 
-    let attempt = 1;
-    await promiseRetry({ times: 3, interval: 0 }, async function () {
-        debug(`getCertificate: attempt ${attempt++}`);
+    await promiseRetry({ times: 3, interval: 0, debug }, async function () {
+        debug(`getCertificate: for vhost ${vhost} and domain ${domain}`);
 
         const acme = new Acme2(options || { });
         return await acme.getCertificate(vhost, domain, paths);

src/apphealthmonitor.js

@@ -38,14 +38,14 @@ async function setHealth(app, health) {
             debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app: app });
         }
     } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
         if (lastHealth === apps.HEALTH_HEALTHY) {
             debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app: app });
         }
     } else {
         debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);

src/apps.js

@@ -56,7 +56,7 @@ exports = module.exports = {
     listBackups,
 
     getTask,
-    getLocalLogfilePaths,
+    getLogPaths,
     getLogs,
 
     getCertificate,
@@ -126,9 +126,10 @@ exports = module.exports = {
     HEALTH_DEAD: 'dead',
 
     // subdomain table types
-    SUBDOMAIN_TYPE_PRIMARY: 'primary',
-    SUBDOMAIN_TYPE_REDIRECT: 'redirect',
-    SUBDOMAIN_TYPE_ALIAS: 'alias',
+    LOCATION_TYPE_PRIMARY: 'primary',
+    LOCATION_TYPE_SECONDARY: 'secondary',
+    LOCATION_TYPE_REDIRECT: 'redirect',
+    LOCATION_TYPE_ALIAS: 'alias',
 
     // exported for testing
     _validatePortBindings: validatePortBindings,
@@ -186,7 +187,6 @@ const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationS
 
 // const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
 
-// validate the port bindings
 function validatePortBindings(portBindings, manifest) {
     assert.strictEqual(typeof portBindings, 'object');
     assert.strictEqual(typeof manifest, 'object');
@@ -233,13 +233,13 @@ function validatePortBindings(portBindings, manifest) {
     if (!portBindings) return null;
 
     for (let portName in portBindings) {
-        if (!/^[a-zA-Z0-9_]+$/.test(portName)) return new BoxError(BoxError.BAD_FIELD, `${portName} is not a valid environment variable`, { field: 'portBindings', portName: portName });
+        if (!/^[a-zA-Z0-9_]+$/.test(portName)) return new BoxError(BoxError.BAD_FIELD, `${portName} is not a valid environment variable in portBindings`);
 
         const hostPort = portBindings[portName];
-        if (!Number.isInteger(hostPort)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not an integer`, { field: 'portBindings', portName: portName });
-        if (RESERVED_PORTS.indexOf(hostPort) !== -1) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
-        if (RESERVED_PORT_RANGES.find(range => (hostPort >= range[0] && hostPort <= range[1]))) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
-        if (ALLOWED_PORTS.indexOf(hostPort) === -1 && (hostPort <= 1023 || hostPort > 65535)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not in permitted range`, { field: 'portBindings', portName: portName });
+        if (!Number.isInteger(hostPort)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not an integer in ${portName} portBindings`);
+        if (RESERVED_PORTS.indexOf(hostPort) !== -1) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} for ${portName} is reserved in portBindings`);
+        if (RESERVED_PORT_RANGES.find(range => (hostPort >= range[0] && hostPort <= range[1]))) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} for ${portName} is reserved in portBindings`);
+        if (ALLOWED_PORTS.indexOf(hostPort) === -1 && (hostPort <= 1023 || hostPort > 65535)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} for ${portName} is not in permitted range in portBindings`);
     }
 
     // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
@@ -247,7 +247,7 @@ function validatePortBindings(portBindings, manifest) {
     const tcpPorts = manifest.tcpPorts || { };
     const udpPorts = manifest.udpPorts || { };
     for (let portName in portBindings) {
-        if (!(portName in tcpPorts) && !(portName in udpPorts)) return new BoxError(BoxError.BAD_FIELD, `Invalid portBindings ${portName}`, { field: 'portBindings', portName: portName });
+        if (!(portName in tcpPorts) && !(portName in udpPorts)) return new BoxError(BoxError.BAD_FIELD, `Invalid portBindings ${portName}`);
     }
 
     return null;
@@ -269,6 +269,33 @@ function translatePortBindings(portBindings, manifest) {
     return result;
 }
 
+function validateSecondaryDomains(secondaryDomains, manifest) {
+    assert.strictEqual(typeof secondaryDomains, 'object');
+    assert.strictEqual(typeof manifest, 'object');
+
+    const httpPorts = manifest.httpPorts || {};
+
+    for (const envName in httpPorts) { // each httpPort is required
+        if (!(envName in secondaryDomains)) return new BoxError(BoxError.BAD_FIELD, `secondaryDomain ${envName} is required`);
+    }
+
+    for (const envName in secondaryDomains) {
+        if (!(envName in httpPorts)) return new BoxError(BoxError.BAD_FIELD, `secondaryDomain ${envName} is not listed in manifest`);
+    }
+
+    return null;
+}
+
+function translateSecondaryDomains(secondaryDomains) {
+    assert(secondaryDomains && typeof secondaryDomains === 'object');
+
+    const result = [];
+    for (const envName in secondaryDomains) {
+        result.push({ domain: secondaryDomains[envName].domain, subdomain: secondaryDomains[envName].subdomain, environmentVariable: envName });
+    }
+    return result;
+}
+
 function parseCrontab(crontab) {
     assert(crontab === null || typeof crontab === 'string');
 
@@ -337,8 +364,8 @@ function validateMemoryLimit(manifest, memoryLimit) {
     assert.strictEqual(typeof manifest, 'object');
     assert.strictEqual(typeof memoryLimit, 'number');
 
-    var min = manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
-    var max = os.totalmem() * 2; // this will overallocate since we don't allocate equal swap always (#466)
+    const min = manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+    const max = os.totalmem() * 2; // this will overallocate since we don't allocate equal swap always (#466)
 
     // allow 0, which indicates that it is not set, the one from the manifest will be choosen but we don't commit any user value
     // this is needed so an app update can change the value in the manifest, and if not set by the user, the new value should be used
@@ -375,7 +402,7 @@ function validateRobotsTxt(robotsTxt) {
     if (robotsTxt === null) return null;
 
     // this is the nginx limit on inline strings. if we really hit this, we have to generate a file
-    if (robotsTxt.length > 4096) return new BoxError(BoxError.BAD_FIELD, 'robotsTxt must be less than 4096', { field: 'robotsTxt' });
+    if (robotsTxt.length > 4096) return new BoxError(BoxError.BAD_FIELD, 'robotsTxt must be less than 4096');
 
     // TODO: validate the robots file? we escape the string when templating the nginx config right now
 
@@ -385,9 +412,9 @@ function validateRobotsTxt(robotsTxt) {
 function validateCsp(csp) {
     if (csp === null) return null;
 
-    if (csp.length > 4096) return new BoxError(BoxError.BAD_FIELD, 'CSP must be less than 4096', { field: 'csp' });
+    if (csp.length > 4096) return new BoxError(BoxError.BAD_FIELD, 'CSP must be less than 4096');
 
-    if (csp.includes('"')) return new BoxError(BoxError.BAD_FIELD, 'CSP cannot contains double quotes', { field: 'csp' });
+    if (csp.includes('"')) return new BoxError(BoxError.BAD_FIELD, 'CSP cannot contains double quotes');
 
     return null;
 }
@@ -403,25 +430,25 @@ function validateBackupFormat(format) {
 function validateLabel(label) {
     if (label === null) return null;
 
-    if (label.length > 128) return new BoxError(BoxError.BAD_FIELD, 'label must be less than 128', { field: 'label' });
+    if (label.length > 128) return new BoxError(BoxError.BAD_FIELD, 'label must be less than 128');
 
     return null;
 }
 
 function validateTags(tags) {
-    if (tags.length > 64) return new BoxError(BoxError.BAD_FIELD, 'Can only set up to 64 tags', { field: 'tags' });
+    if (tags.length > 64) return new BoxError(BoxError.BAD_FIELD, 'Can only set up to 64 tags');
 
-    if (tags.some(tag => tag.length == 0)) return new BoxError(BoxError.BAD_FIELD, 'tag cannot be empty', { field: 'tags' });
-    if (tags.some(tag => tag.length > 128)) return new BoxError(BoxError.BAD_FIELD, 'tag must be less than 128', { field: 'tags' });
+    if (tags.some(tag => tag.length == 0)) return new BoxError(BoxError.BAD_FIELD, 'tag cannot be empty');
+    if (tags.some(tag => tag.length > 128)) return new BoxError(BoxError.BAD_FIELD, 'tag must be less than 128');
 
     return null;
 }
 
 function validateEnv(env) {
     for (let key in env) {
-        if (key.length > 512) return new BoxError(BoxError.BAD_FIELD, 'Max env var key length is 512', { field: 'env', env: env });
+        if (key.length > 512) return new BoxError(BoxError.BAD_FIELD, 'Max env var key length is 512');
         // http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap08.html
-        if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) return new BoxError(BoxError.BAD_FIELD, `"${key}" is not a valid environment variable`, { field: 'env', env: env });
+        if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) return new BoxError(BoxError.BAD_FIELD, `"${key}" is not a valid environment variable`);
     }
 
     return null;
@@ -430,25 +457,25 @@ function validateEnv(env) {
 function validateDataDir(dataDir) {
     if (dataDir === null) return null;
 
-    if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`, { field: 'dataDir' });
-    if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`, { field: 'dataDir' });
-    if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`, { field: 'dataDir' });
+    if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`);
+    if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`);
+    if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`);
 
     // nfs shares will have the directory mounted already
     let stat = safe.fs.lstatSync(dataDir);
     if (stat) {
-        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`, { field: 'dataDir' });
+        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`);
         let entries = safe.fs.readdirSync(dataDir);
-        if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`, { field: 'dataDir' });
-        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`, { field: 'dataDir' });
+        if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`);
+        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`);
     }
 
     // backup logic relies on paths not overlapping (because it recurses)
-    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`, { field: 'dataDir' });
+    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`);
 
     // if we made it this far, it cannot start with any of these realistically
     const fhs = [ '/bin', '/boot', '/etc', '/lib', '/lib32', '/lib64', '/proc', '/run', '/sbin', '/tmp', '/usr' ];
-    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`, { field: 'dataDir' });
+    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`);
 
     return null;
 }
@@ -459,29 +486,29 @@ function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, port
     assert.strictEqual(typeof domainObjectMap, 'object');
     assert.strictEqual(typeof portBindings, 'object');
 
-    var match = errorMessage.match(/ER_DUP_ENTRY: Duplicate entry '(.*)' for key '(.*)'/);
+    const match = errorMessage.match(/ER_DUP_ENTRY: Duplicate entry '(.*)' for key '(.*)'/);
     if (!match) {
         debug('Unexpected SQL error message.', errorMessage);
         return new BoxError(BoxError.DATABASE_ERROR, errorMessage);
     }
 
     // check if a location conflicts
-    if (match[2] === 'subdomain') {
+    if (match[2] === 'locations.subdomain') {
         for (let i = 0; i < locations.length; i++) {
-            const { subdomain, domain } = locations[i];
+            const { subdomain, domain, type } = locations[i];
             if  (match[1] !== `${subdomain}-${domain}`) continue;
 
-            return new BoxError(BoxError.ALREADY_EXISTS, `Domain '${dns.fqdn(subdomain, domainObjectMap[domain])}' is in use`, { subdomain, domain });
+            return new BoxError(BoxError.ALREADY_EXISTS, `${type} location '${dns.fqdn(subdomain, domainObjectMap[domain])}' is in use`);
         }
     }
 
     // check if any of the port bindings conflict
-    for (let portName in portBindings) {
-        if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is in use`, { portName });
+    for (const portName in portBindings) {
+        if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is in use`);
     }
 
     if (match[2] === 'dataDir') {
-        return new BoxError(BoxError.BAD_FIELD, `Data directory ${match[1]} is in use`, { field: 'dataDir' });
+        return new BoxError(BoxError.BAD_FIELD, `Data directory ${match[1]} is in use`);
     }
 
     return new BoxError(BoxError.ALREADY_EXISTS, `${match[2]} '${match[1]}' is in use`);
@@ -496,18 +523,18 @@ function getDataDir(app, dataDir) {
 function removeInternalFields(app) {
     return _.pick(app,
         'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId',
-        'location', 'domain', 'fqdn', 'crontab',
+        'subdomain', 'domain', 'fqdn', 'crontab',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'operators',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'alternateDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts',
+        'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts',
         'enableMailbox', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
 }
 
 // non-admins can only see these
 function removeRestrictedFields(app) {
     return _.pick(app,
-        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'accessRestriction', 'alternateDomains', 'aliasDomains', 'sso',
-        'location', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup');
+        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'accessRestriction', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'sso',
+        'subdomain', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup');
 }
 
 async function getIcon(app, options) {
@@ -602,20 +629,28 @@ function postProcess(result) {
     result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
     delete result.servicesConfigJson;
 
-    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
+    const subdomains = JSON.parse(result.subdomains),
+        domains = JSON.parse(result.domains),
+        subdomainTypes = JSON.parse(result.subdomainTypes),
+        subdomainEnvironmentVariables = JSON.parse(result.subdomainEnvironmentVariables);
+
     delete result.subdomains;
     delete result.domains;
     delete result.subdomainTypes;
+    delete result.subdomainEnvironmentVariables;
 
-    result.alternateDomains = [];
+    result.secondaryDomains = [];
+    result.redirectDomains = [];
     result.aliasDomains = [];
     for (let i = 0; i < subdomainTypes.length; i++) {
-        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
-            result.location = subdomains[i];
+        if (subdomainTypes[i] === exports.LOCATION_TYPE_PRIMARY) {
+            result.subdomain = subdomains[i];
             result.domain = domains[i];
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
-            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
+        } else if (subdomainTypes[i] === exports.LOCATION_TYPE_SECONDARY) {
+            result.secondaryDomains.push({ domain: domains[i], subdomain: subdomains[i], environmentVariable: subdomainEnvironmentVariables[i] });
+        } else if (subdomainTypes[i] === exports.LOCATION_TYPE_REDIRECT) {
+            result.redirectDomains.push({ domain: domains[i], subdomain: subdomains[i] });
+        } else if (subdomainTypes[i] === exports.LOCATION_TYPE_ALIAS) {
             result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
         }
     }
@@ -651,8 +686,9 @@ function attachProperties(app, domainObjectMap) {
     }
     app.portBindings = result;
     app.iconUrl = app.hasIcon || app.hasAppStoreIcon  ? `/api/v1/apps/${app.id}/icon` : null;
-    app.fqdn = dns.fqdn(app.location, domainObjectMap[app.domain]);
-    app.alternateDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    app.fqdn = dns.fqdn(app.subdomain, domainObjectMap[app.domain]);
+    app.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    app.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
     app.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 }
 
@@ -694,12 +730,12 @@ function accessLevel(app, user) {
     return canAccess(app, user) ? 'user' : null;
 }
 
-async function add(id, appStoreId, manifest, location, domain, portBindings, data) {
+async function add(id, appStoreId, manifest, subdomain, domain, portBindings, data) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof manifest.version, 'string');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof portBindings, 'object');
     assert(data && typeof data === 'object');
@@ -737,8 +773,8 @@ async function add(id, appStoreId, manifest, location, domain, portBindings, dat
     });
 
     queries.push({
-        query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-        args: [ id, domain, location, exports.SUBDOMAIN_TYPE_PRIMARY ]
+        query: 'INSERT INTO locations (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+        args: [ id, domain, subdomain, exports.LOCATION_TYPE_PRIMARY ]
     });
 
     Object.keys(portBindings).forEach(function (env) {
@@ -755,11 +791,20 @@ async function add(id, appStoreId, manifest, location, domain, portBindings, dat
         });
     });
 
-    if (data.alternateDomains) {
-        data.alternateDomains.forEach(function (d) {
+    if (data.secondaryDomains) {
+        data.secondaryDomains.forEach(function (d) {
             queries.push({
-                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]
+                query: 'INSERT INTO locations (appId, domain, subdomain, type, environmentVariable) VALUES (?, ?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.LOCATION_TYPE_SECONDARY, d.environmentVariable ]
+            });
+        });
+    }
+
+    if (data.redirectDomains) {
+        data.redirectDomains.forEach(function (d) {
+            queries.push({
+                query: 'INSERT INTO locations (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.LOCATION_TYPE_REDIRECT ]
             });
         });
     }
@@ -767,8 +812,8 @@ async function add(id, appStoreId, manifest, location, domain, portBindings, dat
     if (data.aliasDomains) {
         data.aliasDomains.forEach(function (d) {
             queries.push({
-                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
+                query: 'INSERT INTO locations (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.LOCATION_TYPE_ALIAS ]
             });
         });
     }
@@ -793,7 +838,8 @@ async function updateWithConstraints(id, app, constraints) {
     assert.strictEqual(typeof constraints, 'string');
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
     assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
-    assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
+    assert(!('secondaryDomains' in app) || Array.isArray(app.secondaryDomains));
+    assert(!('redirectDomains' in app) || Array.isArray(app.redirectDomains));
     assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
     assert(!('tags' in app) || Array.isArray(app.tags));
     assert(!('env' in app) || typeof app.env === 'object');
@@ -801,11 +847,11 @@ async function updateWithConstraints(id, app, constraints) {
     const queries = [ ];
 
     if ('portBindings' in app) {
-        var portBindings = app.portBindings || { };
+        const portBindings = app.portBindings || { };
         // replace entries by app id
         queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
         Object.keys(portBindings).forEach(function (env) {
-            var values = [ portBindings[env].hostPort, portBindings[env].type, env, id ];
+            const values = [ portBindings[env].hostPort, portBindings[env].type, env, id ];
             queries.push({ query: 'INSERT INTO appPortBindings (hostPort, type, environmentVariable, appId) VALUES(?, ?, ?, ?)', args: values });
         });
     }
@@ -821,19 +867,25 @@ async function updateWithConstraints(id, app, constraints) {
         });
     }
 
-    if ('location' in app && 'domain' in app) { // must be updated together as they are unique together
-        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ]}); // all locations of an app must be updated together
-        queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, app.domain, app.location, exports.SUBDOMAIN_TYPE_PRIMARY ]});
+    if ('subdomain' in app && 'domain' in app) { // must be updated together as they are unique together
+        queries.push({ query: 'DELETE FROM locations WHERE appId = ?', args: [ id ]}); // all locations of an app must be updated together
+        queries.push({ query: 'INSERT INTO locations (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, app.domain, app.subdomain, exports.LOCATION_TYPE_PRIMARY ]});
 
-        if ('alternateDomains' in app) {
-            app.alternateDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
+        if ('secondaryDomains' in app) {
+            app.secondaryDomains.forEach(function (d) {
+                queries.push({ query: 'INSERT INTO locations (appId, domain, subdomain, type, environmentVariable) VALUES (?, ?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.LOCATION_TYPE_SECONDARY, d.environmentVariable ]});
+            });
+        }
+
+        if ('redirectDomains' in app) {
+            app.redirectDomains.forEach(function (d) {
+                queries.push({ query: 'INSERT INTO locations (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.LOCATION_TYPE_REDIRECT ]});
             });
         }
 
         if ('aliasDomains' in app) {
             app.aliasDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
+                queries.push({ query: 'INSERT INTO locations (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.LOCATION_TYPE_ALIAS ]});
             });
         }
     }
@@ -850,7 +902,7 @@ async function updateWithConstraints(id, app, constraints) {
         if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig' || p === 'operators') {
             fields.push(`${p}Json = ?`);
             values.push(JSON.stringify(app[p]));
-        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
+        } else if (p !== 'portBindings' && p !== 'subdomain' && p !== 'domain' && p !== 'secondaryDomains' && p !== 'redirectDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
             fields.push(p + ' = ?');
             values.push(app[p]);
         }
@@ -902,7 +954,7 @@ async function del(id) {
     assert.strictEqual(typeof id, 'string');
 
     const queries = [
-        { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM locations WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
@@ -923,7 +975,7 @@ async function delPortBinding(hostPort, type) {
 }
 
 async function clear() {
-    await database.query('DELETE FROM subdomains');
+    await database.query('DELETE FROM locations');
     await database.query('DELETE FROM appPortBindings');
     await database.query('DELETE FROM appAddonConfigs');
     await database.query('DELETE FROM appEnvVars');
@@ -940,9 +992,9 @@ async function getDomainObjectMap() {
 // each query simply join apps table with another table by id. we then join the full result together
 const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
 const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
-const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
+const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(locations.subdomain) AS subdomains, JSON_ARRAYAGG(locations.domain) AS domains, JSON_ARRAYAGG(locations.type) AS subdomainTypes, JSON_ARRAYAGG(locations.environmentVariable) AS subdomainEnvironmentVariables FROM apps LEFT JOIN locations ON apps.id = locations.appId GROUP BY apps.id';
 const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
-const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
+const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, subdomainEnvironmentVariables, volumeIds, volumeReadOnlys FROM apps`
     + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
     + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
     + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
@@ -1029,8 +1081,8 @@ async function downloadManifest(appStoreId, manifest) {
     return { appStoreId: parts[0], manifest: response.body.manifest };
 }
 
-function mailboxNameForLocation(location, manifest) {
-    if (location) return `${location}.app`;
+function mailboxNameForSubdomain(subdomain, manifest) {
+    if (subdomain) return `${subdomain}.app`;
     if (manifest.title) return manifest.title.toLowerCase().replace(/[^a-zA-Z0-9]/g, '') + '.app';
     return 'noreply.app';
 }
@@ -1154,16 +1206,16 @@ async function validateLocations(locations) {
     const domainObjectMap = await getDomainObjectMap();
 
     for (let location of locations) {
-        if (!(location.domain in domainObjectMap)) throw new BoxError(BoxError.BAD_FIELD, 'No such domain', { field: 'location', domain: location.domain, subdomain: location.subdomain });
+        if (!(location.domain in domainObjectMap)) throw new BoxError(BoxError.BAD_FIELD, `No such domain in ${location.type} location`);
 
         let subdomain = location.subdomain;
-        if (location.type === 'alias' && subdomain.startsWith('*')) {
+        if (location.type === exports.LOCATION_TYPE_ALIAS && subdomain.startsWith('*')) {
             if (subdomain === '*') continue;
             subdomain = subdomain.replace(/^\*\./, ''); // remove *.
         }
 
         const error = dns.validateHostname(subdomain, domainObjectMap[location.domain]);
-        if (error) throw new BoxError(BoxError.BAD_FIELD, 'Bad location: ' + error.message, { field: 'location', domain: location.domain, subdomain: location.subdomain });
+        if (error) throw new BoxError(BoxError.BAD_FIELD, `Bad ${location.type} location: ${error.message}`);
     }
 
     return domainObjectMap;
@@ -1180,7 +1232,7 @@ async function install(data, auditSource) {
 
     assert.strictEqual(typeof data.manifest, 'object'); // manifest is already downloaded
 
-    const location = data.location.toLowerCase(),
+    const subdomain = data.subdomain.toLowerCase(),
         domain = data.domain.toLowerCase(),
         portBindings = data.portBindings || null,
         accessRestriction = data.accessRestriction || null,
@@ -1188,7 +1240,7 @@ async function install(data, auditSource) {
         debugMode = data.debugMode || null,
         enableBackup = 'enableBackup' in data ? data.enableBackup : true,
         enableAutomaticUpdate = 'enableAutomaticUpdate' in data ? data.enableAutomaticUpdate : true,
-        alternateDomains = data.alternateDomains || [],
+        redirectDomains = data.redirectDomains || [],
         aliasDomains = data.aliasDomains || [],
         env = data.env || {},
         label = data.label || null,
@@ -1222,6 +1274,10 @@ async function install(data, auditSource) {
     error = validateTags(tags);
     if (error) throw error;
 
+    error = validateSecondaryDomains(data.secondaryDomains || {}, manifest);
+    if (error) throw error;
+    const secondaryDomains = translateSecondaryDomains(data.secondaryDomains || {});
+
     let sso = 'sso' in data ? data.sso : null;
     if ('sso' in data && !('optionalSso' in manifest)) throw new BoxError(BoxError.BAD_FIELD, 'sso can only be specified for apps with optionalSso');
     // if sso was unspecified, enable it by default if possible
@@ -1234,18 +1290,19 @@ async function install(data, auditSource) {
 
     // sendmail is enabled by default
     const enableMailbox = 'enableMailbox' in data ? data.enableMailbox : true;
-    const mailboxName = manifest.addons?.sendmail ? mailboxNameForLocation(location, manifest) : null;
+    const mailboxName = manifest.addons?.sendmail ? mailboxNameForSubdomain(subdomain, manifest) : null;
     const mailboxDomain = manifest.addons?.sendmail ? domain : null;
 
     let icon = data.icon || null;
     if (icon) {
-        if (!validator.isBase64(icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' });
+        if (!validator.isBase64(icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
         icon = Buffer.from(icon, 'base64');
     }
 
-    const locations = [{ subdomain: location, domain, type: 'primary' }]
-        .concat(alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
-        .concat(aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));
+    const locations = [{ subdomain: subdomain, domain, type: exports.LOCATION_TYPE_PRIMARY }]
+        .concat(secondaryDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_SECONDARY })))
+        .concat(redirectDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_REDIRECT })))
+        .concat(aliasDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_ALIAS })));
 
     const domainObjectMap = await validateLocations(locations);
 
@@ -1263,7 +1320,8 @@ async function install(data, auditSource) {
         mailboxDomain,
         enableBackup,
         enableAutomaticUpdate,
-        alternateDomains,
+        secondaryDomains,
+        redirectDomains,
         aliasDomains,
         env,
         label,
@@ -1274,7 +1332,7 @@ async function install(data, auditSource) {
         installationState: exports.ISTATE_PENDING_INSTALL
     };
 
-    const [addError] = await safe(add(appId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), app));
+    const [addError] = await safe(add(appId, appStoreId, manifest, subdomain, domain, translatePortBindings(portBindings, manifest), app));
     if (addError && addError.reason === BoxError.ALREADY_EXISTS) throw getDuplicateErrorDetails(addError.message, locations, domainObjectMap, portBindings);
     if (addError) throw addError;
 
@@ -1288,9 +1346,10 @@ async function install(data, auditSource) {
 
     const taskId = await addTask(appId, app.installationState, task, auditSource);
 
-    const newApp = _.extend({}, _.omit(app, 'icon'), { appStoreId, manifest, location, domain, portBindings });
-    newApp.fqdn = dns.fqdn(newApp.location, domainObjectMap[newApp.domain]);
-    newApp.alternateDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    const newApp = _.extend({}, _.omit(app, 'icon'), { appStoreId, manifest, subdomain, domain, portBindings });
+    newApp.fqdn = dns.fqdn(newApp.subdomain, domainObjectMap[newApp.domain]);
+    newApp.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    newApp.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
     newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
     await eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId, app: newApp, taskId });
@@ -1370,7 +1429,7 @@ async function setIcon(app, icon, auditSource) {
     const appId = app.id;
 
     if (icon) {
-        if (!validator.isBase64(icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' });
+        if (!validator.isBase64(icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
         icon = Buffer.from(icon, 'base64');
     }
 
@@ -1517,9 +1576,9 @@ async function setMailbox(app, data, auditSource) {
 
         if (mailboxName) {
             error = mail.validateName(mailboxName);
-            if (error) throw new BoxError(BoxError.BAD_FIELD, error.message, { field: 'mailboxName' });
+            if (error) throw new BoxError(BoxError.BAD_FIELD, error.message);
         } else {
-            mailboxName = mailboxNameForLocation(app.location, app.domain, app.manifest);
+            mailboxName = mailboxNameForSubdomain(app.subdomain, app.domain, app.manifest);
         }
     }
 
@@ -1555,7 +1614,7 @@ async function setInbox(app, data, auditSource) {
         if (!domain.enabled) throw new BoxError(BoxError.BAD_FIELD, 'Domain does not have incoming email enabled');
 
         error = mail.validateName(data.inboxName);
-        if (error) throw new BoxError(BoxError.BAD_FIELD, error.message, { field: 'inboxName' });
+        if (error) throw new BoxError(BoxError.BAD_FIELD, error.message);
     }
 
     const task = {
@@ -1638,12 +1697,13 @@ async function setLocation(app, data, auditSource) {
     let error = checkAppState(app, exports.ISTATE_PENDING_LOCATION_CHANGE);
     if (error) throw error;
 
-    let values = {
-        location: data.location.toLowerCase(),
+    const values = {
+        subdomain: data.subdomain.toLowerCase(),
         domain: data.domain.toLowerCase(),
         // these are intentionally reset, if not set
         portBindings: null,
-        alternateDomains: [],
+        secondaryDomains: [],
+        redirectDomains: [],
         aliasDomains: []
     };
 
@@ -1655,39 +1715,45 @@ async function setLocation(app, data, auditSource) {
     }
 
     // rename the auto-created mailbox to match the new location
-    if (app.manifest.addons?.sendmail && app.mailboxName.endsWith('.app')) {
-        values.mailboxName = mailboxNameForLocation(values.location, app.manifest);
+    if (app.manifest.addons?.sendmail && app.mailboxName?.endsWith('.app')) {
+        values.mailboxName = mailboxNameForSubdomain(values.subdomain, app.manifest);
         values.mailboxDomain = values.domain;
     }
 
-    if ('alternateDomains' in data) {
-        values.alternateDomains = data.alternateDomains;
+    error = validateSecondaryDomains(data.secondaryDomains || {}, app.manifest);
+    if (error) throw error;
+    values.secondaryDomains = translateSecondaryDomains(data.secondaryDomains || {});
+
+    if ('redirectDomains' in data) {
+        values.redirectDomains = data.redirectDomains;
     }
 
     if ('aliasDomains' in data) {
         values.aliasDomains = data.aliasDomains;
     }
 
-    const locations = [{ subdomain: values.location, domain: values.domain, type: 'primary' }]
-        .concat(values.alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
-        .concat(values.aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));
+    const locations = [{ subdomain: values.subdomain, domain: values.domain, type: exports.LOCATION_TYPE_PRIMARY }]
+        .concat(values.secondaryDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_SECONDARY })))
+        .concat(values.redirectDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_REDIRECT })))
+        .concat(values.aliasDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_ALIAS })));
 
     const domainObjectMap = await validateLocations(locations);
 
     const task = {
         args: {
-            oldConfig: _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'aliasDomains', 'portBindings'),
+            oldConfig: _.pick(app, 'subdomain', 'domain', 'fqdn', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'portBindings'),
             skipDnsSetup: !!data.skipDnsSetup,
             overwriteDns: !!data.overwriteDns
         },
         values
     };
     let [taskError, taskId] = await safe(addTask(appId, exports.ISTATE_PENDING_LOCATION_CHANGE, task, auditSource));
-    if (taskError && taskError.reason === BoxError.ALREADY_EXISTS) taskError = getDuplicateErrorDetails(error.message, locations, domainObjectMap, data.portBindings);
+    if (taskError && taskError.reason === BoxError.ALREADY_EXISTS) taskError = getDuplicateErrorDetails(taskError.message, locations, domainObjectMap, data.portBindings);
     if (taskError) throw taskError;
 
-    values.fqdn = dns.fqdn(values.location, domainObjectMap[values.domain]);
-    values.alternateDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    values.fqdn = dns.fqdn(values.subdomain, domainObjectMap[values.domain]);
+    values.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    values.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
     values.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
     await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, _.extend({ appId, app, taskId }, values));
@@ -1759,7 +1825,7 @@ async function updateApp(app, data, auditSource) {
 
     if ('icon' in data) {
         if (data.icon) {
-            if (!validator.isBase64(data.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' });
+            if (!validator.isBase64(data.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
             data.icon = Buffer.from(data.icon, 'base64');
         }
         values.icon = data.icon;
@@ -1777,7 +1843,7 @@ async function updateApp(app, data, auditSource) {
     if (!manifest.addons?.sendmail) { // clear if the update removed addon
         values.mailboxName = values.mailboxDomain = null;
     } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since update added the addon
-        values.mailboxName = mailboxNameForLocation(app.location, manifest);
+        values.mailboxName = mailboxNameForSubdomain(app.subdomain, manifest);
         values.mailboxDomain = app.domain;
     }
 
@@ -1792,16 +1858,32 @@ async function updateApp(app, data, auditSource) {
     return { taskId };
 }
 
-function getLocalLogfilePaths(app) {
+async function getLogPaths(app) {
     assert.strictEqual(typeof app, 'object');
 
     const appId = app.id;
 
-    var filePaths = [];
+    const filePaths = [];
     filePaths.push(path.join(paths.LOG_DIR, appId, 'apptask.log'));
     filePaths.push(path.join(paths.LOG_DIR, appId, 'app.log'));
     if (app.manifest.addons && app.manifest.addons.redis) filePaths.push(path.join(paths.LOG_DIR, `redis-${appId}/app.log`));
 
+    if (app.manifest.logPaths) {
+        const [error, result] = await safe(docker.inspect(app.containerId));
+        if (!error) {
+            const runVolume = result.Mounts.find(function (mount) { return mount.Destination === '/run'; });
+            const tmpVolume = result.Mounts.find(function (mount) { return mount.Destination === '/tmp'; });
+            const dataVolume = result.Mounts.find(function (mount) { return mount.Destination === '/app/data'; });
+
+            // note: wild cards are not supported yet in logPaths since that will require shell expansion
+            for (const logPath of app.manifest.logPaths) {
+                if (logPath.startsWith('/tmp/')) filePaths.push(`${tmpVolume.Source}/${logPath.slice('/tmp/'.length)}`);
+                else if (logPath.startsWith('/run/')) filePaths.push(`${runVolume.Source}/${logPath.slice('/run/'.length)}`);
+                else if (logPath.startsWith('/app/data/')) filePaths.push(`${dataVolume.Source}/${logPath.slice('/app/data/'.length)}`);
+            }
+        }
+    }
+
     return filePaths;
 }
 
@@ -1824,15 +1906,16 @@ async function getLogs(app, options) {
     const args = [ '--lines=' + lines ];
     if (follow) args.push('--follow', '--retry', '--quiet'); // same as -F. to make it work if file doesn't exist, --quiet to not output file headers, which are no logs
 
-    const cp = spawn('/usr/bin/tail', args.concat(getLocalLogfilePaths(app)));
+    const logPaths = await getLogPaths(app);
+    const cp = spawn('/usr/bin/tail', args.concat(logPaths));
 
     const transformStream = split(function mapper(line) {
         if (format !== 'json') return line + '\n';
 
-        var data = line.split(' '); // logs are <ISOtimestamp> <msg>
-        var timestamp = (new Date(data[0])).getTime();
+        const data = line.split(' '); // logs are <ISOtimestamp> <msg>
+        let timestamp = (new Date(data[0])).getTime();
         if (isNaN(timestamp)) timestamp = 0;
-        var message = line.slice(data[0].length+1);
+        const message = line.slice(data[0].length+1);
 
         // ignore faulty empty logs
         if (!timestamp && !message) return;
@@ -1855,7 +1938,7 @@ async function getCertificate(subdomain, domain) {
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
 
-    const result = await database.query('SELECT certificateJson FROM subdomains WHERE subdomain=? AND domain=?', [ subdomain, domain ]);
+    const result = await database.query('SELECT certificateJson FROM locations WHERE subdomain=? AND domain=?', [ subdomain, domain ]);
     if (result.length === 0) return null;
     return JSON.parse(result[0].certificateJson);
 }
@@ -1889,7 +1972,7 @@ async function repair(app, data, auditSource) {
             if (!data.manifest.addons?.sendmail) { // clear if repair removed addon
                 task.values.mailboxName = task.values.mailboxDomain = null;
             } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since repair added the addon
-                task.values.mailboxName = mailboxNameForLocation(app.location, data.manifest);
+                task.values.mailboxName = mailboxNameForSubdomain(app.subdomain, data.manifest);
                 task.values.mailboxDomain = app.domain;
             }
 
@@ -1935,7 +2018,7 @@ async function restore(app, backupId, auditSource) {
     if (!backupInfo.manifest.addons?.sendmail) { // clear if restore removed addon
         values.mailboxName = values.mailboxDomain = null;
     } else if (!app.mailboxName || app.mailboxName.endsWith('.app')) { // allocate since restore added the addon
-        values.mailboxName = mailboxNameForLocation(app.location, backupInfo.manifest);
+        values.mailboxName = mailboxNameForSubdomain(app.subdomain, backupInfo.manifest);
         values.mailboxDomain = app.domain;
     }
 
@@ -1979,14 +2062,13 @@ async function importApp(app, data, auditSource) {
 
     // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
     if (backupConfig) {
-        if (mounts.isMountProvider(backupConfig.provider)) {
-            backupConfig.mountPoint = `/mnt/appimport-${app.id}`;
+        if (mounts.isManagedProvider(backupConfig.provider)) {
             error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions);
             if (error) throw error;
 
             const mountObject = { // keep this in sync with the import code in apptask
                 name: `appimport-${app.id}`,
-                hostPath: backupConfig.mountPoint,
+                hostPath: `/mnt/appimport-${app.id}`,
                 mountType: backupConfig.provider,
                 mountOptions: backupConfig.mountOptions
             };
@@ -2060,7 +2142,7 @@ async function clone(app, data, user, auditSource) {
     assert(user && typeof user === 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
-    const location = data.location.toLowerCase(),
+    const subdomain = data.subdomain.toLowerCase(),
         domain = data.domain.toLowerCase(),
         portBindings = data.portBindings || null,
         backupId = data.backupId,
@@ -2069,12 +2151,10 @@ async function clone(app, data, user, auditSource) {
         appId = app.id;
 
     assert.strictEqual(typeof backupId, 'string');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof portBindings, 'object');
 
-    const locations = [{ subdomain: location, domain, type: 'primary' }];
-    const domainObjectMap = await validateLocations(locations);
     const backupInfo = await backups.get(backupId);
 
     if (!backupInfo.manifest) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Could not get restore config');
@@ -2082,15 +2162,24 @@ async function clone(app, data, user, auditSource) {
 
     const manifest = backupInfo.manifest, appStoreId = app.appStoreId;
 
+    let error = validateSecondaryDomains(data.secondaryDomains || {}, manifest);
+    if (error) throw error;
+    const secondaryDomains = translateSecondaryDomains(data.secondaryDomains || {});
+
+    const locations = [{ subdomain: subdomain, domain, type: exports.LOCATION_TYPE_PRIMARY }]
+        .concat(secondaryDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_SECONDARY })));
+
+    const domainObjectMap = await validateLocations(locations);
+
     // re-validate because this new box version may not accept old configs
-    let error = checkManifestConstraints(manifest);
+    error = checkManifestConstraints(manifest);
     if (error) throw error;
 
     error = validatePortBindings(portBindings, manifest);
     if (error) throw error;
 
     // should we copy the original app's mailbox settings instead?
-    const mailboxName = manifest.addons?.sendmail ? mailboxNameForLocation(location, manifest) : null;
+    const mailboxName = manifest.addons?.sendmail ? mailboxNameForSubdomain(subdomain, manifest) : null;
     const mailboxDomain = manifest.addons?.sendmail ? domain : null;
 
     const newAppId = uuid.v4();
@@ -2104,12 +2193,13 @@ async function clone(app, data, user, auditSource) {
         cpuShares: app.cpuShares,
         accessRestriction: app.accessRestriction,
         sso: !!app.sso,
-        mailboxName: mailboxName,
-        mailboxDomain: mailboxDomain,
+        mailboxName,
+        mailboxDomain,
         enableBackup: app.enableBackup,
         reverseProxyConfig: app.reverseProxyConfig,
         env: app.env,
-        alternateDomains: [],
+        secondaryDomains,
+        redirectDomains: [],
         aliasDomains: [],
         servicesConfig: app.servicesConfig,
         label: app.label ? `${app.label}-clone` : '',
@@ -2119,7 +2209,7 @@ async function clone(app, data, user, auditSource) {
         enableMailbox: app.enableMailbox
     };
 
-    const [addError] = await safe(add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), obj));
+    const [addError] = await safe(add(newAppId, appStoreId, manifest, subdomain, domain, translatePortBindings(portBindings, manifest), obj));
     if (addError && addError.reason === BoxError.ALREADY_EXISTS) throw getDuplicateErrorDetails(addError.message, locations, domainObjectMap, portBindings);
     if (addError) throw addError;
 
@@ -2133,9 +2223,10 @@ async function clone(app, data, user, auditSource) {
     };
     const taskId = await addTask(newAppId, exports.ISTATE_PENDING_CLONE, task, auditSource);
 
-    const newApp = _.extend({}, _.omit(obj, 'icon'), { appStoreId, manifest, location, domain, portBindings });
-    newApp.fqdn = dns.fqdn(newApp.location, domainObjectMap[newApp.domain]);
-    newApp.alternateDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    const newApp = _.extend({}, _.omit(obj, 'icon'), { appStoreId, manifest, subdomain, domain, portBindings });
+    newApp.fqdn = dns.fqdn(newApp.subdomain, domainObjectMap[newApp.domain]);
+    newApp.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    newApp.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
     newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
     await eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId, oldApp: app, newApp, taskId });
@@ -2222,7 +2313,7 @@ async function restart(app, auditSource) {
 function checkManifestConstraints(manifest) {
     assert(manifest && typeof manifest === 'object');
 
-    if (manifest.manifestVersion > 2) return new BoxError(BoxError.BAD_FIELD, 'Manifest version must be <= 2');
+    if (manifest.manifestVersion !== 2) return new BoxError(BoxError.BAD_FIELD, 'Manifest version must be 2');
 
     if (!manifest.dockerImage) return new BoxError(BoxError.BAD_FIELD, 'Missing dockerImage'); // dockerImage is optional in manifest
 
@@ -2522,12 +2613,12 @@ async function downloadFile(app, filePath) {
             for (;;) {
                 if (this._buffer.length < 8) break; // header is 8 bytes
 
-                var type = this._buffer.readUInt8(0);
-                var len = this._buffer.readUInt32BE(4);
+                const type = this._buffer.readUInt8(0);
+                const len = this._buffer.readUInt32BE(4);
 
                 if (this._buffer.length < (8 + len)) break; // not enough
 
-                var payload = this._buffer.slice(8, 8 + len);
+                const payload = this._buffer.slice(8, 8 + len);
 
                 this._buffer = this._buffer.slice(8+len); // consumed
 

src/appstore.js

@@ -49,7 +49,7 @@ let gFeatures = {
     privateDockerRegistry: false,
     branding: false,
     support: false,
-    directoryConfig: false,
+    profileConfig: false,
     mailboxMaxCount: 5,
     emailPremium: false
 };
@@ -393,10 +393,11 @@ async function createTicket(info, auditSource) {
     if (info.app) {
         request.field('infoJSON', JSON.stringify(info));
 
-        apps.getLocalLogfilePaths(info.app).forEach(function (filePath) {
-            const logs = safe.child_process.execSync(`tail --lines=1000 ${filePath}`);
-            if (logs) request.attach(path.basename(filePath), logs, path.basename(filePath));
-        });
+        const logPaths = await apps.getLogPaths(info.app);
+        for (const logPath of logPaths) {
+            const logs = safe.child_process.execSync(`tail --lines=1000 ${logPath}`);
+            if (logs) request.attach(path.basename(logPath), logs, path.basename(logPath));
+        }
     } else {
         request.send(info);
     }
@@ -407,7 +408,7 @@ async function createTicket(info, auditSource) {
     if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+    await eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
 
     return { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` };
 }

src/apptask.js

@@ -71,7 +71,7 @@ async function updateApp(app, values) {
 async function allocateContainerIp(app) {
     assert.strictEqual(typeof app, 'object');
 
-    await promiseRetry({ times: 10, interval: 0}, async function () {
+    await promiseRetry({ times: 10, interval: 0, debug }, async function () {
         const iprange = iputils.intFromIp('172.18.20.255') - iputils.intFromIp('172.18.16.1');
         let rnd = Math.floor(Math.random() * iprange);
         const containerIp = iputils.ipFromInt(iputils.intFromIp('172.18.16.1') + rnd);
@@ -147,7 +147,7 @@ async function deleteAppDir(app, options) {
                 if (safe.error.code !== 'ENOENT') throw new BoxError(BoxError.FS_ERROR, `Error unlinking dir ${appDataDir} : ${safe.error.message}`);
             }
         } else {
-            if (!safe.fs.rmdirSync(appDataDir)) {
+            if (!safe.fs.rmSync(appDataDir, { recursive: true })) {
                 if (safe.error.code !== 'ENOENT') throw new BoxError(BoxError.FS_ERROR, `Error removing dir ${appDataDir} : ${safe.error.message}`);
             }
         }
@@ -205,10 +205,10 @@ async function verifyManifest(manifest) {
     assert.strictEqual(typeof manifest, 'object');
 
     let error = manifestFormat.parse(manifest);
-    if (error) throw new BoxError(BoxError.BAD_FIELD, `Manifest error: ${error.message}`, { field: 'manifest' });
+    if (error) throw new BoxError(BoxError.BAD_FIELD, `Manifest error: ${error.message}`);
 
     error = apps.checkManifestConstraints(manifest);
-    if (error) throw new BoxError(BoxError.CONFLICT, `Manifest constraint check failed: ${error.message}`, { field: 'manifest' });
+    if (error) throw new BoxError(BoxError.CONFLICT, `Manifest constraint check failed: ${error.message}`);
 }
 
 async function downloadIcon(app) {
@@ -221,7 +221,7 @@ async function downloadIcon(app) {
 
     const iconUrl = settings.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';
 
-    await promiseRetry({ times: 10, interval: 5000 }, async function () {
+    await promiseRetry({ times: 10, interval: 5000, debug }, async function () {
         const [networkError, response] = await safe(superagent.get(iconUrl)
             .buffer(true)
             .timeout(30 * 1000)
@@ -242,14 +242,26 @@ async function waitForDnsPropagation(app) {
         return;
     }
 
-    const ip = await sysinfo.getServerIp();
-    const [error] = await safe(dns.waitForDnsRecord(app.location, app.domain, 'A', ip, { times: 240 }));
-    if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: app.location, domain: app.domain });
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();
 
-    // now wait for alternateDomains and aliasDomains, if any
-    for (const domain of app.alternateDomains.concat(app.aliasDomains)) {
-        const [error] = await safe(dns.waitForDnsRecord(domain.subdomain, domain.domain, 'A', ip, { times: 240 }));
-        if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: domain.subdomain, domain: domain.domain });
+    let error;
+    [error] = await safe(dns.waitForDnsRecord(app.subdomain, app.domain, 'A', ipv4, { times: 240 }));
+    if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS A Record is not synced yet: ${error.message}`, { ipv4, subdomain: app.subdomain, domain: app.domain });
+    if (ipv6) {
+        [error] = await safe(dns.waitForDnsRecord(app.subdomain, app.domain, 'AAAA', ipv6, { times: 240 }));
+        if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS AAAA Record is not synced yet: ${error.message}`, { ipv6, subdomain: app.subdomain, domain: app.domain });
+    }
+
+    // now wait for redirectDomains and aliasDomains, if any
+    const allDomains = app.secondaryDomains.concat(app.redirectDomains).concat(app.aliasDomains);
+    for (const domain of allDomains) {
+        [error] = await safe(dns.waitForDnsRecord(domain.subdomain, domain.domain, 'A', ipv4, { times: 240 }));
+        if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS A Record is not synced yet: ${error.message}`, { ipv4, subdomain: domain.subdomain, domain: domain.domain });
+        if (ipv6) {
+            [error] = await safe(dns.waitForDnsRecord(domain.subdomain, domain.domain, 'AAAA', ipv6, { times: 240 }));
+            if (error) throw new BoxError(BoxError.DNS_ERROR, `DNS AAAA Record is not synced yet: ${error.message}`, { ipv6, subdomain: domain.subdomain, domain: domain.domain });
+        }
     }
 }
 
@@ -333,7 +345,7 @@ async function install(app, args, progressCallback) {
     if (!skipDnsSetup) {
         await progressCallback({ percent: 30, message: 'Registering subdomains' });
 
-        await dns.registerLocations([ { subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback);
+        await dns.registerLocations([ { subdomain: app.subdomain, domain: app.domain }].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback);
     }
 
     await progressCallback({ percent: 40, message: 'Downloading image' });
@@ -357,7 +369,7 @@ async function install(app, args, progressCallback) {
         await services.clearAddons(app, app.manifest.addons);
         const backupConfig = restoreConfig.backupConfig; // can be null
         let mountObject = null;
-        if (backupConfig && mounts.isMountProvider(backupConfig.provider)) {
+        if (backupConfig && mounts.isManagedProvider(backupConfig.provider)) {
             await progressCallback({ percent: 70, message: 'Setting up mount for importing' });
             mountObject = {  // keep this in sync with importApp in apps.js
                 name:  `appimport-${app.id}`,
@@ -449,9 +461,19 @@ async function changeLocation(app, args, progressCallback) {
     await deleteContainers(app, { managedOnly: true });
 
     // unregister old domains
-    let obsoleteDomains = oldConfig.alternateDomains.filter(function (o) {
-        return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
-    });
+    let obsoleteDomains = [];
+
+    if (oldConfig.secondaryDomains) {
+        obsoleteDomains = obsoleteDomains.concat(oldConfig.secondaryDomains.filter(function (o) {
+            return !app.secondaryDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
+        }));
+    }
+
+    if (oldConfig.redirectDomains) {
+        obsoleteDomains = obsoleteDomains.concat(oldConfig.redirectDomains.filter(function (o) {
+            return !app.redirectDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
+        }));
+    }
 
     if (oldConfig.aliasDomains) {
         obsoleteDomains = obsoleteDomains.concat(oldConfig.aliasDomains.filter(function (o) {
@@ -459,14 +481,14 @@ async function changeLocation(app, args, progressCallback) {
         }));
     }
 
-    if (locationChanged) obsoleteDomains.push({ subdomain: oldConfig.location, domain: oldConfig.domain });
+    if (locationChanged) obsoleteDomains.push({ subdomain: oldConfig.subdomain, domain: oldConfig.domain });
 
     if (obsoleteDomains.length !== 0) await dns.unregisterLocations(obsoleteDomains, progressCallback);
 
     // setup dns
     if (!skipDnsSetup) {
         await progressCallback({ percent: 30, message: 'Registering subdomains' });
-        await dns.registerLocations([ { subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback);
+        await dns.registerLocations([ { subdomain: app.subdomain, domain: app.domain }].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback);
     }
 
     // re-setup addons since they rely on the app's fqdn (e.g oauth)
@@ -615,7 +637,7 @@ async function update(app, args, progressCallback) {
         delete app.portBindings[portName];
     }
 
-    await updateApp(app, _.pick(updateConfig, 'manifest', 'appStoreId', 'memoryLimit')), // switch over to the new config
+    await updateApp(app, _.pick(updateConfig, 'manifest', 'appStoreId', 'memoryLimit')); // switch over to the new config
 
     await progressCallback({ percent: 45, message: 'Downloading icon' });
     await downloadIcon(app);
@@ -665,6 +687,7 @@ async function stop(app, args, progressCallback) {
     assert.strictEqual(typeof progressCallback, 'function');
 
     await progressCallback({ percent: 20, message: 'Stopping container' });
+    await reverseProxy.unconfigureApp(app); // removing nginx configs also means that we can auto-cleanup old certs since they are not referenced
     await docker.stopContainers(app.id);
 
     await progressCallback({ percent: 50, message: 'Stopping app services' });
@@ -710,7 +733,7 @@ async function uninstall(app, args, progressCallback) {
     await docker.deleteImage(app.manifest);
 
     await progressCallback({ percent: 70, message: 'Unregistering domains' });
-    await dns.unregisterLocations([ { subdomain: app.location, domain: app.domain } ].concat(app.alternateDomains).concat(app.aliasDomains), progressCallback);
+    await dns.unregisterLocations([ { subdomain: app.subdomain, domain: app.domain } ].concat(app.secondaryDomains).concat(app.redirectDomains).concat(app.aliasDomains), progressCallback);
 
     await progressCallback({ percent: 90, message: 'Cleanup logs' });
     await cleanupLogs(app);

src/backups.js

@@ -268,15 +268,15 @@ async function startCleanupTask(auditSource) {
 
     const taskId = await tasks.add(tasks.TASK_CLEAN_BACKUPS, []);
 
-    tasks.startTask(taskId, {}, (error, result) => { // result is { removedBoxBackupIds, removedAppBackupIds, removedMailBackupIds, missingBackupIds }
-        eventlog.add(eventlog.ACTION_BACKUP_CLEANUP_FINISH, auditSource, {
+    tasks.startTask(taskId, {}, async (error, result) => { // result is { removedBoxBackupIds, removedAppBackupIds, removedMailBackupIds, missingBackupIds }
+        await safe(eventlog.add(eventlog.ACTION_BACKUP_CLEANUP_FINISH, auditSource, {
             taskId,
             errorMessage: error ? error.message : null,
             removedBoxBackupIds: result ? result.removedBoxBackupIds : [],
             removedMailBackupIds: result ? result.removedMailBackupIds : [],
             removedAppBackupIds: result ? result.removedAppBackupIds : [],
             missingBackupIds: result ? result.missingBackupIds : []
-        });
+        }), { debug });
     });
 
     return taskId;
@@ -297,26 +297,26 @@ async function testConfig(backupConfig) {
     assert.strictEqual(typeof backupConfig, 'object');
 
     const func = storage.api(backupConfig.provider);
-    if (!func) return new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' });
+    if (!func) return new BoxError(BoxError.BAD_FIELD, 'unknown storage provider');
 
-    if (backupConfig.format !== 'tgz' && backupConfig.format !== 'rsync') return new BoxError(BoxError.BAD_FIELD, 'unknown format', { field: 'format' });
+    if (backupConfig.format !== 'tgz' && backupConfig.format !== 'rsync') return new BoxError(BoxError.BAD_FIELD, 'unknown format');
 
     const job = safe.safeCall(function () { return new CronJob(backupConfig.schedulePattern); });
-    if (!job) return new BoxError(BoxError.BAD_FIELD, 'Invalid schedule pattern', { field: 'schedulePattern' });
+    if (!job) return new BoxError(BoxError.BAD_FIELD, 'Invalid schedule pattern');
 
     if ('password' in backupConfig) {
-        if (typeof backupConfig.password !== 'string') return new BoxError(BoxError.BAD_FIELD, 'password must be a string', { field: 'password' });
-        if (backupConfig.password.length < 8) return new BoxError(BoxError.BAD_FIELD, 'password must be atleast 8 characters', { field: 'password' });
+        if (typeof backupConfig.password !== 'string') return new BoxError(BoxError.BAD_FIELD, 'password must be a string');
+        if (backupConfig.password.length < 8) return new BoxError(BoxError.BAD_FIELD, 'password must be atleast 8 characters');
     }
 
     const policy = backupConfig.retentionPolicy;
-    if (!policy) return new BoxError(BoxError.BAD_FIELD, 'retentionPolicy is required', { field: 'retentionPolicy' });
-    if (!['keepWithinSecs','keepDaily','keepWeekly','keepMonthly','keepYearly'].find(k => !!policy[k])) return new BoxError(BoxError.BAD_FIELD, 'properties missing', { field: 'retentionPolicy' });
-    if ('keepWithinSecs' in policy && typeof policy.keepWithinSecs !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepWithinSecs must be a number', { field: 'retentionPolicy' });
-    if ('keepDaily' in policy && typeof policy.keepDaily !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepDaily must be a number', { field: 'retentionPolicy' });
-    if ('keepWeekly' in policy && typeof policy.keepWeekly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepWeekly must be a number', { field: 'retentionPolicy' });
-    if ('keepMonthly' in policy && typeof policy.keepMonthly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepMonthly must be a number', { field: 'retentionPolicy' });
-    if ('keepYearly' in policy && typeof policy.keepYearly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepYearly must be a number', { field: 'retentionPolicy' });
+    if (!policy) return new BoxError(BoxError.BAD_FIELD, 'retentionPolicy is required');
+    if (!['keepWithinSecs','keepDaily','keepWeekly','keepMonthly','keepYearly'].find(k => !!policy[k])) return new BoxError(BoxError.BAD_FIELD, 'properties missing');
+    if ('keepWithinSecs' in policy && typeof policy.keepWithinSecs !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepWithinSecs must be a number');
+    if ('keepDaily' in policy && typeof policy.keepDaily !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepDaily must be a number');
+    if ('keepWeekly' in policy && typeof policy.keepWeekly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepWeekly must be a number');
+    if ('keepMonthly' in policy && typeof policy.keepMonthly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepMonthly must be a number');
+    if ('keepYearly' in policy && typeof policy.keepYearly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepYearly must be a number');
 
     const [error] = await safe(util.promisify(storage.api(backupConfig.provider).testConfig)(backupConfig));
     return error;
@@ -327,7 +327,7 @@ async function testProviderConfig(backupConfig) {
     assert.strictEqual(typeof backupConfig, 'object');
 
     const func = storage.api(backupConfig.provider);
-    if (!func) return new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' });
+    if (!func) return new BoxError(BoxError.BAD_FIELD, 'unknown storage provider');
 
     const [error] = await safe(util.promisify(storage.api(backupConfig.provider).testConfig)(backupConfig));
     return error;
@@ -339,7 +339,7 @@ async function remount(auditSource) {
     const backupConfig = await settings.getBackupConfig();
 
     const func = storage.api(backupConfig.provider);
-    if (!func) throw new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' });
+    if (!func) throw new BoxError(BoxError.BAD_FIELD, 'unknown storage provider');
 
     await maybePromisify(storage.api(backupConfig.provider).remount)(backupConfig);
 }

src/blobs.js

@@ -4,13 +4,16 @@
 
 exports = module.exports = {
     get,
+    getString,
     set,
+    setString,
     del,
 
     ACME_ACCOUNT_KEY: 'acme_account_key',
     ADDON_TURN_SECRET: 'addon_turn_secret',
     SFTP_PUBLIC_KEY: 'sftp_public_key',
     SFTP_PRIVATE_KEY: 'sftp_private_key',
+    PROXY_AUTH_TOKEN_SECRET: 'proxy_auth_token_secret',
 
     CERT_PREFIX: 'cert',
 
@@ -30,6 +33,14 @@ async function get(id) {
     return result[0].value;
 }
 
+async function getString(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query(`SELECT ${BLOBS_FIELDS} FROM blobs WHERE id = ?`, [ id ]);
+    if (result.length === 0) return null;
+    return result[0].value.toString('utf8');
+}
+
 async function set(id, value) {
     assert.strictEqual(typeof id, 'string');
     assert(value === null || Buffer.isBuffer(value));
@@ -37,6 +48,13 @@ async function set(id, value) {
     await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, value ]);
 }
 
+async function setString(id, value) {
+    assert.strictEqual(typeof id, 'string');
+    assert(value === null || typeof value === 'string');
+
+    await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, Buffer.from(value) ]);
+}
+
 async function del(id) {
     await database.query('DELETE FROM blobs WHERE id=?', [ id ]);
 }

src/cloudron.js

@@ -34,6 +34,7 @@ const apps = require('./apps.js'),
     debug = require('debug')('box:cloudron'),
     delay = require('delay'),
     dns = require('./dns.js'),
+    dockerProxy = require('./dockerproxy.js'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
@@ -63,6 +64,7 @@ async function initialize() {
 
 async function uninitialize() {
     await cron.stopJobs();
+    await dockerProxy.stop();
     await platform.stopAllTasks();
 }
 
@@ -76,6 +78,7 @@ async function onActivated(options) {
     // 2. the restore code path can run without sudo (since mail/ is non-root)
     await platform.start(options);
     await cron.startJobs();
+    await dockerProxy.start(); // this relies on the 'cloudron' docker network interface to be available
 
     // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
     // the UI some time to query the dashboard domain in the restore code path
@@ -112,7 +115,8 @@ async function runStartupTasks() {
     tasks.push(async function () {
         if (!settings.dashboardDomain()) return;
 
-        await reverseProxy.writeDashboardConfig(settings.dashboardDomain());
+        const domainObject = await domains.get(settings.dashboardDomain());
+        await reverseProxy.writeDashboardConfig(domainObject);
     });
 
     tasks.push(async function () {
@@ -157,8 +161,8 @@ async function getConfig() {
         cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
         footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
         features: appstore.getFeatures(),
-        profileLocked: allSettings[settings.DIRECTORY_CONFIG_KEY].lockUserProfiles,
-        mandatory2FA: allSettings[settings.DIRECTORY_CONFIG_KEY].mandatory2FA
+        profileLocked: allSettings[settings.PROFILE_CONFIG_KEY].lockUserProfiles,
+        mandatory2FA: allSettings[settings.PROFILE_CONFIG_KEY].mandatory2FA
     };
 }
 
@@ -223,7 +227,7 @@ async function getLogs(unit, options) {
     // need to handle box.log without subdir
     if (unit === 'box') args.push(path.join(paths.LOG_DIR, 'box.log'));
     else if (unit.startsWith('crash-')) args.push(path.join(paths.CRASH_LOG_DIR, unit.slice(6) + '.log'));
-    else throw new BoxError(BoxError.BAD_FIELD, 'No such unit', { field: 'unit' });
+    else throw new BoxError(BoxError.BAD_FIELD, `No such unit '${unit}'`);
 
     const cp = spawn('/usr/bin/tail', args);
 
@@ -281,7 +285,7 @@ async function setDashboardDomain(domain, auditSource) {
     const domainObject = await domains.get(domain);
     if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');
 
-    await reverseProxy.writeDashboardConfig(domain);
+    await reverseProxy.writeDashboardConfig(domainObject);
     const fqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
     await settings.setDashboardLocation(domain, fqdn);
@@ -323,13 +327,16 @@ async function setupDnsAndCert(subdomain, domain, auditSource, progressCallback)
     const domainObject = await domains.get(domain);
     const dashboardFqdn = dns.fqdn(subdomain, domainObject);
 
-    const ip = await sysinfo.getServerIp();
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();
 
-    progressCallback({ message: `Updating DNS of ${dashboardFqdn}` });
-    await dns.upsertDnsRecords(subdomain, domain, 'A', [ ip ]);
-    progressCallback({ message: `Waiting for DNS of ${dashboardFqdn}` });
-    await dns.waitForDnsRecord(subdomain, domain, 'A', ip, { interval: 30000, times: 50000 });
-    progressCallback({ message: `Getting certificate of ${dashboardFqdn}` });
+    progressCallback({ percent: 20, message: `Updating DNS of ${dashboardFqdn}` });
+    await dns.upsertDnsRecords(subdomain, domain, 'A', [ ipv4 ]);
+    if (ipv6) await dns.upsertDnsRecords(subdomain, domain, 'AAAA', [ ipv6 ]);
+    progressCallback({ percent: 40, message: `Waiting for DNS of ${dashboardFqdn}` });
+    await dns.waitForDnsRecord(subdomain, domain, 'A', ipv4, { interval: 30000, times: 50000 });
+    if (ipv6) await dns.waitForDnsRecord(subdomain, domain, 'AAAA', ipv6, { interval: 30000, times: 50000 });
+    progressCallback({ percent: 60, message: `Getting certificate of ${dashboardFqdn}` });
     await reverseProxy.ensureCertificate(dns.fqdn(subdomain, domainObject), domain, auditSource);
 }
 

src/constants.js

@@ -29,6 +29,7 @@ exports = module.exports = {
     AUTHWALL_PORT: 3001,
     LDAP_PORT: 3002,
     DOCKER_PROXY_PORT: 3003,
+    USER_DIRECTORY_LDAPS_PORT: 3004, // user directory LDAP with TLS rerouting in iptables, public port is 636
 
     NGINX_DEFAULT_CONFIG_FILE_NAME: 'default.conf',
 
@@ -60,8 +61,12 @@ exports = module.exports = {
     CLOUDRON: CLOUDRON,
     TEST: TEST,
 
+    PORT25_CHECK_SERVER: 'port25check.cloudron.io',
+
     SUPPORT_EMAIL: 'support@cloudron.io',
 
+    USER_DIRECTORY_LDAP_DN: 'cn=admin,ou=system,dc=cloudron',
+
     FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
 
     VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '7.0.0-test'

src/cron.js

@@ -34,6 +34,7 @@ const appHealthMonitor = require('./apphealthmonitor.js'),
     system = require('./system.js'),
     updater = require('./updater.js'),
     updateChecker = require('./updatechecker.js'),
+    userdirectory = require('./userdirectory.js'),
     _ = require('underscore');
 
 const gJobs = {
@@ -147,6 +148,10 @@ async function handleSettingsChanged(key, value) {
         await stopJobs();
         await startJobs();
         break;
+    case settings.USER_DIRECTORY_KEY:
+        if (value.enabled) await userdirectory.start();
+        else await userdirectory.stop();
+        break;
     default:
         break;
     }

src/dig.js

@@ -0,0 +1,41 @@
+'use strict';
+
+exports = module.exports = {
+    resolve,
+};
+
+const assert = require('assert'),
+    constants = require('./constants.js'),
+    dns = require('dns'),
+    safe = require('safetydance'),
+    _ = require('underscore');
+
+// a note on TXT records. It doesn't have quotes ("") at the DNS level. Those quotes
+// are added for DNS server software to enclose spaces. Such quotes may also be returned
+// by the DNS REST API of some providers
+async function resolve(hostname, rrtype, options) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof rrtype, 'string');
+    assert(options && typeof options === 'object');
+
+    const defaultOptions = { server: '127.0.0.1', timeout: 5000 }; // unbound runs on 127.0.0.1
+    const resolver = new dns.promises.Resolver();
+    options = _.extend({ }, defaultOptions, options);
+
+    // Only use unbound on a Cloudron
+    if (constants.CLOUDRON) resolver.setServers([ options.server ]);
+
+    // should callback with ECANCELLED but looks like we might hit https://github.com/nodejs/node/issues/14814
+    const timerId = setTimeout(resolver.cancel.bind(resolver), options.timeout || 5000);
+
+    const [error, result] = await safe(resolver.resolve(hostname, rrtype));
+    clearTimeout(timerId);
+
+    if (error && error.code === 'ECANCELLED') error.code = 'TIMEOUT';
+    if (error) throw error;
+
+    // result is an empty array if there was no error but there is no record. when you query a random
+    // domain, it errors with ENOTFOUND. But if you query an existing domain (A record) but with different
+    // type (CNAME) it is not an error and empty array. for TXT records, result is 2d array of strings
+    return result;
+}

src/dns.js

@@ -19,12 +19,6 @@ module.exports = exports = {
 
     checkDnsRecords,
     syncDnsRecords,
-
-    resolve,
-
-    promises: {
-        resolve: require('util').promisify(resolve)
-    }
 };
 
 const apps = require('./apps.js'),
@@ -32,16 +26,14 @@ const apps = require('./apps.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:dns'),
-    dns = require('dns'),
     domains = require('./domains.js'),
+    ipaddr = require('ipaddr.js'),
     mail = require('./mail.js'),
     promiseRetry = require('./promise-retry.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     sysinfo = require('./sysinfo.js'),
-    tld = require('tldjs'),
-    util = require('util'),
-    _ = require('underscore');
+    tld = require('tldjs');
 
 // choose which subdomain backend we use for test purpose we use route53
 function api(provider) {
@@ -66,42 +58,42 @@ function api(provider) {
     }
 }
 
-function fqdn(location, domainObject) {
-    assert.strictEqual(typeof location, 'string');
+function fqdn(subdomain, domainObject) {
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domainObject, 'object');
 
-    return location + (location ? '.' : '') + domainObject.domain;
+    return subdomain + (subdomain ? '.' : '') + domainObject.domain;
 }
 
 // Hostname validation comes from RFC 1123 (section 2.1)
 // Domain name validation comes from RFC 2181 (Name syntax)
 // https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
 // We are validating the validity of the location-fqdn as host name (and not dns name)
-function validateHostname(location, domainObject) {
-    assert.strictEqual(typeof location, 'string');
+function validateHostname(subdomain, domainObject) {
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domainObject, 'object');
 
-    const hostname = fqdn(location, domainObject);
+    const hostname = fqdn(subdomain, domainObject);
 
     const RESERVED_LOCATIONS = [
         constants.SMTP_LOCATION,
         constants.IMAP_LOCATION
     ];
-    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
+    if (RESERVED_LOCATIONS.indexOf(subdomain) !== -1) return new BoxError(BoxError.BAD_FIELD, `subdomain '${subdomain}' is reserved`);
 
-    if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
+    if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, `subdomain '${subdomain}' is reserved`);
 
     // workaround https://github.com/oncletom/tld.js/issues/73
     var tmp = hostname.replace('_', '-');
-    if (!tld.isValid(tmp)) return new BoxError(BoxError.BAD_FIELD, 'Hostname is not a valid domain name', { field: 'location' });
+    if (!tld.isValid(tmp)) return new BoxError(BoxError.BAD_FIELD, 'Hostname is not a valid domain name');
 
-    if (hostname.length > 253) return new BoxError(BoxError.BAD_FIELD, 'Hostname length exceeds 253 characters', { field: 'location' });
+    if (hostname.length > 253) return new BoxError(BoxError.BAD_FIELD, 'Hostname length exceeds 253 characters');
 
-    if (location) {
+    if (subdomain) {
         // label validation
-        if (location.split('.').some(function (p) { return p.length > 63 || p.length < 1; })) return new BoxError(BoxError.BAD_FIELD, 'Invalid subdomain length', { field: 'location' });
-        if (location.match(/^[A-Za-z0-9-.]+$/) === null) return new BoxError(BoxError.BAD_FIELD, 'Subdomain can only contain alphanumeric, hyphen and dot', { field: 'location' });
-        if (/^[-.]/.test(location)) return new BoxError(BoxError.BAD_FIELD, 'Subdomain cannot start or end with hyphen or dot', { field: 'location' });
+        if (subdomain.split('.').some(function (p) { return p.length > 63 || p.length < 1; })) return new BoxError(BoxError.BAD_FIELD, 'Invalid subdomain length');
+        if (subdomain.match(/^[A-Za-z0-9-.]+$/) === null) return new BoxError(BoxError.BAD_FIELD, 'Subdomain can only contain alphanumeric, hyphen and dot');
+        if (/^[-.]/.test(subdomain)) return new BoxError(BoxError.BAD_FIELD, 'Subdomain cannot start or end with hyphen or dot');
     }
 
     return null;
@@ -109,72 +101,74 @@ function validateHostname(location, domainObject) {
 
 // returns the 'name' that needs to be inserted into zone
 // eslint-disable-next-line no-unused-vars
-function getName(domain, location, type) {
+function getName(domain, subdomain, type) {
     const part = domain.domain.slice(0, -domain.zoneName.length - 1);
 
-    if (location === '') return part;
+    if (subdomain === '') return part;
 
-    return part ? `${location}.${part}` : location;
+    return part ? `${subdomain}.${part}` : subdomain;
 }
 
-function maybePromisify(func) {
-    if (util.types.isAsyncFunction(func)) return func;
-    return util.promisify(func);
-}
-
-async function getDnsRecords(location, domain, type) {
-    assert.strictEqual(typeof location, 'string');
+async function getDnsRecords(subdomain, domain, type) {
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof type, 'string');
 
     const domainObject = await domains.get(domain);
-    return await maybePromisify(api(domainObject.provider).get)(domainObject, location, type);
+    return await api(domainObject.provider).get(domainObject, subdomain, type);
 }
 
-async function checkDnsRecords(location, domain) {
-    assert.strictEqual(typeof location, 'string');
+async function checkDnsRecords(subdomain, domain) {
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
 
-    const values = await getDnsRecords(location, domain, 'A');
+    const ipv4Records = await getDnsRecords(subdomain, domain, 'A');
+    const ipv4 = await sysinfo.getServerIPv4();
 
-    const ip = await sysinfo.getServerIp();
+    // if empty OR exactly one record with the ip, we don't need to overwrite
+    if (ipv4Records.length !== 0 && (ipv4Records.length !== 1 || ipv4Records[0] !== ipv4)) return { needsOverwrite: true };
 
-    if (values.length === 0) return { needsOverwrite: false }; // does not exist
-    if (values[0] === ip) return { needsOverwrite: false }; // exists but in sync
+    const ipv6 = await sysinfo.getServerIPv6();
+    if (ipv6) {
+        const ipv6Records = await getDnsRecords(subdomain, domain, 'AAAA');
 
-    return { needsOverwrite: true };
+        // if empty OR exactly one record with the ip, we don't need to overwrite
+        if (ipv6Records.length !== 0 && (ipv6Records.length !== 1 || ipaddr.parse(ipv6Records[0]).toRFC5952String() !== ipv6)) return { needsOverwrite: true };
+    }
+
+    return { needsOverwrite: false }; // one record exists and in sync
 }
 
 // note: for TXT records the values must be quoted
-async function upsertDnsRecords(location, domain, type, values) {
-    assert.strictEqual(typeof location, 'string');
+async function upsertDnsRecords(subdomain, domain, type, values) {
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
 
-    debug(`upsertDNSRecord: location ${location} on domain ${domain} of type ${type} with values ${JSON.stringify(values)}`);
+    debug(`upsertDNSRecord: location ${subdomain} on domain ${domain} of type ${type} with values ${JSON.stringify(values)}`);
 
     const domainObject = await domains.get(domain);
-    await maybePromisify(api(domainObject.provider).upsert)(domainObject, location, type, values);
+    await api(domainObject.provider).upsert(domainObject, subdomain, type, values);
 }
 
-async function removeDnsRecords(location, domain, type, values) {
-    assert.strictEqual(typeof location, 'string');
+async function removeDnsRecords(subdomain, domain, type, values) {
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
 
-    debug('removeDNSRecord: %s on %s type %s values', location, domain, type, values);
+    debug('removeDNSRecord: %s on %s type %s values', subdomain, domain, type, values);
 
     const domainObject = await domains.get(domain);
-    const [error] = await safe(maybePromisify(api(domainObject.provider).del)(domainObject, location, type, values));
-    if (error && error.reason !== BoxError.NOT_FOUND) throw error;
+    const [error] = await safe(api(domainObject.provider).del(domainObject, subdomain, type, values));
+    if (error && error.reason !== BoxError.NOT_FOUND) throw error; // this is never returned afaict
 }
 
-async function waitForDnsRecord(location, domain, type, value, options) {
-    assert.strictEqual(typeof location, 'string');
+async function waitForDnsRecord(subdomain, domain, type, value, options) {
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert(type === 'A' || type === 'TXT');
+    assert(type === 'A' || type === 'AAAA' || type === 'TXT');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
@@ -183,7 +177,7 @@ async function waitForDnsRecord(location, domain, type, value, options) {
     // linode DNS takes ~15mins
     if (!options.interval) options.interval = domainObject.provider === 'linode' ? 20000 : 5000;
 
-    await maybePromisify(api(domainObject.provider).wait)(domainObject, location, type, value, options);
+    await api(domainObject.provider).wait(domainObject, subdomain, type, value, options);
 }
 
 function makeWildcard(vhost) {
@@ -195,6 +189,30 @@ function makeWildcard(vhost) {
     return parts.join('.');
 }
 
+async function registerLocation(location, options, recordType, recordValue) {
+    const overwriteDns = options.overwriteDns || false;
+
+    // get the current record before updating it
+    const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, recordType));
+    if (getError) {
+        const retryable = getError.reason !== BoxError.ACCESS_DENIED && getError.reason !== BoxError.NOT_FOUND;
+        debug(`registerLocation: Get error. retryable: ${retryable}. ${getError.message}`);
+        throw new BoxError(getError.reason, getError.message, { domain: location, retryable });
+    }
+
+    if (values.length === 1 && values[0] === recordValue) return; // up-to-date
+
+    // refuse to update any existing DNS record for custom domains that we did not create
+    if (values.length !== 0 && !overwriteDns) throw new BoxError(BoxError.ALREADY_EXISTS, `DNS ${recordType} record already exists`, { domain: location, retryable: false });
+
+    const [upsertError] = await safe(upsertDnsRecords(location.subdomain, location.domain, recordType, [ recordValue ]));
+    if (upsertError) {
+        const retryable = upsertError.reason === BoxError.BUSY || upsertError.reason === BoxError.EXTERNAL_ERROR;
+        debug(`registerLocation: Upsert error. retryable: ${retryable}. ${upsertError.message}`);
+        throw new BoxError(BoxError.EXTERNAL_ERROR, upsertError.message, { domain: location, retryable });
+    }
+}
+
 async function registerLocations(locations, options, progressCallback) {
     assert(Array.isArray(locations));
     assert.strictEqual(typeof options, 'object');
@@ -202,61 +220,43 @@ async function registerLocations(locations, options, progressCallback) {
 
     debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
 
-    const overwriteDns = options.overwriteDns || false;
-
-    const ip = await sysinfo.getServerIp();
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();
 
     for (const location of locations) {
-        const error = await promiseRetry({ times: 200, interval: 5000 }, async function () {
-            progressCallback({ message: `Registering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+        progressCallback({ message: `Registering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
 
-            // get the current record before updating it
-            const [error, values] = await safe(getDnsRecords(location.subdomain, location.domain, 'A'));
-            if (error && error.reason === BoxError.EXTERNAL_ERROR) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }); // try again
-            // give up for other errors
-            if (error && error.reason === BoxError.ACCESS_DENIED) return new BoxError(BoxError.ACCESS_DENIED, error.message, { domain: location });
-            if (error && error.reason === BoxError.NOT_FOUND) return new BoxError(BoxError.NOT_FOUND, error.message, { domain: location });
-            if (error) return new BoxError(BoxError.EXTERNAL_ERROR, error.message, location);
-
-            if (values.length !== 0 && values[0] === ip) return null; // up-to-date
-
-            // refuse to update any existing DNS record for custom domains that we did not create
-            if (values.length !== 0 && !overwriteDns) return new BoxError(BoxError.ALREADY_EXISTS, 'DNS Record already exists', { domain: location });
-
-            const [upsertError] = await safe(upsertDnsRecords(location.subdomain, location.domain, 'A', [ ip ]));
-            if (upsertError && (upsertError.reason === BoxError.BUSY || upsertError.reason === BoxError.EXTERNAL_ERROR)) {
-                progressCallback({ message: `registerSubdomains: Upsert error. Will retry. ${upsertError.message}` });
-                throw new BoxError(BoxError.EXTERNAL_ERROR, upsertError.message, { domain: location }); // try again
-            }
-
-            return upsertError ? new BoxError(BoxError.EXTERNAL_ERROR, upsertError.message, location) : null;
+        await promiseRetry({ times: 200, interval: 5000, debug, retry: (error) => error.retryable }, async function () {
+            await registerLocation(location, options, 'A', ipv4);
+            if (ipv6) await registerLocation(location, options, 'AAAA', ipv6);
         });
-
-        if (error) throw error;
     }
 }
 
+async function unregisterLocation(location, recordType, recordValue) {
+    const [error] = await safe(removeDnsRecords(location.subdomain, location.domain, recordType, [ recordValue ]));
+    if (!error || error.reason === BoxError.NOT_FOUND) return;
+
+    const retryable = error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR;
+    debug(`unregisterLocation: Error unregistering location ${recordType}. retryable: ${retryable}. ${error.message}`);
+
+    throw new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location, retryable });
+}
+
 async function unregisterLocations(locations, progressCallback) {
     assert(Array.isArray(locations));
     assert.strictEqual(typeof progressCallback, 'function');
 
-    const ip = await sysinfo.getServerIp();
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();
 
     for (const location of locations) {
-        const error = await promiseRetry({ times: 30, interval: 5000 }, async function () {
-            progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+        progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
 
-            const [error] = await safe(removeDnsRecords(location.subdomain, location.domain, 'A', [ ip ]));
-            if (error && error.reason === BoxError.NOT_FOUND) return;
-            if (error && (error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                progressCallback({ message: `Error unregistering location. Will retry. ${error.message}`});
-                throw new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }); // try again
-            }
-
-            return error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }) : null; // give up for other errors
+        await promiseRetry({ times: 30, interval: 5000, debug, retry: (error) => error.retryable }, async function () {
+            await unregisterLocation(location, 'A', ipv4);
+            if (ipv6) await unregisterLocation(location, 'AAAA', ipv6);
         });
-
-        if (error) throw error;
     }
 }
 
@@ -285,10 +285,10 @@ async function syncDnsRecords(options, progressCallback) {
         if (domain.domain === settings.dashboardDomain()) locations.push({ subdomain: constants.DASHBOARD_LOCATION, domain: settings.dashboardDomain() });
         if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.dashboardFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
 
-        allApps.forEach(function (app) {
-            const appLocations = [{ subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains);
+        for (const app of allApps) {
+            const appLocations = [{ subdomain: app.subdomain, domain: app.domain }].concat(app.redirectDomains).concat(app.aliasDomains);
             locations = locations.concat(appLocations.filter(al => al.domain === domain.domain));
-        });
+        }
 
         try {
             await registerLocations(locations, { overwriteDns: true }, progressCallback);
@@ -301,35 +301,3 @@ async function syncDnsRecords(options, progressCallback) {
 
     return { errors };
 }
-
-// a note on TXT records. It doesn't have quotes ("") at the DNS level. Those quotes
-// are added for DNS server software to enclose spaces. Such quotes may also be returned
-// by the DNS REST API of some providers
-function resolve(hostname, rrtype, options, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof rrtype, 'string');
-    assert(options && typeof options === 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const defaultOptions = { server: '127.0.0.1', timeout: 5000 }; // unbound runs on 127.0.0.1
-    const resolver = new dns.Resolver();
-    options = _.extend({ }, defaultOptions, options);
-
-    // Only use unbound on a Cloudron
-    if (constants.CLOUDRON) resolver.setServers([ options.server ]);
-
-    // should callback with ECANCELLED but looks like we might hit https://github.com/nodejs/node/issues/14814
-    const timerId = setTimeout(resolver.cancel.bind(resolver), options.timeout || 5000);
-
-    resolver.resolve(hostname, rrtype, function (error, result) {
-        clearTimeout(timerId);
-
-        if (error && error.code === 'ECANCELLED') error.code = 'TIMEOUT';
-
-        // result is an empty array if there was no error but there is no record. when you query a random
-        // domain, it errors with ENOTFOUND. But if you query an existing domain (A record) but with different
-        // type (CNAME) it is not an error and empty array
-        // for TXT records, result is 2d array of strings
-        callback(error, result);
-    });
-}

src/dns/cloudflare.js

@@ -7,22 +7,23 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
-    async = require('async'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/cloudflare'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
 // we are using latest v4 stable API https://api.cloudflare.com/#getting-started-endpoints
-var CLOUDFLARE_ENDPOINT = 'https://api.cloudflare.com/client/v4';
+const CLOUDFLARE_ENDPOINT = 'https://api.cloudflare.com/client/v4';
 
 function removePrivateFields(domainObject) {
     domainObject.config.token = constants.SECRET_PLACEHOLDER;
@@ -33,12 +34,11 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function translateRequestError(result, callback) {
+async function translateRequestError(result) {
     assert.strictEqual(typeof result, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, util.format('%s %j', result.statusCode, 'API does not exist')));
-    if (result.statusCode === 422) return callback(new BoxError(BoxError.BAD_FIELD, result.body.message));
+    if (result.statusCode === 404) return new BoxError(BoxError.NOT_FOUND, util.format('%s %j', result.statusCode, 'API does not exist'));
+    if (result.statusCode === 422) return new BoxError(BoxError.BAD_FIELD, result.body.message);
     if (result.statusCode === 400 || result.statusCode === 401 || result.statusCode === 403) {
         let message = 'Unknown error';
         if (typeof result.body.error === 'string') {
@@ -47,288 +47,230 @@ function translateRequestError(result, callback) {
             let error = result.body.errors[0];
             message = `message: ${error.message} statusCode: ${result.statusCode} code:${error.code}`;
         }
-        return callback(new BoxError(BoxError.ACCESS_DENIED, message));
+        return new BoxError(BoxError.ACCESS_DENIED, message);
     }
 
-    callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+    return new BoxError(BoxError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body));
 }
 
-function createRequest(method, url, dnsConfig) {
+function createRequest(method, url, domainConfig) {
     assert.strictEqual(typeof method, 'string');
     assert.strictEqual(typeof url, 'string');
-    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof domainConfig, 'object');
 
-    let request = superagent(method, url)
-        .timeout(30 * 1000);
+    const request = superagent(method, url).timeout(30 * 1000).ok(() => true);
 
-    if (dnsConfig.tokenType === 'GlobalApiKey') {
-        request.set('X-Auth-Key', dnsConfig.token).set('X-Auth-Email', dnsConfig.email);
+    if (domainConfig.tokenType === 'GlobalApiKey') {
+        request.set('X-Auth-Key', domainConfig.token).set('X-Auth-Email', domainConfig.email);
     } else {
-        request.set('Authorization', 'Bearer ' + dnsConfig.token);
+        request.set('Authorization', 'Bearer ' + domainConfig.token);
     }
 
     return request;
 }
 
-function getZoneByName(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneByName(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active', dnsConfig)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(error);
-            if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
-            if (!result.body.result.length) return callback(new BoxError(BoxError.NOT_FOUND, util.format('%s %j', result.statusCode, result.body)));
+    const [error, response] = await safe(createRequest('GET', `${CLOUDFLARE_ENDPOINT}/zones?name=${zoneName}&status=active`, domainConfig));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
+    if (!response.body.result.length) throw new BoxError(BoxError.NOT_FOUND, util.format('%s %j', response.statusCode, response.body));
 
-            callback(null, result.body.result[0]);
-        });
+    return response.body.result[0];
 }
 
 // gets records filtered by zone, type and fqdn
-function getDnsRecords(dnsConfig, zoneId, fqdn, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getDnsRecords(domainConfig, zoneId, fqdn, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneId, 'string');
     assert.strictEqual(typeof fqdn, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
-        .query({ type: type, name: fqdn })
-        .end(function (error, result) {
-            if (error && !error.response) return callback(error);
-            if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
+    const [error, response] = await safe(createRequest('GET', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig)
+        .query({ type: type, name: fqdn }));
 
-            var tmp = result.body.result;
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
 
-            return callback(null, tmp);
-        });
+    return response.body.result;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         fqdn = dns.fqdn(location, domainObject);
 
     debug('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
-    getZoneByName(dnsConfig, zoneName, function(error, result) {
-        if (error) return callback(error);
+    const result = await getZoneByName(domainConfig, zoneName);
+    const zoneId = result.id;
 
-        let zoneId = result.id;
+    const records = await getDnsRecords(domainConfig, zoneId, fqdn, type);
 
-        getDnsRecords(dnsConfig, zoneId, fqdn, type, function (error, dnsRecords) {
-            if (error) return callback(error);
+    let i = 0; // // used to track available records to update instead of create
 
-            let i = 0; // // used to track available records to update instead of create
+    for (let value of values) {
+        let priority = null;
 
-            async.eachSeries(values, function (value, iteratorCallback) {
-                var priority = null;
+        if (type === 'MX') {
+            priority = parseInt(value.split(' ')[0], 10);
+            value = value.split(' ')[1];
+        }
 
-                if (type === 'MX') {
-                    priority = parseInt(value.split(' ')[0], 10);
-                    value = value.split(' ')[1];
-                }
+        const data = {
+            type: type,
+            name: fqdn,
+            content: value,
+            priority: priority,
+            proxied: false,
+            ttl: 120  // 1 means "automatic" (meaning 300ms) and 120 is the lowest supported
+        };
 
-                var data = {
-                    type: type,
-                    name: fqdn,
-                    content: value,
-                    priority: priority,
-                    proxied: false,
-                    ttl: 120  // 1 means "automatic" (meaning 300ms) and 120 is the lowest supported
-                };
+        if (i >= records.length) { // create a new record
+            debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: false`);
 
-                if (i >= dnsRecords.length) { // create a new record
-                    debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: false`);
+            const [error, response] = await safe(createRequest('POST', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records`, domainConfig)
+                .send(data));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
+        } else { // replace existing record
+            data.proxied = records[i].proxied; // preserve proxied parameter
 
-                    createRequest('POST', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
-                        .send(data)
-                        .end(function (error, result) {
-                            if (error && !error.response) return iteratorCallback(error);
-                            if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, iteratorCallback);
+            debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
 
-                            iteratorCallback(null);
-                        });
-                } else { // replace existing record
-                    data.proxied = dnsRecords[i].proxied; // preserve proxied parameter
+            const [error, response] = await safe(createRequest('PUT', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[i].id}`, domainConfig)
+                .send(data));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
+            ++i; // increment, as we have consumed the record
+        }
+    }
 
-                    debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
-
-                    createRequest('PUT', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id, dnsConfig)
-                        .send(data)
-                        .end(function (error, result) {
-                            ++i; // increment, as we have consumed the record
-
-                            if (error && !error.response) return iteratorCallback(error);
-                            if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, iteratorCallback);
-
-                            iteratorCallback(null);
-                        });
-                }
-            }, callback);
-        });
-    });
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${records[j].id}`, domainConfig));
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(dnsConfig, zoneName, function(error, zone) {
-        if (error) return callback(error);
-
-        getDnsRecords(dnsConfig, zone.id, fqdn, type, function (error, result) {
-            if (error) return callback(error);
-
-            var tmp = result.map(function (record) { return record.content; });
-            debug('get: %j', tmp);
-
-            callback(null, tmp);
-        });
-    });
+    const zone = await getZoneByName(domainConfig, zoneName);
+    const result = await getDnsRecords(domainConfig, zone.id, fqdn, type);
+    const tmp = result.map(function (record) { return record.content; });
+    return tmp;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(dnsConfig, zoneName, function(error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        getDnsRecords(dnsConfig, zone.id, fqdn, type, function(error, result) {
-            if (error) return callback(error);
-            if (result.length === 0) return callback(null);
+    const result = await getDnsRecords(domainConfig, zone.id, fqdn, type);
+    if (result.length === 0) return;
 
-            var zoneId = result[0].zone_id;
+    const zoneId = result[0].zone_id;
 
-            var tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); });
-            debug('del: %j', tmp);
+    const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.content; }); });
+    debug('del: %j', tmp);
 
-            if (tmp.length === 0) return callback(null);
+    if (tmp.length === 0) return;
 
-            async.eachSeries(tmp, function (record, callback) {
-                createRequest('DELETE', CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id, dnsConfig)
-                    .end(function (error, result) {
-                        if (error && !error.response) return callback(error);
-                        if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
-
-                        debug('del: done');
-
-                        callback(null);
-                    });
-            }, function (error) {
-                if (error) return callback(error);
-
-                callback(null, 'unused');
-            });
-        });
-    });
+    for (const r of tmp) {
+        const [error, response] = await safe(createRequest('DELETE', `${CLOUDFLARE_ENDPOINT}/zones/${zoneId}/dns_records/${r.id}`, domainConfig));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode !== 200 || response.body.success !== true) throw translateRequestError(response);
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject);
+        fqdn = dns.fqdn(subdomain, domainObject);
 
     debug('wait: %s for zone %s of type %s', fqdn, zoneName, type);
 
-    getZoneByName(dnsConfig, zoneName, function(error, result) {
-        if (error) return callback(error);
+    const result = await getZoneByName(domainConfig, zoneName);
+    const zoneId = result.id;
 
-        let zoneId = result.id;
+    const dnsRecords = await getDnsRecords(domainConfig, zoneId, fqdn, type);
+    if (dnsRecords.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
 
-        getDnsRecords(dnsConfig, zoneId, fqdn, type, function (error, dnsRecords) {
-            if (error) return callback(error);
-            if (dnsRecords.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
+    if (!dnsRecords[0].proxied) return await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 
-            if (!dnsRecords[0].proxied) return waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    debug('wait: skipping wait of proxied domain');
 
-            debug('wait: skipping wait of proxied domain');
-
-            callback(null); // maybe we can check for dns to be cloudflare IPs? https://api.cloudflare.com/#cloudflare-ips-cloudflare-ip-details
-        });
-    });
+    // maybe we can check for dns to be cloudflare IPs? https://api.cloudflare.com/#cloudflare-ips-cloudflare-ip-details
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
     // token can be api token or global api key
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
-    if (dnsConfig.tokenType !== 'GlobalApiKey' && dnsConfig.tokenType !== 'ApiToken')  return callback(new BoxError(BoxError.BAD_FIELD, 'tokenType is required', { field: 'tokenType' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
+    if (domainConfig.tokenType !== 'GlobalApiKey' && domainConfig.tokenType !== 'ApiToken') throw new BoxError(BoxError.BAD_FIELD, 'tokenType is required');
 
-    if (dnsConfig.tokenType === 'GlobalApiKey') {
-        if (typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    if (domainConfig.tokenType === 'GlobalApiKey') {
+        if (typeof domainConfig.email !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string');
     }
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        token: dnsConfig.token,
-        tokenType: dnsConfig.tokenType,
-        email: dnsConfig.email || null
+    const credentials = {
+        token: domainConfig.token,
+        tokenType: domainConfig.tokenType,
+        email: domainConfig.email || null
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        getZoneByName(dnsConfig, zoneName, function(error, zone) {
-            if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-            if (!_.isEqual(zone.name_servers.sort(), nameservers.sort())) {
-                debug('verifyDnsConfig: %j and %j do not match', nameservers, zone.name_servers);
-                return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Cloudflare', { field: 'nameservers' }));
-            }
+    if (!_.isEqual(zone.name_servers.sort(), nameservers.sort())) {
+        debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.name_servers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Cloudflare');
+    }
 
-            const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-            upsert(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-                debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-                del(domainObject, location, 'A', [ ip ], function (error) {
-                    if (error) return callback(error);
-
-                    debug('verifyDnsConfig: Test A record removed again');
-
-                    callback(null, credentials);
-                });
-            });
-        });
-    });
+    return credentials;
 }

src/dns/digitalocean.js

@@ -7,14 +7,14 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
-    async = require('async'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/digitalocean'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
@@ -36,244 +36,208 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function getInternal(dnsConfig, zoneName, name, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneRecords(domainConfig, zoneName, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var nextPage = null, matchingRecords = [];
+    let nextPage = null, matchingRecords = [];
 
     debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
 
-    async.doWhilst(function (iteratorDone) {
-        var url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records';
+    do {
+        const url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records';
 
-        superagent.get(url)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
+        const [error, response] = await safe(superagent.get(url)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
             .timeout(30 * 1000)
             .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return iteratorDone(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return iteratorDone(new BoxError(BoxError.NOT_FOUND, formatError(result)));
-                if (result.statusCode === 403 || result.statusCode === 401) return iteratorDone(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return iteratorDone(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            .ok(() => true));
 
-                matchingRecords = matchingRecords.concat(result.body.domain_records.filter(function (record) {
-                    return (record.type === type && record.name === name);
-                }));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                nextPage = (result.body.links && result.body.links.pages) ? result.body.links.pages.next : null;
+        matchingRecords = matchingRecords.concat(response.body.domain_records.filter(function (record) {
+            return (record.type === type && record.name === name);
+        }));
 
-                iteratorDone();
-            });
-    }, function (testDone) { return testDone(null, !!nextPage); }, function (error) {
-        debug('getInternal:', error, JSON.stringify(matchingRecords));
+        nextPage = (response.body.links && response.body.links.pages) ? response.body.links.pages.next : null;
+    } while (nextPage);
 
-        if (error) return callback(error);
-
-        return callback(null, matchingRecords);
-    });
+    return matchingRecords;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const records = await getZoneRecords(domainConfig, zoneName, name, type);
 
-        // used to track available records to update instead of create
-        var i = 0, recordIds = [];
+    // used to track available records to update instead of create
+    let i = 0, recordIds = [];
 
-        async.eachSeries(values, function (value, iteratorCallback) {
-            var priority = null;
+    for (let value of values) {
+        let priority = null;
 
-            if (type === 'MX') {
-                priority = value.split(' ')[0];
-                value = value.split(' ')[1];
-            }
+        if (type === 'MX') {
+            priority = value.split(' ')[0];
+            value = value.split(' ')[1];
+        }
 
-            var data = {
-                type: type,
-                name: name,
-                data: value,
-                priority: priority,
-                ttl: 30 // Recent DO DNS API break means this value must atleast be 30
-            };
+        const data = {
+            type: type,
+            name: name,
+            data: value,
+            priority: priority,
+            ttl: 30 // Recent DO DNS API break means this value must atleast be 30
+        };
 
-            if (i >= result.length) {
-                superagent.post(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records')
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode === 422) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, result.body.message));
-                        if (result.statusCode !== 201) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        if (i >= records.length) {
+            const [error, response] = await safe(superagent.post(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records')
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        recordIds.push(safe.query(result.body, 'domain_record.id'));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
+            if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        return iteratorCallback(null);
-                    });
-            } else {
-                superagent.put(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + result[i].id)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                    // increment, as we have consumed the record
-                        ++i;
+            recordIds.push(safe.query(records.body, 'domain_record.id'));
+        } else {
+            const [error, response] = await safe(superagent.put(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + records[i].id)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode === 422) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, result.body.message));
-                        if (result.statusCode !== 200) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            ++i;
 
-                        recordIds.push(safe.query(result.body, 'domain_record.id'));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        return iteratorCallback(null);
-                    });
-            }
-        }, function (error) {
-            if (error) return callback(error);
+            recordIds.push(safe.query(records.body, 'domain_record.id'));
+        }
+    }
 
-            debug('upsert: completed with recordIds:%j', recordIds);
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] = await safe(superagent.del(`${DIGITALOCEAN_ENDPOINT}/v2/domains/${zoneName}/records/${records[j].id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
 
-            callback();
-        });
-    });
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
+
+    debug('upsert: completed with recordIds:%j', recordIds);
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const result = await getZoneRecords(domainConfig, zoneName, name, type);
 
-        // We only return the value string
-        var tmp = result.map(function (record) { return record.data; });
-
-        debug('get: %j', tmp);
-
-        return callback(null, tmp);
-    });
+    const tmp = result.map(function (record) { return record.data; });
+    return tmp;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const result = await getZoneRecords(domainConfig, zoneName, name, type);
+    if (result.length === 0) return;
 
-        if (result.length === 0) return callback(null);
+    const tmp = result.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
+    if (tmp.length === 0) return;
 
-        var tmp = result.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
-
-        debug('del: %j', tmp);
-
-        if (tmp.length === 0) return callback(null);
-
-        // FIXME we only handle the first one currently
-
-        superagent.del(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + tmp[0].id)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
+    for (const r of tmp) {
+        const [error, response] = await safe(superagent.del(`${DIGITALOCEAN_ENDPOINT}/v2/domains/${zoneName}/records/${r.id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
             .timeout(30 * 1000)
             .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return callback(null);
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            .ok(() => true));
 
-                debug('del: done');
-
-                return callback(null);
-            });
-    });
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) return;
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        token: dnsConfig.token
+    const credentials = {
+        token: domainConfig.token
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.digitalocean.com') === -1) {
-            debug('verifyDnsConfig: %j does not contains DO NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to DigitalOcean', { field: 'nameservers' }));
-        }
+    if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.digitalocean.com') === -1) {
+        debug('verifyDomainConfig: %j does not contains DO NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to DigitalOcean');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/gandi.js

@@ -7,14 +7,16 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/gandi'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -34,146 +36,126 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    var data = {
+    const data = {
         'rrset_ttl': 300, // this is the minimum allowed
         'rrset_values': values // for mx records, value is already of the '<priority> <server>' format
     };
 
-    superagent.put(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
-        .set('X-Api-Key', dnsConfig.token)
+    const [error, response] = await safe(superagent.put(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
+        .set('X-Api-Key', domainConfig.token)
         .timeout(30 * 1000)
         .send(data)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            return callback(null);
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+    if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type}`);
 
-    superagent.get(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
-        .set('X-Api-Key', dnsConfig.token)
+    const [error, response] = await safe(superagent.get(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
+        .set('X-Api-Key', domainConfig.token)
         .timeout(30 * 1000)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode === 404) return callback(null, [ ]);
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            debug('get: %j', result.body);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode === 404) return [];
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-            return callback(null, result.body.rrset_values);
-        });
+    return response.body.rrset_values;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    superagent.del(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
-        .set('X-Api-Key', dnsConfig.token)
+    const [error, response] = await safe(superagent.del(`${GANDI_API}/domains/${zoneName}/records/${name}/${type}`)
+        .set('X-Api-Key', domainConfig.token)
         .timeout(30 * 1000)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 404) return callback(null);
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            debug('del: done');
-
-            return callback(null);
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 404) return;
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
 
-    var credentials = {
-        token: dnsConfig.token
+    const credentials = {
+        token: domainConfig.token
     };
 
     const ip = '127.0.0.1';
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.gandi.net') !== -1; })) {
-            debug('verifyDnsConfig: %j does not contain Gandi NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Gandi', { field: 'nameservers' }));
-        }
+    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.gandi.net') !== -1; })) {
+        debug('verifyDomainConfig: %j does not contain Gandi NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Gandi');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/gcdns.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const safe = require('safetydance');
+
 exports = module.exports = {
     removePrivateFields,
     injectPrivateFields,
@@ -7,13 +9,14 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/gcdns'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
     GCDNS = require('@google-cloud/dns').DNS,
     waitForDns = require('./waitfordns.js'),
@@ -28,210 +31,166 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.credentials.private_key === constants.SECRET_PLACEHOLDER && currentConfig.credentials) newConfig.credentials.private_key = currentConfig.credentials.private_key;
 }
 
-function getDnsCredentials(dnsConfig) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+function getDnsCredentials(domainConfig) {
+    assert.strictEqual(typeof domainConfig, 'object');
 
     return {
-        projectId: dnsConfig.projectId,
+        projectId: domainConfig.projectId,
         credentials: {
-            client_email: dnsConfig.credentials.client_email,
-            private_key: dnsConfig.credentials.private_key
+            client_email: domainConfig.credentials.client_email,
+            private_key: domainConfig.credentials.private_key
         }
     };
 }
 
-function getZoneByName(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneByName(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var gcdns = new GCDNS(getDnsCredentials(dnsConfig));
+    const gcdns = new GCDNS(getDnsCredentials(domainConfig));
 
-    gcdns.getZones(function (error, zones) {
-        if (error && error.message === 'invalid_grant') return callback(new BoxError(BoxError.ACCESS_DENIED, 'The key was probably revoked'));
-        if (error && error.reason === 'No such domain') return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-        if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-        if (error && error.code === 404) return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-        if (error) {
-            debug('gcdns.getZones', error);
-            return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-        }
+    const [error, result] = await safe(gcdns.getZones());
+    if (error && error.message === 'invalid_grant') throw new BoxError(BoxError.ACCESS_DENIED, 'The key was probably revoked');
+    if (error && error.reason === 'No such domain') throw new BoxError(BoxError.NOT_FOUND, error.message);
+    if (error && error.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 404) throw new BoxError(BoxError.NOT_FOUND, error.message);
+    if (error) {
+        debug('gcdns.getZones', error);
+        throw new BoxError(BoxError.EXTERNAL_ERROR, error);
+    }
 
-        var zone = zones.filter(function (zone) {
-            return zone.metadata.dnsName.slice(0, -1) === zoneName;     // the zone name contains a '.' at the end
-        })[0];
+    const zone = result[0].filter(function (zone) {
+        return zone.metadata.dnsName.slice(0, -1) === zoneName;     // the zone name contains a '.' at the end
+    })[0];
 
-        if (!zone) return callback(new BoxError(BoxError.NOT_FOUND, 'no such zone'));
+    if (!zone) throw new BoxError(BoxError.NOT_FOUND, 'no such zone');
 
-        callback(null, zone); //zone.metadata ~= {name="", dnsName="", nameServers:[]}
-    });
+    return zone; //zone.metadata ~= {name="", dnsName="", nameServers:[]}
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         fqdn = dns.fqdn(location, domainObject);
 
     debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
-    getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
 
-        zone.getRecords({ type: type, name: fqdn + '.' }, function (error, oldRecords) {
-            if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) {
-                debug('upsert->zone.getRecords', error);
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-            }
+    const [error, result] = await safe(zone.getRecords({ type: type, name: fqdn + '.' }));
+    if (error && error.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 
-            var newRecord = zone.record(type, {
-                name: fqdn + '.',
-                data: values,
-                ttl: 1
-            });
-
-            zone.createChange({ delete: oldRecords, add: newRecord }, function(error /*, change */) {
-                if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-                if (error && error.code === 412) return callback(new BoxError(BoxError.BUSY, error.message));
-                if (error) {
-                    debug('upsert->zone.createChange', error);
-                    return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-                }
-
-                callback(null);
-            });
-        });
+    const newRecord = zone.record(type, {
+        name: fqdn + '.',
+        data: values,
+        ttl: 1
     });
+
+    const [changeError] = await safe(zone.createChange({ delete: result[0], add: newRecord }));
+    if (changeError && changeError.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, changeError.message);
+    if (changeError && changeError.code === 412) throw new BoxError(BoxError.BUSY, changeError.message);
+    if (changeError) throw new BoxError(BoxError.EXTERNAL_ERROR, changeError.message);
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
 
-        var params = {
-            name: fqdn + '.',
-            type: type
-        };
+    const params = {
+        name: fqdn + '.',
+        type: type
+    };
 
-        zone.getRecords(params, function (error, records) {
-            if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-            if (records.length === 0) return callback(null, [ ]);
+    const [error, result] = await safe(zone.getRecords(params));
+    if (error && error.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
+    if (result[0].length === 0) return [];
 
-            return callback(null, records[0].data);
-        });
-    });
+    return result[0][0].data;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
 
-        zone.getRecords({ type: type, name: fqdn + '.' }, function(error, oldRecords) {
-            if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) {
-                debug('del->zone.getRecords', error);
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-            }
+    const [error, result] = await safe(zone.getRecords({ type: type, name: fqdn + '.' }));
+    if (error && error.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 
-            zone.deleteRecords(oldRecords, function (error, change) {
-                if (error && error.code === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-                if (error && error.code === 412) return callback(new BoxError(BoxError.BUSY, error.message));
-                if (error) {
-                    debug('del->zone.createChange', error);
-                    return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-                }
-
-                callback(null, change.id);
-            });
-        });
-    });
+    const [delError] = await safe(zone.deleteRecords(result[0]));
+    if (delError && delError.code === 403) throw new BoxError(BoxError.ACCESS_DENIED, delError.message);
+    if (delError && delError.code === 412) throw new BoxError(BoxError.BUSY, delError.message);
+    if (delError) throw new BoxError(BoxError.EXTERNAL_ERROR, delError.message);
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (typeof dnsConfig.projectId !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'projectId must be a string', { field: 'projectId' }));
-    if (!dnsConfig.credentials || typeof dnsConfig.credentials !== 'object') return callback(new BoxError(BoxError.BAD_FIELD, 'credentials must be an object', { field: 'credentials' }));
-    if (typeof dnsConfig.credentials.client_email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'credentials.client_email must be a string', { field: 'client_email' }));
-    if (typeof dnsConfig.credentials.private_key !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'credentials.private_key must be a string', { field: 'private_key' }));
+    if (typeof domainConfig.projectId !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'projectId must be a string');
+    if (!domainConfig.credentials || typeof domainConfig.credentials !== 'object') throw new BoxError(BoxError.BAD_FIELD, 'credentials must be an object');
+    if (typeof domainConfig.credentials.client_email !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.client_email must be a string');
+    if (typeof domainConfig.credentials.private_key !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'credentials.private_key must be a string');
 
-    var credentials = getDnsCredentials(dnsConfig);
+    const credentials = getDnsCredentials(domainConfig);
 
     const ip = '127.0.0.1';
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        getZoneByName(credentials, zoneName, function (error, zone) {
-            if (error) return callback(error);
+    const zone = await getZoneByName(credentials, zoneName);
 
-            var definedNS = zone.metadata.nameServers.sort().map(function(r) { return r.replace(/\.$/, ''); });
-            if (!_.isEqual(definedNS, nameservers.sort())) {
-                debug('verifyDnsConfig: %j and %j do not match', nameservers, definedNS);
-                return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS', { field: 'nameservers' }));
-            }
+    const definedNS = zone.metadata.nameServers.sort().map(function(r) { return r.replace(/\.$/, ''); });
+    if (!_.isEqual(definedNS, nameservers.sort())) {
+        debug('verifyDomainConfig: %j and %j do not match', nameservers, definedNS);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS');
+    }
 
-            const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            upsert(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-                debug('verifyDnsConfig: Test A record added');
-
-                del(domainObject, location, 'A', [ ip ], function (error) {
-                    if (error) return callback(error);
-
-                    debug('verifyDnsConfig: Test A record removed again');
-
-                    callback(null, credentials);
-                });
-            });
-        });
-    });
+    return credentials;
 }

src/dns/godaddy.js

@@ -7,14 +7,16 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/godaddy'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -25,6 +27,7 @@ const GODADDY_API = 'https://api.godaddy.com/v1/domains';
 // this is a workaround for godaddy not having a delete API
 // https://stackoverflow.com/questions/39347464/delete-record-libcloud-godaddy-api
 const GODADDY_INVALID_IP = '0.0.0.0';
+const GODADDY_INVALID_IPv6 = '0:0:0:0:0:0:0:0';
 const GODADDY_INVALID_TXT = '""';
 
 function formatError(response) {
@@ -40,22 +43,21 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.apiSecret === constants.SECRET_PLACEHOLDER) newConfig.apiSecret = currentConfig.apiSecret;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    var records = [ ];
-    values.forEach(function (value) {
-        var record = { ttl: 600 }; // 600 is the min ttl
+    const records = [];
+    for (const value of values) {
+        const record = { ttl: 600 }; // 600 is the min ttl
 
         if (type === 'MX') {
             record.priority = parseInt(value.split(' ')[0], 10);
@@ -65,152 +67,135 @@ function upsert(domainObject, location, type, values, callback) {
         }
 
         records.push(record);
-    });
+    }
 
-    superagent.put(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
-        .set('Authorization', `sso-key ${dnsConfig.apiKey}:${dnsConfig.apiSecret}`)
+    const [error, response] = await safe(superagent.put(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
+        .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
         .timeout(30 * 1000)
         .send(records)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, formatError(result))); // no such zone
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.BAD_FIELD, formatError(result))); // conflict
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            return callback(null);
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response)); // no such zone
+    if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, formatError(response)); // conflict
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type}`);
 
-    superagent.get(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
-        .set('Authorization', `sso-key ${dnsConfig.apiKey}:${dnsConfig.apiSecret}`)
+    const [error, response] = await safe(superagent.get(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
+        .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
         .timeout(30 * 1000)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode === 404) return callback(null, [ ]);
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            debug('get: %j', result.body);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode === 404) return [];
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-            var values = result.body.map(function (record) { return record.data; });
+    const values = response.body.map(function (record) { return record.data; });
 
-            if (values.length === 1 && values[0] === GODADDY_INVALID_IP) return callback(null, [ ]); // pretend this record doesn't exist
+    if (values.length === 1) {
+        if ((type === 'A' && values[0] === GODADDY_INVALID_IP)
+            || (type === 'AAAA' && values[0] === GODADDY_INVALID_IPv6)
+            || (type === 'TXT' && values[0] === GODADDY_INVALID_TXT)) return []; // pretend this record doesn't exist
+    }
 
-            return callback(null, values);
-        });
+    return values;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Record deletion is not supported by GoDaddy API'));
+    if (type !== 'A' && type !== 'AAAA' && type !== 'TXT') throw new BoxError(BoxError.EXTERNAL_ERROR, 'Record deletion is not supported by GoDaddy API');
 
     // check if the record exists at all so that we don't insert the "Dead" record for no reason
-    get(domainObject, location, type, function (error, values) {
-        if (error) return callback(error);
-        if (values.length === 0) return callback();
+    const existingRecords = await get(domainObject, location, type);
+    if (existingRecords.length === 0) return;
 
-        // godaddy does not have a delete API. so fill it up with an invalid IP that we can ignore in future get()
-        var records = [{
-            ttl: 600,
-            data: type === 'A' ? GODADDY_INVALID_IP : GODADDY_INVALID_TXT
-        }];
+    // godaddy does not have a delete API. so fill it up with an invalid IP that we can ignore in future get()
+    const records = [{
+        ttl: 600,
+        data: type === 'A' ? GODADDY_INVALID_IP : (type === 'AAAA' ? GODADDY_INVALID_IPv6 : GODADDY_INVALID_TXT)
+    }];
 
-        superagent.put(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
-            .set('Authorization', `sso-key ${dnsConfig.apiKey}:${dnsConfig.apiSecret}`)
-            .send(records)
-            .timeout(30 * 1000)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return callback(null);
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+    const [error, response] = await safe(superagent.put(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
+        .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
+        .send(records)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-                debug('del: done');
-
-                return callback(null);
-            });
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 404) return;
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.apiKey || typeof dnsConfig.apiKey !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string', { field: 'apiKey' }));
-    if (!dnsConfig.apiSecret || typeof dnsConfig.apiSecret !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiSecret must be a non-empty string', { field: 'apiSecret' }));
+    if (!domainConfig.apiKey || typeof domainConfig.apiKey !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string');
+    if (!domainConfig.apiSecret || typeof domainConfig.apiSecret !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'apiSecret must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        apiKey: dnsConfig.apiKey,
-        apiSecret: dnsConfig.apiSecret
+    const credentials = {
+        apiKey: domainConfig.apiKey,
+        apiSecret: domainConfig.apiSecret
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1; })) {
-            debug('verifyDnsConfig: %j does not contain GoDaddy NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to GoDaddy', { field: 'nameservers' }));
-        }
+    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1; })) {
+        debug('verifyDomainConfig: %j does not contain GoDaddy NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to GoDaddy');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/interface.js

@@ -13,7 +13,7 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
@@ -29,57 +29,50 @@ function injectPrivateFields(newConfig, currentConfig) {
     // in-place injection of tokens and api keys which came in with constants.SECRET_PLACEHOLDER
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, subdomain, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     // Result: none
 
-    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upsert is not implemented'));
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'upsert is not implemented');
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, subdomain, type) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // Result: Array of matching DNS records in string format
 
-    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'get is not implemented'));
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'get is not implemented');
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, subdomain, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     // Result: none
 
-    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'del is not implemented'));
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'del is not implemented');
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
-
-    callback();
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    // Result: dnsConfig object
+    // Result: domainConfig object
 
-    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'verifyDnsConfig is not implemented'));
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'verifyDomainConfig is not implemented');
 }

src/dns/linode.js

@@ -7,15 +7,16 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
-const async = require('async'),
-    assert = require('assert'),
+const assert = require('assert'),
     constants = require('../constants.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/linode'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -35,278 +36,232 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function getZoneId(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneId(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // returns 100 at a time
-    superagent.get(`${LINODE_ENDPOINT}/domains`)
-        .set('Authorization', 'Bearer ' + dnsConfig.token)
+    const [error, response] = await safe(superagent.get(`${LINODE_ENDPOINT}/domains`)
+        .set('Authorization', 'Bearer ' + domainConfig.token)
         .timeout(30 * 1000)
         .retry(5)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            if (!Array.isArray(result.body.data)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-            const zone = result.body.data.find(d => d.domain === zoneName);
+    if (!Array.isArray(response.body.data)) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response');
 
-            if (!zone || !zone.id) return callback(new BoxError(BoxError.NOT_FOUND, 'Zone not found'));
+    const zone = response.body.data.find(d => d.domain === zoneName);
 
-            debug(`getZoneId: zone id of ${zoneName} is ${zone.id}`);
+    if (!zone || !zone.id) throw new BoxError(BoxError.NOT_FOUND, 'Zone not found');
 
-            callback(null, zone.id);
-        });
+    debug(`getZoneId: zone id of ${zoneName} is ${zone.id}`);
+
+    return zone.id;
 }
 
-function getZoneRecords(dnsConfig, zoneName, name, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneRecords(domainConfig, zoneName, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
 
-    getZoneId(dnsConfig, zoneName, function (error, zoneId) {
-        if (error) return callback(error);
+    const zoneId = await getZoneId(domainConfig, zoneName);
 
-        let page = 0, more = false;
-        let records = [];
+    let page = 0, more = false;
+    let records = [];
 
-        async.doWhilst(function (iteratorDone) {
-            const url = `${LINODE_ENDPOINT}/domains/${zoneId}/records?page=${++page}`;
+    do {
+        const url = `${LINODE_ENDPOINT}/domains/${zoneId}/records?page=${++page}`;
 
-            superagent.get(url)
-                .set('Authorization', 'Bearer ' + dnsConfig.token)
-                .timeout(30 * 1000)
-                .retry(5)
-                .end(function (error, result) {
-                    if (error && !error.response) return iteratorDone(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                    if (result.statusCode === 404) return iteratorDone(new BoxError(BoxError.NOT_FOUND, formatError(result)));
-                    if (result.statusCode === 403 || result.statusCode === 401) return iteratorDone(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                    if (result.statusCode !== 200) return iteratorDone(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        const [error, response] = await safe(superagent.get(url)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
 
-                    records = records.concat(result.body.data.filter(function (record) {
-                        return (record.type === type && record.name === name);
-                    }));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                    more = result.body.page !== result.body.pages;
+        records = records.concat(response.body.data.filter(function (record) {
+            return (record.type === type && record.name === name);
+        }));
 
-                    iteratorDone();
-                });
-        }, function (testDone) { return testDone(null, more); }, function (error) {
-            debug('getZoneRecords:', error, JSON.stringify(records));
+        more = response.body.page !== response.body.pages;
+    } while (more);
 
-            if (error) return callback(error);
-
-            callback(null, { zoneId, records });
-        });
-    });
+    return { zoneId, records };
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
-
-        const { records } = result;
-        var tmp = records.map(function (record) { return record.target; });
-
-        debug('get: %j', tmp);
-
-        return callback(null, tmp);
-    });
+    const { records } = await getZoneRecords(domainConfig, zoneName, name, type);
+    const tmp = records.map(function (record) { return record.target; });
+    return tmp;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const { zoneId, records } = await getZoneRecords(domainConfig, zoneName, name, type);
+    let i = 0, recordIds = []; // used to track available records to update instead of create
 
-        const { zoneId, records } = result;
-        let i = 0, recordIds = []; // used to track available records to update instead of create
+    for (const value of values) {
+        const data = {
+            type: type,
+            ttl_sec: 300 // lowest
+        };
 
-        async.eachSeries(values, function (value, iteratorCallback) {
-            let data = {
-                type: type,
-                ttl_sec: 300 // lowest
-            };
+        if (type === 'MX') {
+            data.priority = parseInt(value.split(' ')[0], 10);
+            data.target = value.split(' ')[1];
+        } else if (type === 'TXT') {
+            data.target = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
+        } else {
+            data.target = value;
+        }
 
-            if (type === 'MX') {
-                data.priority = parseInt(value.split(' ')[0], 10);
-                data.target = value.split(' ')[1];
-            } else if (type === 'TXT') {
-                data.target = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
-            } else {
-                data.target = value;
-            }
+        if (i >= records.length) {
+            data.name = name; // only set for new records
 
-            if (i >= records.length) {
-                data.name = name; // only set for new records
+            const [error, response] = await safe(superagent.post(`${LINODE_ENDPOINT}/domains/${zoneId}/records`)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                superagent.post(`${LINODE_ENDPOINT}/domains/${zoneId}/records`)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode !== 200) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        recordIds.push(result.body.id);
+            recordIds.push(response.body.id);
+        } else {
+            const [error, response] = await safe(superagent.put(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${records[i].id}`)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        return iteratorCallback(null);
-                    });
-            } else {
-                superagent.put(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${records[i].id}`)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        // increment, as we have consumed the record
-                        ++i;
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode !== 200) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            ++i;
 
-                        recordIds.push(result.body.id);
+            recordIds.push(response.body.id);
+        }
+    }
 
-                        return iteratorCallback(null);
-                    });
-            }
-        }, function (error) {
-            if (error) return callback(error);
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] =  await safe(superagent.del(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${records[j].id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
 
-            debug('upsert: completed with recordIds:%j', recordIds);
-
-            callback();
-        });
-    });
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const { zoneId, records } = await getZoneRecords(domainConfig, zoneName, name, type);
+    if (records.length === 0) return;
 
-        const { zoneId, records } = result;
-        if (records.length === 0) return callback(null);
+    const tmp = records.filter(function (record) { return values.some(function (value) { return value === record.target; }); });
+    if (tmp.length === 0) return;
 
-        var tmp = records.filter(function (record) { return values.some(function (value) { return value === record.target; }); });
-
-        debug('del: %j', tmp);
-
-        if (tmp.length === 0) return callback(null);
-
-        // FIXME we only handle the first one currently
-
-        superagent.del(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${tmp[0].id}`)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
+    for (const r of tmp) {
+        const [error, response] = await safe(superagent.del(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${r.id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
             .timeout(30 * 1000)
             .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return callback(null);
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                debug('del: done');
-
-                return callback(null);
-            });
-    });
+            .ok(() => true));
+        if (error && !error.response) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) return;
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        token: dnsConfig.token
+    const credentials = {
+        token: domainConfig.token
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) {
-            debug('verifyDnsConfig: %j does not contains linode NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode', { field: 'nameservers' }));
-        }
+    if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) {
+        debug('verifyDomainConfig: %j does not contains linode NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/manual.js

@@ -1,19 +1,21 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/manual'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
+    safe = require('safetydance'),
     waitForDns = require('./waitfordns.js');
 
 function removePrivateFields(domainObject) {
@@ -25,61 +27,55 @@ function injectPrivateFields(newConfig, currentConfig) {
 
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
 
-    return callback(null);
+    return;
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    callback(null, [ ]); // returning ip confuses apptask into thinking the entry already exists
+    return []; // returning ip confuses apptask into thinking the entry already exists
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    return callback();
+    return;
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     const zoneName = domainObject.zoneName;
 
     // Very basic check if the nameservers can be fetched
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        callback(null, {});
-    });
+    return {};
 }

src/dns/namecheap.js

@@ -6,7 +6,7 @@ exports = module.exports = {
     upsert,
     get,
     del,
-    verifyDnsConfig,
+    verifyDomainConfig,
     wait
 };
 
@@ -14,8 +14,8 @@ const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecheap'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
-    querystring = require('querystring'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     sysinfo = require('../sysinfo.js'),
@@ -34,286 +34,247 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-async function getQuery(dnsConfig) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getQuery(domainConfig) {
+    assert.strictEqual(typeof domainConfig, 'object');
 
-    const ip = await sysinfo.getServerIp();
+    const ip = await sysinfo.getServerIPv4(); // only supports ipv4
 
     return {
-        ApiUser: dnsConfig.username,
-        ApiKey: dnsConfig.token,
-        UserName: dnsConfig.username,
+        ApiUser: domainConfig.username,
+        ApiKey: domainConfig.token,
+        UserName: domainConfig.username,
         ClientIp: ip
     };
 }
 
-function getZone(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZone(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    util.callbackify(getQuery)(dnsConfig, function (error, query) {
-        if (error) return callback(error);
+    const query = await getQuery(domainConfig);
+    query.Command = 'namecheap.domains.dns.getHosts';
+    query.SLD = zoneName.split('.')[0];
+    query.TLD = zoneName.split('.')[1];
 
-        query.Command = 'namecheap.domains.dns.getHosts';
-        query.SLD = zoneName.split('.')[0];
-        query.TLD = zoneName.split('.')[1];
+    const [error, response] = await safe(superagent.get(ENDPOINT).query(query).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
 
-        superagent.get(ENDPOINT).query(query).end(function (error, result) {
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+    const parser = new xml2js.Parser();
+    const [parserError, result] = await safe(util.promisify(parser.parseString)(response.text));
+    if (parserError) throw new BoxError(BoxError.EXTERNAL_ERROR, parserError);
 
-            var parser = new xml2js.Parser();
-            parser.parseString(result.text, function (error, result) {
-                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+    const tmp = result.ApiResponse;
+    if (tmp['$'].Status !== 'OK') {
+        const errorMessage = safe.query(tmp, 'Errors[0].Error[0]._', 'Invalid response');
+        if (errorMessage === 'API Key is invalid or API access has not been enabled') throw new BoxError(BoxError.ACCESS_DENIED, errorMessage);
 
-                var tmp = result.ApiResponse;
-                if (tmp['$'].Status !== 'OK') {
-                    var errorMessage = safe.query(tmp, 'Errors[0].Error[0]._', 'Invalid response');
-                    if (errorMessage === 'API Key is invalid or API access has not been enabled') return callback(new BoxError(BoxError.ACCESS_DENIED, errorMessage));
+        throw new BoxError(BoxError.EXTERNAL_ERROR, errorMessage);
+    }
+    const host = safe.query(tmp, 'CommandResponse[0].DomainDNSGetHostsResult[0].host');
+    if (!host) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response: ${JSON.stringify(tmp)}`);
+    if (!Array.isArray(host)) throw new BoxError(BoxError.EXTERNAL_ERROR, `host is not an array: ${JSON.stringify(tmp)}`);
 
-                    return callback(new BoxError(BoxError.EXTERNAL_ERROR, errorMessage));
-                }
-                const host = safe.query(tmp, 'CommandResponse[0].DomainDNSGetHostsResult[0].host');
-                if (!host) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response: ${JSON.stringify(tmp)}`));
-                if (!Array.isArray(host)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `host is not an array: ${JSON.stringify(tmp)}`));
-
-                const hosts = host.map(h => h['$']);
-                callback(null, hosts);
-            });
-        });
-    });
+    const hosts = host.map(h => h['$']);
+    return hosts;
 }
 
-function setZone(dnsConfig, zoneName, hosts, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function setZone(domainConfig, zoneName, hosts) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert(Array.isArray(hosts));
-    assert.strictEqual(typeof callback, 'function');
 
-    util.callbackify(getQuery)(dnsConfig, function (error, query) {
-        if (error) return callback(error);
+    const query = await getQuery(domainConfig);
+    query.Command = 'namecheap.domains.dns.setHosts';
+    query.SLD = zoneName.split('.')[0];
+    query.TLD = zoneName.split('.')[1];
 
-        query.Command = 'namecheap.domains.dns.setHosts';
-        query.SLD = zoneName.split('.')[0];
-        query.TLD = zoneName.split('.')[1];
+    // Map to query params https://www.namecheap.com/support/api/methods/domains-dns/set-hosts.aspx
+    hosts.forEach(function (host, i) {
+        var n = i+1; // api starts with 1 not 0
+        query['TTL' + n] = '300'; // keep it low
+        query['HostName' + n] = host.HostName || host.Name;
+        query['RecordType' + n] = host.RecordType || host.Type;
+        query['Address' + n] = host.Address;
 
-        // Map to query params https://www.namecheap.com/support/api/methods/domains-dns/set-hosts.aspx
-        hosts.forEach(function (host, i) {
-            var n = i+1; // api starts with 1 not 0
-            query['TTL' + n] = '300'; // keep it low
-            query['HostName' + n] = host.HostName || host.Name;
-            query['RecordType' + n] = host.RecordType || host.Type;
-            query['Address' + n] = host.Address;
-
-            if (host.Type === 'MX') {
-                query['EmailType' + n] = 'MX';
-                if (host.MXPref) query['MXPref' + n] = host.MXPref;
-            }
-        });
-
-        // namecheap recommends sending as POSTDATA with > 10 records
-        const qs = querystring.stringify(query);
-
-        superagent.post(ENDPOINT).set('Content-Type', 'application/x-www-form-urlencoded').send(qs).end(function (error, result) {
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-            var parser = new xml2js.Parser();
-            parser.parseString(result.text, function (error, result) {
-                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-                var tmp = result.ApiResponse;
-                if (tmp['$'].Status !== 'OK') {
-                    var errorMessage = safe.query(tmp, 'Errors[0].Error[0]._', 'Invalid response');
-                    if (errorMessage === 'API Key is invalid or API access has not been enabled') return callback(new BoxError(BoxError.ACCESS_DENIED, errorMessage));
-
-                    return callback(new BoxError(BoxError.EXTERNAL_ERROR, errorMessage));
-                }
-                if (!tmp.CommandResponse[0]) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
-                if (!tmp.CommandResponse[0].DomainDNSSetHostsResult[0]) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
-                if (tmp.CommandResponse[0].DomainDNSSetHostsResult[0]['$'].IsSuccess !== 'true') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
-
-                callback(null);
-            });
-        });
+        if (host.Type === 'MX') {
+            query['EmailType' + n] = 'MX';
+            if (host.MXPref) query['MXPref' + n] = host.MXPref;
+        }
     });
+
+    // namecheap recommends sending as POSTDATA with > 10 records
+    const qs = new URLSearchParams(query).toString();
+
+    const [error, response] = await safe(superagent.post(ENDPOINT).set('Content-Type', 'application/x-www-form-urlencoded').send(qs).ok(() => true));
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error);
+
+    const parser = new xml2js.Parser();
+    const [parserError, result] = await safe(util.promisify(parser.parseString)(response.text));
+    if (parserError) throw new BoxError(BoxError.EXTERNAL_ERROR, parserError.message);
+
+    const tmp = result.ApiResponse;
+    if (tmp['$'].Status !== 'OK') {
+        const errorMessage = safe.query(tmp, 'Errors[0].Error[0]._', 'Invalid response');
+        if (errorMessage === 'API Key is invalid or API access has not been enabled') throw new BoxError(BoxError.ACCESS_DENIED, errorMessage);
+
+        throw new BoxError(BoxError.EXTERNAL_ERROR, errorMessage);
+    }
+    if (!tmp.CommandResponse[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response');
+    if (!tmp.CommandResponse[0].DomainDNSSetHostsResult[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response');
+    if (tmp.CommandResponse[0].DomainDNSSetHostsResult[0]['$'].IsSuccess !== 'true') throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response');
 }
 
-function upsert(domainObject, subdomain, type, values, callback) {
+async function upsert(domainObject, subdomain, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config;
+    const domainConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
     subdomain = dns.getName(domainObject, subdomain, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getZone(dnsConfig, zoneName, function (error, result) {
-        if (error) return callback(error);
+    const result = await getZone(domainConfig, zoneName);
 
-        // Array to keep track of records that need to be inserted
-        let toInsert = [];
+    // Array to keep track of records that need to be inserted
+    let toInsert = [];
 
-        for (let i = 0; i < values.length; i++) {
-            let curValue = values[i];
-            let wasUpdate = false;
+    for (let i = 0; i < values.length; i++) {
+        let curValue = values[i];
+        let wasUpdate = false;
 
-            for (let j = 0; j < result.length; j++) {
-                let curHost = result[j];
+        for (let j = 0; j < result.length; j++) {
+            let curHost = result[j];
 
-                if (curHost.Type === type && curHost.Name === subdomain) {
-                    // Updating an already existing host
-                    wasUpdate = true;
-                    if (type === 'MX') {
-                        curHost.MXPref = curValue.split(' ')[0];
-                        curHost.Address = curValue.split(' ')[1];
-                    } else {
-                        curHost.Address = curValue;
-                    }
-                }
-            }
-
-            // We don't have this host at all yet, let's push to toInsert array
-            if (!wasUpdate) {
-                let newRecord = {
-                    RecordType: type,
-                    HostName: subdomain,
-                    Address: curValue
-                };
-
-                // Special case for MX records
+            if (curHost.Type === type && curHost.Name === subdomain) {
+                // Updating an already existing host
+                wasUpdate = true;
                 if (type === 'MX') {
-                    newRecord.MXPref = curValue.split(' ')[0];
-                    newRecord.Address = curValue.split(' ')[1];
+                    curHost.MXPref = curValue.split(' ')[0];
+                    curHost.Address = curValue.split(' ')[1];
+                } else {
+                    curHost.Address = curValue;
                 }
-
-                toInsert.push(newRecord);
-
             }
         }
 
-        const hosts = result.concat(toInsert);
+        // We don't have this host at all yet, let's push to toInsert array
+        if (!wasUpdate) {
+            let newRecord = {
+                RecordType: type,
+                HostName: subdomain,
+                Address: curValue
+            };
 
-        setZone(dnsConfig, zoneName, hosts, callback);
-    });
+            // Special case for MX records
+            if (type === 'MX') {
+                newRecord.MXPref = curValue.split(' ')[0];
+                newRecord.Address = curValue.split(' ')[1];
+            }
+
+            toInsert.push(newRecord);
+
+        }
+    }
+
+    const hosts = result.concat(toInsert);
+
+    return await setZone(domainConfig, zoneName, hosts);
 }
 
-function get(domainObject, subdomain, type, callback) {
+async function get(domainObject, subdomain, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config;
+    const domainConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
     subdomain = dns.getName(domainObject, subdomain, type) || '@';
 
-    getZone(dnsConfig, zoneName, function (error, result) {
-        if (error) return callback(error);
+    const result = await getZone(domainConfig, zoneName);
 
-        // We need to filter hosts to ones with this subdomain and type
-        const actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
+    // We need to filter hosts to ones with this subdomain and type
+    const actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
 
-        // We only return the value string
-        const tmp = actualHosts.map(function (record) { return record.Address; });
-
-        debug(`get: subdomain: ${subdomain} type:${type} value:${JSON.stringify(tmp)}`);
-
-        return callback(null, tmp);
-    });
+    const tmp = actualHosts.map(function (record) { return record.Address; });
+    return tmp;
 }
 
-function del(domainObject, subdomain, type, values, callback) {
+async function del(domainObject, subdomain, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config;
+    const domainConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
     subdomain = dns.getName(domainObject, subdomain, type) || '@';
 
     debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getZone(dnsConfig, zoneName, function (error, result) {
-        if (error) return callback(error);
+    let result = await getZone(domainConfig, zoneName);
+    if (result.length === 0) return;
+    const originalLength = result.length;
 
-        if (result.length === 0) return callback();
-        const originalLength = result.length;
+    for (let i = 0; i < values.length; i++) {
+        let curValue = values[i];
 
-        for (let i = 0; i < values.length; i++) {
-            let curValue = values[i];
+        result = result.filter(curHost => curHost.Type !== type || curHost.Name !== subdomain || curHost.Address !== curValue);
+    }
 
-            result = result.filter(curHost => curHost.Type !== type || curHost.Name !== subdomain || curHost.Address !== curValue);
-        }
-
-        if (result.length !== originalLength) return setZone(dnsConfig, zoneName, result, callback);
-
-        callback();
-    });
+    if (result.length !== originalLength) return await setZone(domainConfig, zoneName, result);
 }
 
-function verifyDnsConfig(domainObject, callback) {
-    assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const dnsConfig = domainObject.config;
-    const zoneName = domainObject.zoneName;
-    const ip = '127.0.0.1';
-
-    if (!dnsConfig.username || typeof dnsConfig.username !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'username must be a non-empty string', { field: 'username' }));
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
-
-    let credentials = {
-        username: dnsConfig.username,
-        token: dnsConfig.token
-    };
-
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
-
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
-
-        if (nameservers.some(function (n) { return n.toLowerCase().indexOf('.registrar-servers.com') === -1; })) {
-            debug('verifyDnsConfig: %j does not contains NC NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to NameCheap', { field: 'nameservers' }));
-        }
-
-        const testSubdomain = 'cloudrontestdns';
-
-        upsert(domainObject, testSubdomain, 'A', [ip], function (error, changeId) {
-            if (error) return callback(error);
-
-            debug('verifyDnsConfig: Test A record added with change id %s', changeId);
-
-            del(domainObject, testSubdomain, 'A', [ip], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
-}
-
-function wait(domainObject, subdomain, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
     const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
+}
+
+async function verifyDomainConfig(domainObject) {
+    assert.strictEqual(typeof domainObject, 'object');
+
+    const domainConfig = domainObject.config;
+    const zoneName = domainObject.zoneName;
+    const ip = '127.0.0.1';
+
+    if (!domainConfig.username || typeof domainConfig.username !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'username must be a non-empty string');
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
+
+    const credentials = {
+        username: domainConfig.username,
+        token: domainConfig.token
+    };
+
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
+
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
+
+    if (nameservers.some(function (n) { return n.toLowerCase().indexOf('.registrar-servers.com') === -1; })) {
+        debug('verifyDomainConfig: %j does not contains NC NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to NameCheap');
+    }
+
+    const testSubdomain = 'cloudrontestdns';
+
+    await upsert(domainObject, testSubdomain, 'A', [ip]);
+    debug('verifyDomainConfig: Test A record added');
+
+    await del(domainObject, testSubdomain, 'A', [ip]);
+    debug('verifyDomainConfig: Test A record removed again');
+
+    return credentials;
 }

src/dns/namecom.js

@@ -7,13 +7,14 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecom'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
@@ -34,17 +35,16 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function addRecord(dnsConfig, zoneName, name, type, values, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function addRecord(domainConfig, zoneName, name, type, values) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`add: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    var data = {
+    const data = {
         host: name,
         type: type,
         ttl: 300    // 300 is the lowest
@@ -61,31 +61,28 @@ function addRecord(dnsConfig, zoneName, name, type, values, callback) {
         data.answer = values[0];
     }
 
-    superagent.post(`${NAMECOM_API}/domains/${zoneName}/records`)
-        .auth(dnsConfig.username, dnsConfig.token)
+    const [error, response] = await safe(superagent.post(`${NAMECOM_API}/domains/${zoneName}/records`)
+        .auth(domainConfig.username, domainConfig.token)
         .timeout(30 * 1000)
         .send(data)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            return callback(null, 'unused-id');
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function updateRecord(dnsConfig, zoneName, recordId, name, type, values, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function updateRecord(domainConfig, zoneName, recordId, name, type, values) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof recordId, 'number');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`update:${recordId} on ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    var data = {
+    const data = {
         host: name,
         type: type,
         ttl: 300    // 300 is the lowest
@@ -102,184 +99,152 @@ function updateRecord(dnsConfig, zoneName, recordId, name, type, values, callbac
         data.answer = values[0];
     }
 
-    superagent.put(`${NAMECOM_API}/domains/${zoneName}/records/${recordId}`)
-        .auth(dnsConfig.username, dnsConfig.token)
+    const [error, response] = await safe(superagent.put(`${NAMECOM_API}/domains/${zoneName}/records/${recordId}`)
+        .auth(domainConfig.username, domainConfig.token)
         .timeout(30 * 1000)
         .send(data)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            return callback(null);
-        });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function getInternal(dnsConfig, zoneName, name, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getInternal(domainConfig, zoneName, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`getInternal: ${name} in zone ${zoneName} of type ${type}`);
 
-    superagent.get(`${NAMECOM_API}/domains/${zoneName}/records`)
-        .auth(dnsConfig.username, dnsConfig.token)
+    const [error, response] = await safe(superagent.get(`${NAMECOM_API}/domains/${zoneName}/records`)
+        .auth(domainConfig.username, domainConfig.token)
         .timeout(30 * 1000)
-        .end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        .ok(() => true));
 
-            // name.com does not return the correct content-type
-            result.body = safe.JSON.parse(result.text);
-            if (!result.body.records) result.body.records = [];
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-            result.body.records.forEach(function (r) {
-                // name.com api simply strips empty properties
-                r.host = r.host || '';
-            });
+    // name.com does not return the correct content-type
+    response.body = safe.JSON.parse(response.text);
+    if (!response.body.records) response.body.records = [];
 
-            var results = result.body.records.filter(function (r) {
-                return (r.host === name && r.type === type);
-            });
+    response.body.records.forEach(function (r) {
+        // name.com api simply strips empty properties
+        r.host = r.host || '';
+    });
 
-            debug('getInternal: %j', results);
+    const results = response.body.records.filter(function (r) {
+        return (r.host === name && r.type === type);
+    });
 
-            return callback(null, results);
-        });
+    return results;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const result = await getInternal(domainConfig, zoneName, name, type);
+    if (result.length === 0) return await addRecord(domainConfig, zoneName, name, type, values);
 
-        if (result.length === 0) return addRecord(dnsConfig, zoneName, name, type, values, callback);
-
-        return updateRecord(dnsConfig, zoneName, result[0].id, name, type, values, callback);
-    });
+    return await updateRecord(domainConfig, zoneName, result[0].id, name, type, values);
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
-
-        var tmp = result.map(function (record) { return record.answer; });
-
-        debug('get: %j', tmp);
-
-        return callback(null, tmp);
-    });
+    const result = await getInternal(domainConfig, zoneName, name, type);
+    const tmp = result.map(function (record) { return record.answer; });
+    return tmp;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
     debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    getInternal(dnsConfig, zoneName, name, type, function (error, result) {
-        if (error) return callback(error);
+    const result = await getInternal(domainConfig, zoneName, name, type);
+    if (result.length === 0) return;
 
-        if (result.length === 0) return callback();
-
-        superagent.del(`${NAMECOM_API}/domains/${zoneName}/records/${result[0].id}`)
-            .auth(dnsConfig.username, dnsConfig.token)
-            .timeout(30 * 1000)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                return callback(null);
-            });
-    });
+    const [error, response] = await safe(superagent.del(`${NAMECOM_API}/domains/${zoneName}/records/${result[0].id}`)
+        .auth(domainConfig.username, domainConfig.token)
+        .timeout(30 * 1000)
+        .ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (typeof dnsConfig.username !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'username must be a string', { field: 'username' }));
-    if (typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a string', { field: 'token' }));
+    if (typeof domainConfig.username !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'username must be a string');
+    if (typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a string');
 
-    var credentials = {
-        username: dnsConfig.username,
-        token: dnsConfig.token
+    const credentials = {
+        username: domainConfig.username,
+        token: domainConfig.token
     };
 
     const ip = '127.0.0.1';
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.name.com') !== -1; })) {
-            debug('verifyDnsConfig: %j does not contain Name.com NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to name.com', { field: 'nameservers' }));
-        }
+    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.name.com') !== -1; })) {
+        debug('verifyDomainConfig: %j does not contain Name.com NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to name.com');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/netcup.js

@@ -7,14 +7,16 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/netcup'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
+    safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -36,267 +38,226 @@ function injectPrivateFields(newConfig, currentConfig) {
 }
 
 // returns a api session id
-function login(dnsConfig, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
+async function login(domainConfig) {
+    assert.strictEqual(typeof domainConfig, 'object');
 
     const data = {
         action: 'login',
         param:{
-            apikey: dnsConfig.apiKey,
-            apipassword: dnsConfig.apiPassword,
-            customernumber: dnsConfig.customerNumber
+            apikey: domainConfig.apiKey,
+            apipassword: domainConfig.apiPassword,
+            customernumber: domainConfig.customerNumber
         }
     };
 
-    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+    const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    if (!response.body.responsedata.apisessionid) throw new BoxError(BoxError.ACCESS_DENIED, 'invalid api password');
 
-        callback(null, result.body.responsedata.apisessionid);
-    });
+    return response.body.responsedata.apisessionid;
 }
 
-function getAllRecords(dnsConfig, apiSessionId, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getAllRecords(domainConfig, apiSessionId, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof apiSessionId, 'string');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`getAllRecords: getting dns records of ${zoneName}`);
 
     const data = {
         action: 'infoDnsRecords',
         param:{
-            apikey: dnsConfig.apiKey,
+            apikey: domainConfig.apiKey,
             apisessionid: apiSessionId,
-            customernumber: dnsConfig.customerNumber,
+            customernumber: domainConfig.customerNumber,
             domainname: zoneName,
         }
     };
 
-    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+    const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-        debug('getAllRecords:', JSON.stringify(result.body.responsedata.dnsrecords || []));
-
-        callback(null, result.body.responsedata.dnsrecords || []);
-    });
+    return response.body.responsedata.dnsrecords || [];
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    login(dnsConfig, function (error, apiSessionId) {
-        if (error) return callback(error);
+    const apiSessionId = await login(domainConfig);
 
-        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
-            if (error) return callback(error);
+    const result = await getAllRecords(domainConfig, apiSessionId, zoneName);
 
-            let records = [];
+    let records = [];
 
-            values.forEach(function (value) {
-                // remove possible quotation
-                if (value.charAt(0) === '"') value = value.slice(1);
-                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
+    values.forEach(function (value) {
+        // remove possible quotation
+        if (value.charAt(0) === '"') value = value.slice(1);
+        if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
 
-                let priority = null;
-                if (type === 'MX') {
-                    priority = parseInt(value.split(' ')[0], 10);
-                    value = value.split(' ')[1];
-                }
+        let priority = null;
+        if (type === 'MX') {
+            priority = parseInt(value.split(' ')[0], 10);
+            value = value.split(' ')[1];
+        }
 
-                let record = result.find(function (r) { return r.hostname === name && r.type === type; });
-                if (!record) record = { hostname: name, type: type, destination: value, deleterecord: false };
-                else record.destination = value;
+        let record = result.find(function (r) { return r.hostname === name && r.type === type; });
+        if (!record) record = { hostname: name, type: type, destination: value, deleterecord: false };
+        else record.destination = value;
 
-                if (priority !== null) record.priority = priority;
+        if (priority !== null) record.priority = priority;
 
-                records.push(record);
-            });
-
-            const data = {
-                action: 'updateDnsRecords',
-                param:{
-                    apikey: dnsConfig.apiKey,
-                    apisessionid: apiSessionId,
-                    customernumber: dnsConfig.customerNumber,
-                    domainname: zoneName,
-                    dnsrecordset: {
-                        dnsrecords: records
-                    }
-                }
-            };
-
-            debug('upserting', JSON.stringify(data));
-
-            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                debug('upsert:', result.body);
-
-                callback(null);
-            });
-        });
+        records.push(record);
     });
+
+    const data = {
+        action: 'updateDnsRecords',
+        param:{
+            apikey: domainConfig.apiKey,
+            apisessionid: apiSessionId,
+            customernumber: domainConfig.customerNumber,
+            domainname: zoneName,
+            dnsrecordset: {
+                dnsrecords: records
+            }
+        }
+    };
+
+    const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    if (response.body.statuscode !== 2000) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug('get: %s for zone %s of type %s', name, zoneName, type);
 
-    login(dnsConfig, function (error, apiSessionId) {
-        if (error) return callback(error);
+    const apiSessionId = await login(domainConfig);
 
-        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
-            if (error) return callback(error);
+    const result = await getAllRecords(domainConfig, apiSessionId, zoneName);
 
-            // We only return the value string
-            callback(null, result.filter(function (r) { return r.hostname === name && r.type === type; }).map(function (r) { return r.destination; }));
-        });
-    });
+    // We only return the value string
+    return result.filter(function (r) { return r.hostname === name && r.type === type; }).map(function (r) { return r.destination; });
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
     debug('del: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    login(dnsConfig, function (error, apiSessionId) {
-        if (error) return callback(error);
+    const apiSessionId = await login(domainConfig);
 
-        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
-            if (error) return callback(error);
+    const result = await getAllRecords(domainConfig, apiSessionId, zoneName);
 
-            let records = [];
+    let records = [];
 
-            values.forEach(function (value) {
-                // remove possible quotation
-                if (value.charAt(0) === '"') value = value.slice(1);
-                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
+    values.forEach(function (value) {
+        // remove possible quotation
+        if (value.charAt(0) === '"') value = value.slice(1);
+        if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
 
-                let record = result.find(function (r) { return r.hostname === name && r.type === type && r.destination === value; });
-                if (!record) return;
+        let record = result.find(function (r) { return r.hostname === name && r.type === type && r.destination === value; });
+        if (!record) return;
 
-                record.deleterecord = true;
+        record.deleterecord = true;
 
-                records.push(record);
-            });
-
-            if (records.length === 0) return callback(null);
-
-            const data = {
-                action: 'updateDnsRecords',
-                param:{
-                    apikey: dnsConfig.apiKey,
-                    apisessionid: apiSessionId,
-                    customernumber: dnsConfig.customerNumber,
-                    domainname: zoneName,
-                    dnsrecordset: {
-                        dnsrecords: records
-                    }
-                }
-            };
-
-            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                debug('del:', result.body.responsedata);
-
-                callback(null);
-            });
-        });
+        records.push(record);
     });
+
+    if (records.length === 0) return;
+
+    const data = {
+        action: 'updateDnsRecords',
+        param:{
+            apikey: domainConfig.apiKey,
+            apisessionid: apiSessionId,
+            customernumber: domainConfig.customerNumber,
+            domainname: zoneName,
+            dnsrecordset: {
+                dnsrecords: records
+            }
+        }
+    };
+
+    const [error, response] = await safe(superagent.post(API_ENDPOINT).send(data).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    if (response.body.statuscode !== 2000) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.customerNumber || typeof dnsConfig.customerNumber !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'customerNumber must be a non-empty string', { field: 'customerNumber' }));
-    if (!dnsConfig.apiKey || typeof dnsConfig.apiKey !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string', { field: 'apiKey' }));
-    if (!dnsConfig.apiPassword || typeof dnsConfig.apiPassword !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiPassword must be a non-empty string', { field: 'apiPassword' }));
+    if (!domainConfig.customerNumber || typeof domainConfig.customerNumber !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'customerNumber must be a non-empty string');
+    if (!domainConfig.apiKey || typeof domainConfig.apiKey !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string');
+    if (!domainConfig.apiPassword || typeof domainConfig.apiPassword !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'apiPassword must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        customerNumber: dnsConfig.customerNumber,
-        apiKey: dnsConfig.apiKey,
-        apiPassword: dnsConfig.apiPassword,
+    const credentials = {
+        customerNumber: domainConfig.customerNumber,
+        apiKey: domainConfig.apiKey,
+        apiPassword: domainConfig.apiPassword,
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) {
-            debug('verifyDnsConfig: %j does not contains Netcup NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup', { field: 'nameservers' }));
-        }
+    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) {
+        debug('verifyDomainConfig: %j does not contains Netcup NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/noop.js

@@ -7,7 +7,7 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
@@ -21,51 +21,46 @@ function removePrivateFields(domainObject) {
 function injectPrivateFields(newConfig, currentConfig) {
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
 
-    return callback(null);
+    return;
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    callback(null, [ ]); // returning ip confuses apptask into thinking the entry already exists
+    return []; // returning ip confuses apptask into thinking the entry already exists
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    return callback();
+    return;
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, location, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    callback();
+    // do nothing
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    return callback(null, { });
+    return {};
 }

src/dns/route53.js

@@ -7,7 +7,7 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
@@ -15,7 +15,9 @@ const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/route53'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
+    safe = require('safetydance'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
@@ -28,272 +30,233 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.secretAccessKey === constants.SECRET_PLACEHOLDER) newConfig.secretAccessKey = currentConfig.secretAccessKey;
 }
 
-function getDnsCredentials(dnsConfig) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+function getDnsCredentials(domainConfig) {
+    assert.strictEqual(typeof domainConfig, 'object');
 
-    var credentials = {
-        accessKeyId: dnsConfig.accessKeyId,
-        secretAccessKey: dnsConfig.secretAccessKey,
-        region: dnsConfig.region
+    const credentials = {
+        accessKeyId: domainConfig.accessKeyId,
+        secretAccessKey: domainConfig.secretAccessKey,
+        region: domainConfig.region
     };
 
-    if (dnsConfig.endpoint) credentials.endpoint = new AWS.Endpoint(dnsConfig.endpoint);
+    if (domainConfig.endpoint) credentials.endpoint = new AWS.Endpoint(domainConfig.endpoint);
 
     return credentials;
 }
 
-function getZoneByName(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneByName(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
 
     // backward compat for 2.2, where we only required access to "listHostedZones"
     let listHostedZones;
-    if (dnsConfig.listHostedZonesByName) {
-        listHostedZones = route53.listHostedZonesByName.bind(route53, { MaxItems: '1', DNSName: zoneName + '.' });
+    if (domainConfig.listHostedZonesByName) {
+        listHostedZones = route53.listHostedZonesByName({ MaxItems: '1', DNSName: zoneName + '.' }).promise();
     } else {
-        listHostedZones = route53.listHostedZones.bind(route53, {}); // currently, this route does not support > 100 zones
+        listHostedZones = route53.listHostedZones({}).promise(); // currently, this route does not support > 100 zones
     }
 
-    listHostedZones(function (error, result) {
-        if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-        if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    const [error, result] = await safe(listHostedZones);
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 
-        var zone = result.HostedZones.filter(function (zone) {
-            return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
-        })[0];
+    const zone = result.HostedZones.filter(function (zone) {
+        return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
+    })[0];
 
-        if (!zone) return callback(new BoxError(BoxError.NOT_FOUND, 'no such zone'));
+    if (!zone) throw new BoxError(BoxError.NOT_FOUND, 'no such zone');
 
-        callback(null, zone);
-    });
+    return zone;
 }
 
-function getHostedZone(dnsConfig, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getHostedZone(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    getZoneByName(dnsConfig, zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
-        route53.getHostedZone({ Id: zone.Id }, function (error, result) {
-            if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
+    const [error, result] = await safe(route53.getHostedZone({ Id: zone.Id }).promise());
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 
-            callback(null, result);
-        });
-    });
+    return result;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         fqdn = dns.fqdn(location, domainObject);
 
     debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
-    getZoneByName(dnsConfig, zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        var records = values.map(function (v) { return { Value: v }; });  // for mx records, value is already of the '<priority> <server>' format
+    const records = values.map(function (v) { return { Value: v }; });  // for mx records, value is already of the '<priority> <server>' format
 
-        var params = {
-            ChangeBatch: {
-                Changes: [{
-                    Action: 'UPSERT',
-                    ResourceRecordSet: {
-                        Type: type,
-                        Name: fqdn,
-                        ResourceRecords: records,
-                        TTL: 1
-                    }
-                }]
-            },
-            HostedZoneId: zone.Id
-        };
+    const params = {
+        ChangeBatch: {
+            Changes: [{
+                Action: 'UPSERT',
+                ResourceRecordSet: {
+                    Type: type,
+                    Name: fqdn,
+                    ResourceRecords: records,
+                    TTL: 1
+                }
+            }]
+        },
+        HostedZoneId: zone.Id
+    };
 
-        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
-        route53.changeResourceRecordSets(params, function(error) {
-            if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'PriorRequestNotComplete') return callback(new BoxError(BoxError.BUSY, error.message));
-            if (error && error.code === 'InvalidChangeBatch') return callback(new BoxError(BoxError.BAD_FIELD, error.message));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-
-            callback(null);
-        });
-    });
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
+    const [error] = await safe(route53.changeResourceRecordSets(params).promise());
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'PriorRequestNotComplete') throw new BoxError(BoxError.BUSY, error.message);
+    if (error && error.code === 'InvalidChangeBatch') throw new BoxError(BoxError.BAD_FIELD, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(dnsConfig, zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        var params = {
-            HostedZoneId: zone.Id,
-            MaxItems: '1',
-            StartRecordName: fqdn + '.',
-            StartRecordType: type
-        };
+    const params = {
+        HostedZoneId: zone.Id,
+        MaxItems: '1',
+        StartRecordName: fqdn + '.',
+        StartRecordType: type
+    };
 
-        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
-        route53.listResourceRecordSets(params, function (error, result) {
-            if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-            if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
-            if (result.ResourceRecordSets[0].Name !== params.StartRecordName || result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
+    const [error, result] = await safe(route53.listResourceRecordSets(params).promise());
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
+    if (result.ResourceRecordSets.length === 0) return [];
+    if (result.ResourceRecordSets[0].Name !== params.StartRecordName || result.ResourceRecordSets[0].Type !== params.StartRecordType) return [];
 
-            var values = result.ResourceRecordSets[0].ResourceRecords.map(function (record) { return record.Value; });
-
-            callback(null, values);
-        });
-    });
+    const values = result.ResourceRecordSets[0].ResourceRecords.map(function (record) { return record.Value; });
+    return values;
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         fqdn = dns.fqdn(location, domainObject);
 
-    getZoneByName(dnsConfig, zoneName, function (error, zone) {
-        if (error) return callback(error);
+    const zone = await getZoneByName(domainConfig, zoneName);
 
-        var records = values.map(function (v) { return { Value: v }; });
+    const records = values.map(function (v) { return { Value: v }; });
 
-        var resourceRecordSet = {
-            Name: fqdn,
-            Type: type,
-            ResourceRecords: records,
-            TTL: 1
-        };
+    const resourceRecordSet = {
+        Name: fqdn,
+        Type: type,
+        ResourceRecords: records,
+        TTL: 1
+    };
 
-        var params = {
-            ChangeBatch: {
-                Changes: [{
-                    Action: 'DELETE',
-                    ResourceRecordSet: resourceRecordSet
-                }]
-            },
-            HostedZoneId: zone.Id
-        };
+    const params = {
+        ChangeBatch: {
+            Changes: [{
+                Action: 'DELETE',
+                ResourceRecordSet: resourceRecordSet
+            }]
+        },
+        HostedZoneId: zone.Id
+    };
 
-        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
-        route53.changeResourceRecordSets(params, function(error) {
-            if (error && error.code === 'AccessDenied') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.code === 'InvalidClientTokenId') return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
-            if (error && error.message && error.message.indexOf('it was not found') !== -1) {
-                debug('del: resource record set not found.', error);
-                return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-            } else if (error && error.code === 'NoSuchHostedZone') {
-                debug('del: hosted zone not found.', error);
-                return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-            } else if (error && error.code === 'PriorRequestNotComplete') {
-                debug('del: resource is still busy', error);
-                return callback(new BoxError(BoxError.BUSY, error.message));
-            } else if (error && error.code === 'InvalidChangeBatch') {
-                debug('del: invalid change batch. No such record to be deleted.');
-                return callback(new BoxError(BoxError.NOT_FOUND, error.message));
-            } else if (error) {
-                debug('del: error', error);
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-            }
-
-            callback(null);
-        });
-    });
+    const route53 = new AWS.Route53(getDnsCredentials(domainConfig));
+    const [error] = await safe(route53.changeResourceRecordSets(params).promise());
+    if (error && error.code === 'AccessDenied') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.code === 'InvalidClientTokenId') throw new BoxError(BoxError.ACCESS_DENIED, error.message);
+    if (error && error.message && error.message.indexOf('it was not found') !== -1) {
+        throw new BoxError(BoxError.NOT_FOUND, error.message);
+    } else if (error && error.code === 'NoSuchHostedZone') {
+        throw new BoxError(BoxError.NOT_FOUND, error.message);
+    } else if (error && error.code === 'PriorRequestNotComplete') {
+        throw new BoxError(BoxError.BUSY, error.message);
+    } else if (error && error.code === 'InvalidChangeBatch') {
+        throw new BoxError(BoxError.NOT_FOUND, error.message);
+    } else if (error) {
+        throw new BoxError(BoxError.EXTERNAL_ERROR, error.message);
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.accessKeyId || typeof dnsConfig.accessKeyId !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'accessKeyId must be a non-empty string', { field: 'accessKeyId' }));
-    if (!dnsConfig.secretAccessKey || typeof dnsConfig.secretAccessKey !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'secretAccessKey must be a non-empty string', { field: 'secretAccessKey' }));
+    if (!domainConfig.accessKeyId || typeof domainConfig.accessKeyId !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'accessKeyId must be a non-empty string');
+    if (!domainConfig.secretAccessKey || typeof domainConfig.secretAccessKey !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'secretAccessKey must be a non-empty string');
 
-    var credentials = {
-        accessKeyId: dnsConfig.accessKeyId,
-        secretAccessKey: dnsConfig.secretAccessKey,
-        region: dnsConfig.region || 'us-east-1',
-        endpoint: dnsConfig.endpoint || null,
+    const credentials = {
+        accessKeyId: domainConfig.accessKeyId,
+        secretAccessKey: domainConfig.secretAccessKey,
+        region: domainConfig.region || 'us-east-1',
+        endpoint: domainConfig.endpoint || null,
         listHostedZonesByName: true, // new/updated creds require this perm
     };
 
     const ip = '127.0.0.1';
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        getHostedZone(credentials, zoneName, function (error, zone) {
-            if (error) return callback(error);
+    const zone = await getHostedZone(credentials, zoneName);
 
-            if (!_.isEqual(zone.DelegationSet.NameServers.sort(), nameservers.sort())) {
-                debug('verifyDnsConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers);
-                return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Route53', { field: 'nameservers' }));
-            }
+    if (!_.isEqual(zone.DelegationSet.NameServers.sort(), nameservers.sort())) {
+        debug('verifyDomainConfig: %j and %j do not match', nameservers, zone.DelegationSet.NameServers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Route53');
+    }
 
-            const location = 'cloudrontestdns';
-            const newDomainObject = Object.assign({ }, domainObject, { config: credentials });
+    const location = 'cloudrontestdns';
+    const newDomainObject = Object.assign({ }, domainObject, { config: credentials });
 
-            upsert(newDomainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
+    await upsert(newDomainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-                debug('verifyDnsConfig: Test A record added');
+    await del(newDomainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-                del(newDomainObject, location, 'A', [ ip ], function (error) {
-                    if (error) return callback(error);
-
-                    debug('verifyDnsConfig: Test A record removed again');
-
-                    callback(null, credentials);
-                });
-            });
-        });
-    });
+    return credentials;
 }

src/dns/vultr.js

@@ -7,14 +7,14 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
-const async = require('async'),
-    assert = require('assert'),
+const assert = require('assert'),
     constants = require('../constants.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/vultr'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
@@ -36,244 +36,202 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function getZoneRecords(dnsConfig, zoneName, name, type, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function getZoneRecords(domainConfig, zoneName, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
 
-    let per_page = 100, cursor= null;
+    let per_page = 100, cursor = null;
     let records = [];
 
-    async.doWhilst(function (iteratorDone) {
+    do {
         const url = `${VULTR_ENDPOINT}/domains/${zoneName}/records?per_page=${per_page}` + (cursor ? `&cursor=${cursor}` : '');
 
-        superagent.get(url)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
-            .timeout(30 * 1000)
-            .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return iteratorDone(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return iteratorDone(new BoxError(BoxError.NOT_FOUND, formatError(result)));
-                if (result.statusCode === 403 || result.statusCode === 401) return iteratorDone(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 200) return iteratorDone(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+        const [error, response] = await safe(superagent.get(url).set('Authorization', 'Bearer ' + domainConfig.token).timeout(30 * 1000).retry(5).ok(() => true));
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                records = records.concat(result.body.records.filter(function (record) {
-                    return (record.type === type && record.name === name);
-                }));
+        records = records.concat(response.body.records.filter(function (record) {
+            return (record.type === type && record.name === name);
+        }));
 
-                cursor = safe.query(result.body, 'meta.links.next');
+        cursor = safe.query(response.body, 'meta.links.next');
+    } while (cursor);
 
-                iteratorDone();
-            });
-    }, function (testDone) { return testDone(null, !!cursor); }, function (error) {
-        debug('getZoneRecords: error:', error, JSON.stringify(records));
-
-        if (error) return callback(error);
-
-        callback(null, records);
-    });
+    return records;
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
-        if (error) return callback(error);
-
-        var tmp = records.map(function (record) { return record.data; });
-
-        debug('get: %j', tmp);
-
-        return callback(null, tmp);
-    });
+    const records = await getZoneRecords(domainConfig, zoneName, name, type);
+    const tmp = records.map(function (record) { return record.data; });
+    return tmp;
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
-        if (error) return callback(error);
+    const records = await getZoneRecords(domainConfig, zoneName, name, type);
 
-        let i = 0, recordIds = []; // used to track available records to update instead of create
+    let i = 0, recordIds = []; // used to track available records to update instead of create
 
-        async.eachSeries(values, function (value, iteratorCallback) {
-            let data = {
-                type,
-                ttl: 300 // lowest
-            };
+    for (const value of values) {
+        const data = {
+            type,
+            ttl: 300 // lowest
+        };
 
-            if (type === 'MX') {
-                data.priority = parseInt(value.split(' ')[0], 10);
-                data.data = value.split(' ')[1];
-            } else if (type === 'TXT') {
-                data.data = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
-            } else {
-                data.data = value;
-            }
+        if (type === 'MX') {
+            data.priority = parseInt(value.split(' ')[0], 10);
+            data.data = value.split(' ')[1];
+        } else if (type === 'TXT') {
+            data.data = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
+        } else {
+            data.data = value;
+        }
 
-            if (i >= records.length) {
-                data.name = name; // only set for new records
+        if (i >= records.length) {
+            data.name = name; // only set for new records
 
-                superagent.post(`${VULTR_ENDPOINT}/domains/${zoneName}/records`)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode !== 201) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            const [error, response] = await safe(superagent.post(`${VULTR_ENDPOINT}/domains/${zoneName}/records`)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        recordIds.push(result.body.record.id);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        return iteratorCallback(null);
-                    });
-            } else {
-                superagent.patch(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${records[i].id}`)
-                    .set('Authorization', 'Bearer ' + dnsConfig.token)
-                    .send(data)
-                    .timeout(30 * 1000)
-                    .retry(5)
-                    .end(function (error, result) {
-                        // increment, as we have consumed the record
-                        ++i;
+            recordIds.push(response.body.record.id);
+        } else {
+            const [error, response] = await safe(superagent.patch(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${records[i].id}`)
+                .set('Authorization', 'Bearer ' + domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
 
-                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
-                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                        if (result.statusCode !== 204) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            ++i;
 
-                        recordIds.push(records[i-1].id);
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, formatError(response));
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 
-                        return iteratorCallback(null);
-                    });
-            }
-        }, function (error) {
-            if (error) return callback(error);
+            recordIds.push(records[i-1].id);
+        }
+    }
 
-            debug('upsert: completed with recordIds:%j', recordIds);
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] = await safe(superagent.del(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${records[j].id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5));
 
-            callback();
-        });
-    });
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
+
+    debug('upsert: completed with recordIds:%j', recordIds);
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
-        if (error) return callback(error);
+    const records = await getZoneRecords(domainConfig, zoneName, name, type);
+    if (records.length === 0) return;
 
-        if (records.length === 0) return callback(null);
+    const tmp = records.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
+    if (tmp.length === 0) return;
 
-        const tmp = records.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
-
-        debug('del: %j', tmp);
-
-        if (tmp.length === 0) return callback(null);
-
-        // FIXME we only handle the first one currently
-
-        superagent.del(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${tmp[0].id}`)
-            .set('Authorization', 'Bearer ' + dnsConfig.token)
+    for (const r of tmp) {
+        const [error, response] = await safe(superagent.del(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${r.id}`)
+            .set('Authorization', 'Bearer ' + domainConfig.token)
             .timeout(30 * 1000)
             .retry(5)
-            .end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 404) return callback(null);
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
-                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+            .ok(() => true));
 
-                debug('del: done');
-
-                return callback(null);
-            });
-    });
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) continue;
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    }
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-function verifyDnsConfig(domainObject, callback) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    const dnsConfig = domainObject.config,
+    const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
-    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
 
     const ip = '127.0.0.1';
 
-    var credentials = {
-        token: dnsConfig.token
+    const credentials = {
+        token: domainConfig.token
     };
 
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+    if (process.env.BOX_ENV === 'test') credentials; // this shouldn't be here
 
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-        if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.vultr.com') === -1) {
-            debug('verifyDnsConfig: %j does not contains vultr NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Vultr', { field: 'nameservers' }));
-        }
+    if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.vultr.com') === -1) {
+        debug('verifyDomainConfig: %j does not contains vultr NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Vultr');
+    }
 
-        const location = 'cloudrontestdns';
+    const location = 'cloudrontestdns';
 
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
 
-            debug('verifyDnsConfig: Test A record added');
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
 
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
+    return credentials;
 }

src/dns/waitfordns.js

@@ -3,108 +3,99 @@
 exports = module.exports = waitForDns;
 
 const assert = require('assert'),
-    async = require('async'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/waitfordns'),
-    dns = require('../dns.js');
+    dig = require('../dig.js'),
+    promiseRetry = require('../promise-retry.js'),
+    safe = require('safetydance');
 
-function resolveIp(hostname, options, callback) {
+async function resolveIp(hostname, type, options) {
     assert.strictEqual(typeof hostname, 'string');
+    assert(type === 'A' || type === 'AAAA');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     // try A record at authoritative server
-    debug(`resolveIp: Checking if ${hostname} has A record at ${options.server}`);
-    dns.resolve(hostname, 'A', options, function (error, results) {
-        if (!error && results.length !== 0) return callback(null, results);
+    debug(`resolveIp: Checking if ${hostname} has ${type} record at ${options.server}`);
+    const [error, results] = await safe(dig.resolve(hostname, type, options));
+    if (!error && results.length !== 0) return results;
 
-        // try CNAME record at authoritative server
-        debug(`resolveIp: Checking if ${hostname} has CNAME record at ${options.server}`);
-        dns.resolve(hostname, 'CNAME', options, function (error, results) {
-            if (error || results.length === 0) return callback(error, results);
+    // try CNAME record at authoritative server
+    debug(`resolveIp: Checking if ${hostname} has CNAME record at ${options.server}`);
+    const cnameResults = await dig.resolve(hostname, 'CNAME', options);
+    if (cnameResults.length === 0) return cnameResults;
 
-            // recurse lookup the CNAME record
-            debug(`resolveIp: Resolving ${hostname}'s CNAME record ${results[0]}`);
-            dns.resolve(results[0], 'A', { server: '127.0.0.1', timeout: options.timeout }, callback);
-        });
-    });
+    // recurse lookup the CNAME record
+    debug(`resolveIp: Resolving ${hostname}'s CNAME record ${cnameResults[0]}`);
+    return await dig.resolve(cnameResults[0], type, options);
 }
 
-function isChangeSynced(hostname, type, value, nameserver, callback) {
+async function isChangeSynced(hostname, type, value, nameserver) {
     assert.strictEqual(typeof hostname, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert.strictEqual(typeof nameserver, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // ns records cannot have cname
-    dns.resolve(nameserver, 'A', { timeout: 5000 }, function (error, nsIps) {
-        if (error || !nsIps || nsIps.length === 0) {
-            debug(`isChangeSynced: cannot resolve NS ${nameserver}`); // it's fine if one or more ns are dead
-            return callback(null, true);
+    const [error, nsIps] = await safe(dig.resolve(nameserver, 'A', { timeout: 5000 }));
+    if (error || !nsIps || nsIps.length === 0) {
+        debug(`isChangeSynced: cannot resolve NS ${nameserver}`); // it's fine if one or more ns are dead
+        return true;
+    }
+
+    const status = [];
+    for (let i = 0; i < nsIps.length; i++) {
+        const nsIp = nsIps[i];
+        const resolveOptions = { server: nsIp, timeout: 5000 };
+        const resolver = type === 'A' || type === 'AAAA' ? resolveIp(hostname, type, resolveOptions) : dig.resolve(hostname, 'TXT', resolveOptions);
+
+        const [error, answer] = await safe(resolver);
+        if (error && error.code === 'TIMEOUT') {
+            debug(`isChangeSynced: NS ${nameserver} (${nsIp}) timed out when resolving ${hostname} (${type})`);
+            status[i] = true; // should be ok if dns server is down
+            continue;
         }
 
-        async.every(nsIps, function (nsIp, iteratorCallback) {
-            const resolveOptions = { server: nsIp, timeout: 5000 };
-            const resolver = type === 'A' ? resolveIp.bind(null, hostname) : dns.resolve.bind(null, hostname, 'TXT');
+        if (error) {
+            debug(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`);
+            status[i] = false;
+            continue;
+        }
 
-            resolver(resolveOptions, function (error, answer) {
-                if (error && error.code === 'TIMEOUT') {
-                    debug(`isChangeSynced: NS ${nameserver} (${nsIp}) timed out when resolving ${hostname} (${type})`);
-                    return iteratorCallback(null, true); // should be ok if dns server is down
-                }
+        let match;
+        if (type === 'A' || type === 'AAAA') {
+            match = answer.length === 1 && answer[0] === value;
+        } else if (type === 'TXT') { // answer is a 2d array of strings
+            match = answer.some(function (a) { return value === a.join(''); });
+        }
 
-                if (error) {
-                    debug(`isChangeSynced: NS ${nameserver} (${nsIp}) errored when resolve ${hostname} (${type}): ${error}`);
-                    return iteratorCallback(null, false);
-                }
+        debug(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`);
+        status[i] = match;
+    }
 
-                let match;
-                if (type === 'A') {
-                    match = answer.length === 1 && answer[0] === value;
-                } else if (type === 'TXT') { // answer is a 2d array of strings
-                    match = answer.some(function (a) { return value === a.join(''); });
-                }
-
-                debug(`isChangeSynced: ${hostname} (${type}) was resolved to ${answer} at NS ${nameserver} (${nsIp}). Expecting ${value}. Match ${match}`);
-
-                iteratorCallback(null, match);
-            });
-        }, callback);
-
-    });
+    return status.every(s => s === true);
 }
 
 // check if IP change has propagated to every nameserver
-function waitForDns(hostname, zoneName, type, value, options, callback) {
+async function waitForDns(hostname, zoneName, type, value, options) {
     assert.strictEqual(typeof hostname, 'string');
     assert.strictEqual(typeof zoneName, 'string');
-    assert(type === 'A' || type === 'TXT');
+    assert(type === 'A' || type === 'AAAA' || type === 'TXT');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    debug('waitForDns: hostname %s to be %s in zone %s.', hostname, value, zoneName);
+    debug(`waitForDns: waiting for ${hostname} to be ${value} in zone ${zoneName}`);
 
-    var attempt = 0;
-    async.retry(options, function (retryCallback) {
-        ++attempt;
-        debug(`waitForDns (try ${attempt}): ${hostname} to be ${value} in zone ${zoneName}`);
+    await promiseRetry(Object.assign({ debug }, options), async function () {
+        const nameservers = await dig.resolve(zoneName, 'NS', { timeout: 5000 });
+        if (!nameservers) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to get nameservers');
+        debug(`waitForDns: nameservers are ${JSON.stringify(nameservers)}`);
 
-        dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-            if (error || !nameservers) return retryCallback(error || new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to get nameservers'));
-
-            async.every(nameservers, isChangeSynced.bind(null, hostname, type, value), function (error, synced) {
-                debug('waitForDns: %s %s ns: %j', hostname, synced ? 'done' : 'not done', nameservers);
-
-                retryCallback(synced ? null : new BoxError(BoxError.EXTERNAL_ERROR, 'ETRYAGAIN'));
-            });
-        });
-    }, function retryDone(error) {
-        if (error) return callback(error);
-
-        debug(`waitForDns: ${hostname} has propagated`);
-
-        callback(null);
+        for (const nameserver of nameservers) {
+            const synced = await isChangeSynced(hostname, type, value, nameserver);
+            debug(`waitForDns: ${hostname} at ns ${nameserver}: ${synced ? 'done' : 'not done'} `);
+            if (!synced) throw new BoxError(BoxError.EXTERNAL_ERROR, 'ETRYAGAIN');
+        }
     });
+
+    debug(`waitForDns: ${hostname} has propagated`);
 }

src/dns/wildcard.js

@@ -7,12 +7,13 @@ exports = module.exports = {
     get,
     del,
     wait,
-    verifyDnsConfig
+    verifyDomainConfig
 };
 
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/manual'),
+    dig = require('../dig.js'),
     dns = require('../dns.js'),
     safe = require('safetydance'),
     sysinfo = require('../sysinfo.js'),
@@ -26,71 +27,73 @@ function removePrivateFields(domainObject) {
 function injectPrivateFields(newConfig, currentConfig) {
 }
 
-function upsert(domainObject, location, type, values, callback) {
+async function upsert(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
 
-    return callback(null);
+    return;
 }
 
-function get(domainObject, location, type, callback) {
+async function get(domainObject, location, type) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    callback(null, [ ]); // returning ip confuses apptask into thinking the entry already exists
+    return []; // returning ip confuses apptask into thinking the entry already exists
 }
 
-function del(domainObject, location, type, values, callback) {
+async function del(domainObject, location, type, values) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
 
-    return callback();
+    return;
 }
 
-function wait(domainObject, location, type, value, options, callback) {
+async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = dns.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-async function verifyDnsConfig(domainObject) {
+async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
 
     const zoneName = domainObject.zoneName;
 
-    // Very basic check if the nameservers can be fetched
-    const [error, nameservers] = await safe(dns.promises.resolve(zoneName, 'NS', { timeout: 5000 }));
-    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' });
-    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' });
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
     const location = 'cloudrontestdns';
     const fqdn = dns.fqdn(location, domainObject);
 
-    const [error2, result] = await safe(dns.promises.resolve(fqdn, 'A', { server: '127.0.0.1', timeout: 5000 }));
-    if (error2 && error2.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve ${fqdn}`, { field: 'nameservers' });
-    if (error2 || !result) throw new BoxError(BoxError.BAD_FIELD, error2 ? error2.message : `Unable to resolve ${fqdn}`, { field: 'nameservers' });
+    const [ipv4Error, ipv4Result] = await safe(dig.resolve(fqdn, 'A', { server: '127.0.0.1', timeout: 5000 }));
+    if (ipv4Error && ipv4Error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv4 of ${fqdn}. Please check if you have set up *.${domainObject.domain} to point to this server's IP`);
+    if (ipv4Error || !ipv4Result) throw new BoxError(BoxError.BAD_FIELD, ipv4Error ? ipv4Error.message : `Unable to resolve IPv4 of ${fqdn}`);
 
-    const [error3, ip] = await safe(sysinfo.getServerIp());
-    if (error3) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to detect IP of this server: ${error3.message}`);
+    const ipv4 = await sysinfo.getServerIPv4();
+    if (ipv4Result.length !== 1 || ipv4 !== ipv4Result[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(ipv4Result)} instead of IPv4 ${ipv4}`);
 
-    if (result.length !== 1 || ip !== result[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(result)} instead of ${ip}`);
+    const ipv6 = await sysinfo.getServerIPv6(); // both should be RFC 5952 format
+    if (ipv6) {
+        const [ipv6Error, ipv6Result] = await safe(dig.resolve(fqdn, 'AAAA', { server: '127.0.0.1', timeout: 5000 }));
+        if (ipv6Error && ipv6Error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv6 of ${fqdn}`);
+        if (ipv6Error || !ipv6Result) throw new BoxError(BoxError.BAD_FIELD, ipv6Error ? ipv6Error.message : `Unable to resolve IPv6 of ${fqdn}`);
+
+        if (ipv6Result.length !== 1 || ipv6 !== ipv6Result[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(ipv6Result)} instead of IPv6 ${ipv6}`);
+    }
 
     return {};
 }

src/docker.js

@@ -58,7 +58,7 @@ async function testRegistryConfig(config) {
     if (config.provider === 'noop') return;
 
     const [error] = await safe(gConnection.checkAuth(config)); // this returns a 500 even for auth errors
-    if (error) throw new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' });
+    if (error) throw new BoxError(BoxError.BAD_FIELD, `Invalid serverAddress: ${error.message}`);
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
@@ -247,9 +247,16 @@ function getAddresses() {
 
     const addresses = [];
     for (const phy of physicalDevices) {
-        const result = safe.JSON.parse(safe.child_process.execSync(`ip -f inet -j addr show ${phy.name}`, { encoding: 'utf8' }));
-        const address = safe.query(result, '[0].addr_info[0].local');
-        if (address) addresses.push(address);
+        const inet = safe.JSON.parse(safe.child_process.execSync(`ip -f inet -j addr show ${phy.name}`, { encoding: 'utf8' }));
+        for (const r of inet) {
+            const address = safe.query(r, 'addr_info[0].local');
+            if (address) addresses.push(address);
+        }
+        const inet6 = safe.JSON.parse(safe.child_process.execSync(`ip -f inet6 -j addr show ${phy.name}`, { encoding: 'utf8' }));
+        for (const r of inet6) {
+            const address = safe.query(r, 'addr_info[0].local');
+            if (address) addresses.push(address);
+        }
     }
 
     return addresses;
@@ -266,18 +273,19 @@ async function createSubcontainer(app, name, cmd, options) {
     const manifest = app.manifest;
     const exposedPorts = {}, dockerPortBindings = { };
     const domain = app.fqdn;
-    const envPrefix = manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
     const stdEnv = [
         'CLOUDRON=1',
         'CLOUDRON_PROXY_IP=172.18.0.1',
         `CLOUDRON_APP_HOSTNAME=${app.id}`,
-        `${envPrefix}WEBADMIN_ORIGIN=${settings.dashboardOrigin()}`,
-        `${envPrefix}API_ORIGIN=${settings.dashboardOrigin()}`,
-        `${envPrefix}APP_ORIGIN=https://${domain}`,
-        `${envPrefix}APP_DOMAIN=${domain}`
+        `CLOUDRON_WEBADMIN_ORIGIN=${settings.dashboardOrigin()}`,
+        `CLOUDRON_API_ORIGIN=${settings.dashboardOrigin()}`,
+        `CLOUDRON_APP_ORIGIN=https://${domain}`,
+        `CLOUDRON_APP_DOMAIN=${domain}`
     ];
 
+    const secondaryDomainsEnv = app.secondaryDomains.map(sd => `${sd.environmentVariable}=${sd.fqdn}`);
+
     const portEnv = [];
     for (const portName in app.portBindings) {
         const hostPort = app.portBindings[portName];
@@ -290,7 +298,7 @@ async function createSubcontainer(app, name, cmd, options) {
         exposedPorts[`${containerPort}/${portType}`] = {};
         portEnv.push(`${portName}=${hostPort}`);
 
-        const hostIps = hostPort === 53 ? getAddresses() : [ '0.0.0.0' ]; // port 53 is special because it is possibly taken by systemd-resolved
+        const hostIps = hostPort === 53 ? getAddresses() : [ '0.0.0.0', '::0' ]; // port 53 is special because it is possibly taken by systemd-resolved
         dockerPortBindings[`${containerPort}/${portType}`] = hostIps.map(hip => { return { HostIp: hip, HostPort: hostPort + '' }; });
     }
 
@@ -312,7 +320,7 @@ async function createSubcontainer(app, name, cmd, options) {
         Tty: isAppContainer,
         Image: app.manifest.dockerImage,
         Cmd: (isAppContainer && app.debugMode && app.debugMode.cmd) ? app.debugMode.cmd : cmd,
-        Env: stdEnv.concat(addonEnv).concat(portEnv).concat(appEnv),
+        Env: stdEnv.concat(addonEnv).concat(portEnv).concat(appEnv).concat(secondaryDomainsEnv),
         ExposedPorts: isAppContainer ? exposedPorts : { },
         Volumes: { // see also ReadonlyRootfs
             '/tmp': {},

src/domains.js

@@ -4,7 +4,8 @@ module.exports = exports = {
     add,
     get,
     list,
-    update,
+    setConfig,
+    setWellKnown,
     del,
     clear,
 
@@ -22,7 +23,6 @@ const assert = require('assert'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     tld = require('tldjs'),
-    util = require('util'),
     _ = require('underscore');
 
 const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(',');
@@ -66,22 +66,17 @@ function api(provider) {
     }
 }
 
-function maybePromisify(func) {
-    if (util.types.isAsyncFunction(func)) return func;
-    return util.promisify(func);
-}
-
-async function verifyDnsConfig(dnsConfig, domain, zoneName, provider) {
-    assert(dnsConfig && typeof dnsConfig === 'object'); // the dns config to test with
+async function verifyDomainConfig(domainConfig, domain, zoneName, provider) {
+    assert(domainConfig && typeof domainConfig === 'object'); // the dns config to test with
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof provider, 'string');
 
     const backend = api(provider);
-    if (!backend) throw new BoxError(BoxError.BAD_FIELD, 'Invalid provider', { field: 'provider' });
+    if (!backend) throw new BoxError(BoxError.BAD_FIELD, 'Invalid provider');
 
-    const domainObject = { config: dnsConfig, domain: domain, zoneName: zoneName };
-    const [error, result] = await safe(maybePromisify(api(provider).verifyDnsConfig)(domainObject));
+    const domainObject = { config: domainConfig, domain: domain, zoneName: zoneName };
+    const [error, result] = await safe(api(provider).verifyDomainConfig(domainObject));
     if (error && error.reason === BoxError.ACCESS_DENIED) return { error: new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`) };
     if (error && error.reason === BoxError.NOT_FOUND) return { error: new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`) };
     if (error && error.reason === BoxError.EXTERNAL_ERROR) return { error: new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`) };
@@ -100,12 +95,12 @@ function validateTlsConfig(tlsConfig, dnsProvider) {
     case 'fallback':
         break;
     default:
-        return new BoxError(BoxError.BAD_FIELD, 'tlsConfig.provider must be fallback, letsencrypt-prod/staging', { field: 'tlsProvider' });
+        return new BoxError(BoxError.BAD_FIELD, 'tlsConfig.provider must be fallback, letsencrypt-prod/staging');
     }
 
     if (tlsConfig.wildcard) {
-        if (!tlsConfig.provider.startsWith('letsencrypt')) return new BoxError(BoxError.BAD_FIELD, 'wildcard can only be set with letsencrypt', { field: 'wildcard' });
-        if (dnsProvider === 'manual' || dnsProvider === 'noop' || dnsProvider === 'wildcard') return new BoxError(BoxError.BAD_FIELD, 'wildcard cert requires a programmable DNS backend', { field: 'tlsProvider' });
+        if (!tlsConfig.provider.startsWith('letsencrypt')) return new BoxError(BoxError.BAD_FIELD, 'wildcard can only be set with letsencrypt');
+        if (dnsProvider === 'manual' || dnsProvider === 'noop' || dnsProvider === 'wildcard') return new BoxError(BoxError.BAD_FIELD, 'wildcard cert requires a programmable DNS backend');
     }
 
     return null;
@@ -127,12 +122,12 @@ async function add(domain, data, auditSource) {
 
     let { zoneName, provider, config, fallbackCertificate, tlsConfig, dkimSelector } = data;
 
-    if (!tld.isValid(domain)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' });
-    if (domain.endsWith('.')) throw new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' });
+    if (!tld.isValid(domain)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid domain');
+    if (domain.endsWith('.')) throw new BoxError(BoxError.BAD_FIELD, 'Invalid domain');
 
     if (zoneName) {
-        if (!tld.isValid(zoneName)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' });
-        if (zoneName.endsWith('.')) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' });
+        if (!tld.isValid(zoneName)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName');
+        if (zoneName.endsWith('.')) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName');
     } else {
         zoneName = tld.getDomain(domain) || domain;
     }
@@ -155,7 +150,7 @@ async function add(domain, data, auditSource) {
         dkimSelector = `cloudron-${suffix}`;
     }
 
-    const result = await verifyDnsConfig(config, domain, zoneName, provider);
+    const result = await verifyDomainConfig(config, domain, zoneName, provider);
     if (result.error) throw result.error;
 
     let queries = [
@@ -170,7 +165,7 @@ async function add(domain, data, auditSource) {
 
     await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
 
-    eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
+    await eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
 
     safe(mail.onDomainAdded(domain)); // background
 }
@@ -189,7 +184,7 @@ async function list() {
     return results;
 }
 
-async function update(domain, data, auditSource) {
+async function setConfig(domain, data, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof data.zoneName, 'string');
     assert.strictEqual(typeof data.provider, 'string');
@@ -198,14 +193,14 @@ async function update(domain, data, auditSource) {
     assert.strictEqual(typeof data.tlsConfig, 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
-    let { zoneName, provider, config, fallbackCertificate, tlsConfig, wellKnown } = data;
+    let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
     let error;
 
     if (settings.isDemo() && (domain === settings.dashboardDomain())) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
 
     const domainObject = await get(domain);
     if (zoneName) {
-        if (!tld.isValid(zoneName)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName', { field: 'zoneName' });
+        if (!tld.isValid(zoneName)) throw new BoxError(BoxError.BAD_FIELD, 'Invalid zoneName');
     } else {
         zoneName = domainObject.zoneName;
     }
@@ -218,12 +213,9 @@ async function update(domain, data, auditSource) {
     error = validateTlsConfig(tlsConfig, provider);
     if (error) throw error;
 
-    error = validateWellKnown(wellKnown, provider);
-    if (error) throw error;
-
     if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config);
 
-    const result = await verifyDnsConfig(config, domain, zoneName, provider);
+    const result = await verifyDomainConfig(config, domain, zoneName, provider);
     if (result.error) throw result.error;
 
     const newData = {
@@ -231,14 +223,13 @@ async function update(domain, data, auditSource) {
         zoneName,
         provider,
         tlsConfig,
-        wellKnown,
     };
 
     if (fallbackCertificate) newData.fallbackCertificate = fallbackCertificate;
 
     let args = [ ], fields = [ ];
     for (const k in newData) {
-        if (k === 'config' || k === 'tlsConfig' || k === 'wellKnown' || k === 'fallbackCertificate') { // json fields
+        if (k === 'config' || k === 'tlsConfig' || k === 'fallbackCertificate') { // json fields
             fields.push(`${k}Json = ?`);
             args.push(JSON.stringify(newData[k]));
         } else {
@@ -256,7 +247,22 @@ async function update(domain, data, auditSource) {
 
     await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
 
-    eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, zoneName, provider });
+    await eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, zoneName, provider });
+}
+
+async function setWellKnown(domain, wellKnown, auditSource) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof wellKnown, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
+
+    let error = validateWellKnown(wellKnown);
+    if (error) throw error;
+
+    [error] = await safe(database.query('UPDATE domains SET wellKnownJson = ? WHERE domain=?', [ JSON.stringify(wellKnown), domain ]));
+    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
+
+    await eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, wellKnown });
 }
 
 async function del(domain, auditSource) {
@@ -274,14 +280,14 @@ async function del(domain, auditSource) {
     const [error, results] = await safe(database.transaction(queries));
     if (error && error.code === 'ER_ROW_IS_REFERENCED_2') {
         if (error.message.indexOf('apps_mailDomain_constraint') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by an app or the mailbox of an app. Check the domains of apps and the Email section of each app.');
-        if (error.message.indexOf('subdomains') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more app(s).');
+        if (error.message.indexOf('locations') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more app(s).');
         if (error.message.indexOf('mail') !== -1) throw new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more mailboxes. Delete them first in the Email view.');
         throw new BoxError(BoxError.CONFLICT, error.message);
     }
     if (error) throw error;
     if (results[1].affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
 
-    eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
+    await eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
 
     safe(mail.onDomainRemoved(domain));
 }

src/dyndns.js

@@ -19,30 +19,39 @@ const apps = require('./apps.js'),
 async function sync(auditSource) {
     assert.strictEqual(typeof auditSource, 'object');
 
-    const ip = await sysinfo.getServerIp();
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();
 
-    let info = safe.JSON.parse(safe.fs.readFileSync(paths.DYNDNS_INFO_FILE, 'utf8')) || { ip: null };
-    if (info.ip === ip) {
-        debug(`refreshDNS: no change in IP ${ip}`);
+    const info = safe.JSON.parse(safe.fs.readFileSync(paths.DYNDNS_INFO_FILE, 'utf8')) || { ipv4: null, ipv6: null };
+    if (info.ip) { // legacy cache file
+        info.ipv4 = info.ip;
+        delete info.ip;
+    }
+    const ipv4Changed = info.ip !== ipv4;
+    const ipv6Changed = ipv6 && info.ipv6 !== ipv6; // both should be RFC 5952 format
+
+    if (!ipv4Changed && !ipv6Changed) {
+        debug(`refreshDNS: no change in IP ipv4: ${ipv4} ipv6: ${ipv6}`);
         return;
     }
 
-    debug(`refreshDNS: updating ip from ${info.ip} to ${ip}`);
-
-    await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'A', [ ip ]);
-    debug('refreshDNS: updated admin location');
+    debug(`refreshDNS: updating IP from ${info.ip} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`);
+    if (ipv4Changed) await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'A', [ ipv4 ]);
+    if (ipv6Changed) await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'AAAA', [ ipv6 ]);
 
     const result = await apps.list();
     for (const app of result) {
         // do not change state of installing apps since apptask will error if dns record already exists
         if (app.installationState !== apps.ISTATE_INSTALLED) continue;
 
-        await dns.upsertDnsRecords(app.location, app.domain, 'A', [ ip ]);
+        if (ipv4Changed) await dns.upsertDnsRecords(app.subdomain, app.domain, 'A', [ ipv4 ]);
+        if (ipv6Changed) await dns.upsertDnsRecords(app.subdomain, app.domain, 'AAAA', [ ipv6 ]);
     }
 
     debug('refreshDNS: updated apps');
 
-    await eventlog.add(eventlog.ACTION_DYNDNS_UPDATE, auditSource, { fromIp: info.ip, toIp: ip });
-    info.ip = ip;
+    await eventlog.add(eventlog.ACTION_DYNDNS_UPDATE, auditSource, { fromIpv4: info.ipv4, fromIpv6: info.ipv6, toIpv4: ipv4, toIpv6: ipv6 });
+    info.ipv4 = ipv4;
+    info.ipv6 = ipv6;
     safe.fs.writeFileSync(paths.DYNDNS_INFO_FILE, JSON.stringify(info), 'utf8');
 }

src/eventlog.js

@@ -36,6 +36,7 @@ exports = module.exports = {
 
     ACTION_CERTIFICATE_NEW: 'certificate.new',
     ACTION_CERTIFICATE_RENEWAL: 'certificate.renew',
+    ACTION_CERTIFICATE_CLEANUP: 'certificate.cleanup',
 
     ACTION_DASHBOARD_DOMAIN_UPDATE: 'dashboard.domain.update',
 
@@ -91,12 +92,14 @@ const assert = require('assert'),
     safe = require('safetydance'),
     uuid = require('uuid');
 
-const EVENTLOG_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
+const EVENTLOG_FIELDS = [ 'id', 'action', 'sourceJson', 'dataJson', 'creationTime' ].join(',');
 
 function postProcess(record) {
     // usually we have sourceJson and dataJson, however since this used to be the JSON data type, we don't
-    record.source = safe.JSON.parse(record.source);
-    record.data = safe.JSON.parse(record.data);
+    record.source = safe.JSON.parse(record.sourceJson);
+    delete record.sourceJson;
+    record.data = safe.JSON.parse(record.dataJson);
+    delete record.dataJson;
 
     return record;
 }
@@ -108,7 +111,7 @@ async function add(action, source, data) {
     assert.strictEqual(typeof data, 'object');
 
     const id = uuid.v4();
-    await database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ]);
+    await database.query('INSERT INTO eventlog (id, action, sourceJson, dataJson) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ]);
     await notifications.onEvent(id, action, source, data);
     return id;
 }
@@ -121,10 +124,10 @@ async function upsertLoginEvent(action, source, data) {
 
     // can't do a real sql upsert, for frequent eventlog entries we only have to do 2 queries once a day
     const queries = [{
-        query: 'UPDATE eventlog SET creationTime=NOW(), data=? WHERE action = ? AND source LIKE ? AND DATE(creationTime)=CURDATE()',
+        query: 'UPDATE eventlog SET creationTime=NOW(), dataJson=? WHERE action = ? AND sourceJson LIKE ? AND DATE(creationTime)=CURDATE()',
         args: [ JSON.stringify(data), action, JSON.stringify(source) ]
     }, {
-        query: 'SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog WHERE action = ? AND source LIKE ? AND DATE(creationTime)=CURDATE()',
+        query: 'SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog WHERE action = ? AND sourceJson LIKE ? AND DATE(creationTime)=CURDATE()',
         args: [ action, JSON.stringify(source) ]
     }];
 
@@ -154,7 +157,7 @@ async function listPaged(actions, search, page, perPage) {
     let query = `SELECT ${EVENTLOG_FIELDS} FROM eventlog`;
 
     if (actions.length || search) query += ' WHERE';
-    if (search) query += ' (source LIKE ' + mysql.escape('%' + search + '%') + ' OR data LIKE ' + mysql.escape('%' + search + '%') + ')';
+    if (search) query += ' (sourceJson LIKE ' + mysql.escape('%' + search + '%') + ' OR dataJson LIKE ' + mysql.escape('%' + search + '%') + ')';
 
     if (actions.length && search) query += ' AND ( ';
     actions.forEach(function (action, i) {

src/externalldap.js

@@ -40,10 +40,11 @@ function removePrivateFields(ldapConfig) {
 function translateUser(ldapConfig, ldapUser) {
     assert.strictEqual(typeof ldapConfig, 'object');
 
+    // RFC: https://datatracker.ietf.org/doc/html/rfc2798
     return {
         username: ldapUser[ldapConfig.usernameField],
         email: ldapUser.mail || ldapUser.mailPrimaryAddress,
-        displayName: ldapUser.cn // user.giveName + ' ' + user.sn
+        displayName: ldapUser.displayName || ldapUser.cn // user.giveName + ' ' + user.sn
     };
 }
 
@@ -105,8 +106,6 @@ async function clientSearch(client, dn, searchOptions) {
     assert.strictEqual(typeof dn, 'string');
     assert.strictEqual(typeof searchOptions, 'object');
 
-    debug(`clientSearch: Get objects at ${dn} with options ${JSON.stringify(searchOptions)}`);
-
     // basic validation to not crash
     try { ldap.parseDN(dn); } catch (e) { throw new BoxError(BoxError.BAD_FIELD, 'invalid DN'); }
 
@@ -163,8 +162,6 @@ async function ldapUserSearch(externalLdapConfig, options) {
         searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
     }
 
-    debug(`Listing users at ${externalLdapConfig.baseDn} with filter ${searchOptions.filter.toString()}`);
-
     const client = await getClient(externalLdapConfig, { bind: true });
     const result = await clientSearch(client, externalLdapConfig.baseDn, searchOptions);
     client.unbind();
@@ -187,8 +184,6 @@ async function ldapGroupSearch(externalLdapConfig, options) {
         searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
     }
 
-    debug(`Listing groups at ${externalLdapConfig.groupBaseDn} with filter ${searchOptions.filter.toString()}`);
-
     const client = await getClient(externalLdapConfig, { bind: true });
     const result = await clientSearch(client, externalLdapConfig.groupBaseDn, searchOptions);
     client.unbind();
@@ -239,21 +234,6 @@ async function testConfig(config) {
     return null;
 }
 
-// eslint-disable-next-line no-unused-vars
-async function search(identifier) {
-    assert.strictEqual(typeof identifier, 'string');
-
-    const externalLdapConfig = await settings.getExternalLdapConfig();
-    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
-
-    const ldapUsers = await ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` });
-
-    // translate ldap properties to ours
-    let users = ldapUsers.map(function (u) { return translateUser(externalLdapConfig, u); });
-
-    return users;
-}
-
 async function maybeCreateUser(identifier) {
     assert.strictEqual(typeof identifier, 'string');
 

src/groups.js

@@ -33,13 +33,13 @@ const GROUPS_FIELDS = [ 'id', 'name', 'source' ].join(',');
 function validateGroupname(name) {
     assert.strictEqual(typeof name, 'string');
 
-    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 char', { field: 'name' });
-    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'name too long', { field: 'name' });
+    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 char');
+    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'name too long');
 
-    if (constants.RESERVED_NAMES.indexOf(name) !== -1) return new BoxError(BoxError.BAD_FIELD, 'name is reserved', { field: name });
+    if (constants.RESERVED_NAMES.indexOf(name) !== -1) return new BoxError(BoxError.BAD_FIELD, 'name is reserved');
 
     // need to consider valid LDAP characters here (e.g '+' is reserved)
-    if (/[^a-zA-Z0-9.-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'name can only contain alphanumerals, hyphen and dot', { field: 'name' });
+    if (/[^a-zA-Z0-9.-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'name can only contain alphanumerals, hyphen and dot');
 
     return null;
 }
@@ -47,7 +47,7 @@ function validateGroupname(name) {
 function validateGroupSource(source) {
     assert.strictEqual(typeof source, 'string');
 
-    if (source !== '' && source !== 'ldap') return new BoxError(BoxError.BAD_FIELD, 'source must be "" or "ldap"', { field: source });
+    if (source !== '' && source !== 'ldap') return new BoxError(BoxError.BAD_FIELD, 'source must be "" or "ldap"');
 
     return null;
 }

src/hat.js

@@ -2,8 +2,8 @@
 
 exports = module.exports = hat;
 
-var crypto = require('crypto');
+const crypto = require('crypto');
 
-function hat (bits) {
+function hat(bits) {
     return crypto.randomBytes(bits / 8).toString('hex');
 }

src/infra_version.js

@@ -6,22 +6,22 @@
 
 exports = module.exports = {
     // a version change recreates all containers with latest docker config
-    'version': '48.20.0',
+    'version': '49.0.0',
 
     'baseImages': [
-        { repo: 'cloudron/base', tag: 'cloudron/base:3.0.0@sha256:455c70428723e3a823198c57472785437eb6eab082e79b3ff04ea584faf46e92' }
+        { repo: 'cloudron/base', tag: 'cloudron/base:3.2.0@sha256:ba1d566164a67c266782545ea9809dc611c4152e27686fd14060332dd88263ea' }
     ],
 
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.3.1@sha256:759cafab7625ff538418a1f2ed5558b1d5bff08c576bba577d865d6d02b49091' },
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.0.7@sha256:6679c2fb96f8d6d62349b607748570640a90fc46b50aad80ca2c0161655d07f4' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.1.1@sha256:86e4e2f4fd43809efca7c9cb1def4d7608cf36cb9ea27052f9b64da4481db43a' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.0.2@sha256:9df297ccc3370f38c54f8d614e214e082b363777cd1c6c9522e29663cc8f5362' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.0.4@sha256:5c60de75d078ae609da5565f32dcd91030f45907e945756cc976ff207b8c6199' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.5.0@sha256:e05d328ea1afa94e31e2eae9b035ff2edde8b90cae902ca49e06053b5bdb5fde' },
-        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.0.1@sha256:bed9f6b5d06fe2c5289e895e806cfa5b74ad62993d705be55d4554a67d128029' },
-        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.4.2@sha256:810306478c3dac7caa7497e5f6381cc7ce2f68aafda849a4945d39a67cc04bc1' }
+        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.4.0@sha256:45817f1631992391d585f171498d257487d872480fd5646723a2b956cc4ef15d' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.2.0@sha256:c70d0a1ff7ce8a27dcf7d6f8d1718ddba82ef254079fee5d20fdf074f28cd009' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.3.0@sha256:9f0cfe83310a6f67885c21804c01e73c8d256606217f44dcefb3e05a5402b2c9' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.0@sha256:c8ebdbe2663b26fcd58b1e6b97906b62565adbe4a06256ba0f86114f78b37e6b' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.2.1@sha256:30a5550c41c3be3a01dbf457497ba2f3c05f3121c595a17ef1aacbef931b6114' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.6.1@sha256:b8b93f007105080d4812a05648e6bc5e15c95c63f511c829cbc14a163d9ea029' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.1.0@sha256:30ec3a01964a1e01396acf265183997c3e17fb07eac1a82b979292cc7719ff4b' },
+        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.6.0@sha256:9c686b10c1a3ba344a743f399d08b4da5426e111f455114980f0ae0229c1ab23' }
     }
 };

src/ldap.js

@@ -48,11 +48,39 @@ async function authenticateApp(req, res, next) {
     next();
 }
 
+// Will attach req.user if successful
+async function userAuthInternal(appId, req, res, next) {
+    // extract the common name which might have different attribute names
+    const attributeName = Object.keys(req.dn.rdns[0].attrs)[0];
+    const commonName = req.dn.rdns[0].attrs[attributeName].value;
+    if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+    let verifyFunc;
+    if (attributeName === 'mail') {
+        verifyFunc = users.verifyWithEmail;
+    } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
+        verifyFunc = users.verifyWithEmail;
+    } else if (commonName.indexOf('uid-') === 0) {
+        verifyFunc = users.verify;
+    } else {
+        verifyFunc = users.verifyWithUsername;
+    }
+
+    const [error, user] = await safe(verifyFunc(commonName, req.credentials || '', appId || ''));
+    if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+    if (error) return next(new ldap.OperationsError(error.message));
+
+    req.user = user;
+
+    next();
+}
+
 async function getUsersWithAccessToApp(req) {
     assert.strictEqual(typeof req.app, 'object');
 
     const result = await users.list();
-    const allowedUsers = result.filter((user) => apps.canAccess(req.app, user));
+    const allowedUsers = result.filter((user) => user.active && apps.canAccess(req.app, user)); // do not list inactive users
     return allowedUsers;
 }
 
@@ -142,7 +170,7 @@ async function userSearch(req, res, next) {
         const obj = {
             dn: dn.toString(),
             attributes: {
-                objectclass: ['user', 'inetorgperson', 'person' ],
+                objectclass: ['user', 'inetorgperson', 'person', 'organizationalperson', 'top' ],
                 objectcategory: 'person',
                 cn: user.id,
                 uid: user.id,
@@ -176,12 +204,13 @@ async function userSearch(req, res, next) {
 async function groupSearch(req, res, next) {
     debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
-    const [error, result] = await safe(getUsersWithAccessToApp(req));
+    const [error, usersWithAccess] = await safe(getUsersWithAccessToApp(req));
     if (error) return next(new ldap.OperationsError(error.toString()));
 
     const results = [];
 
-    const groups = [{
+    // those are the old virtual groups for backwards compat
+    const virtualGroups = [{
         name: 'users',
         admin: false
     }, {
@@ -189,16 +218,45 @@ async function groupSearch(req, res, next) {
         admin: true
     }];
 
-    groups.forEach(function (group) {
+    virtualGroups.forEach(function (group) {
         const dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-        const members = group.admin ? result.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : result;
+        const members = group.admin ? usersWithAccess.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : usersWithAccess;
 
         const obj = {
             dn: dn.toString(),
             attributes: {
                 objectclass: ['group'],
                 cn: group.name,
-                memberuid: members.map(function(entry) { return entry.id; })
+                memberuid: members.map(function(entry) { return entry.id; }).sort()
+            }
+        };
+
+        // ensure all filter values are also lowercase
+        const lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+            results.push(obj);
+        }
+    });
+
+    let [errorGroups, resultGroups] = await safe(groups.listWithMembers());
+    if (errorGroups) return next(new ldap.OperationsError(errorGroups.toString()));
+
+    if (req.app.accessRestriction && req.app.accessRestriction.groups) {
+        resultGroups = resultGroups.filter(function (g) { return req.app.accessRestriction.groups.indexOf(g.id) !== -1; });
+    }
+
+    resultGroups.forEach(function (group) {
+        const dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+        const members = group.userIds.filter(function (uid) { return usersWithAccess.map(function (u) { return u.id; }).indexOf(uid) !== -1; });
+
+        const obj = {
+            dn: dn.toString(),
+            attributes: {
+                objectclass: ['group'],
+                cn: group.name,
+                memberuid: members
             }
         };
 
@@ -279,7 +337,8 @@ async function mailboxSearch(req, res, next) {
             res.end();
         }
     } else { // new sogo
-        let [error, mailboxes] = await safe(mail.listAllMailboxes(1, 1000));
+        // TODO figure out how proper pagination here could work
+        let [error, mailboxes] = await safe(mail.listAllMailboxes(1, 100000));
         if (error) return next(new ldap.OperationsError(error.toString()));
 
         mailboxes = mailboxes.filter(m => m.active);
@@ -289,6 +348,7 @@ async function mailboxSearch(req, res, next) {
         for (const mailbox of mailboxes) {
             const dn = ldap.parseDN(`cn=${mailbox.name}@${mailbox.domain},ou=mailboxes,dc=cloudron`);
 
+            if (mailbox.ownerType === mail.OWNERTYPE_APP) continue; // cannot login with app mailbox anyway
             const [error, ownerObject] = await safe(mailbox.ownerType === mail.OWNERTYPE_USER ? users.get(mailbox.ownerId) : groups.get(mailbox.ownerId));
             if (error || !ownerObject) continue; // skip mailboxes with unknown user
 
@@ -407,30 +467,9 @@ async function mailingListSearch(req, res, next) {
 async function authenticateUser(req, res, next) {
     debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
 
-    // extract the common name which might have different attribute names
-    const attributeName = Object.keys(req.dn.rdns[0].attrs)[0];
-    const commonName = req.dn.rdns[0].attrs[attributeName].value;
-    if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    const appId = req.app.id;
 
-    let verifyFunc;
-    if (attributeName === 'mail') {
-        verifyFunc = users.verifyWithEmail;
-    } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
-        verifyFunc = users.verifyWithEmail;
-    } else if (commonName.indexOf('uid-') === 0) {
-        verifyFunc = users.verify;
-    } else {
-        verifyFunc = users.verifyWithUsername;
-    }
-
-    const [error, user] = await safe(verifyFunc(commonName, req.credentials || '', req.app.id));
-    if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-    if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
-    if (error) return next(new ldap.OperationsError(error.message));
-
-    req.user = user;
-
-    next();
+    await userAuthInternal(appId, req, res, next);
 }
 
 async function authorizeUserForApp(req, res, next) {
@@ -450,20 +489,24 @@ async function verifyMailboxPassword(mailbox, password) {
     assert.strictEqual(typeof mailbox, 'object');
     assert.strictEqual(typeof password, 'string');
 
-    if (mailbox.ownerType === mail.OWNERTYPE_USER) return await users.verify(mailbox.ownerId, password, users.AP_MAIL /* identifier */);
+    if (mailbox.ownerType === mail.OWNERTYPE_USER) {
+        return await users.verify(mailbox.ownerId, password, users.AP_MAIL /* identifier */);
+    } else if (mailbox.ownerType === mail.OWNERTYPE_GROUP) {
+        const userIds = await groups.getMembers(mailbox.ownerId);
 
-    const userIds = await groups.getMembers(mailbox.ownerId);
+        let verifiedUser = null;
+        for (const userId of userIds) {
+            const [error, result] = await safe(users.verify(userId, password, users.AP_MAIL /* identifier */));
+            if (error) continue; // try the next user
+            verifiedUser = result;
+            break; // found a matching validated user
+        }
 
-    let verifiedUser = null;
-    for (const userId of userIds) {
-        const [error, result] = await safe(users.verify(userId, password, users.AP_MAIL /* identifier */));
-        if (error) continue; // try the next user
-        verifiedUser = result;
-        break; // found a matching validated user
+        if (!verifiedUser) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+        return verifiedUser;
+    } else {
+        throw new BoxError(BoxError.INVALID_CREDENTIALS);
     }
-
-    if (!verifiedUser) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    return verifiedUser;
 }
 
 async function authenticateSftp(req, res, next) {
@@ -515,7 +558,7 @@ async function userSearchSftp(req, res, next) {
     const obj = {
         dn: ldap.parseDN(`cn=${username}@${appFqdn},ou=sftp,dc=cloudron`).toString(),
         attributes: {
-            homeDirectory: app.dataDir ? `/mnt/${app.id}` : `/mnt/appsdata/${app.id}/data`,
+            homeDirectory: app.dataDir ? `/mnt/app-${app.id}` : `/mnt/appsdata/${app.id}/data`, // see also sftp.js
             objectclass: ['user'],
             objectcategory: 'person',
             cn: user.id,
@@ -654,5 +697,8 @@ async function start() {
 }
 
 async function stop() {
-    if (gServer) gServer.close();
+    if (!gServer) return;
+
+    gServer.close();
+    gServer = null;
 }

src/mail.js

@@ -57,6 +57,7 @@ exports = module.exports = {
 
     OWNERTYPE_USER: 'user',
     OWNERTYPE_GROUP: 'group',
+    OWNERTYPE_APP: 'app',
 
     DEFAULT_MEMORY_LIMIT: 512 * 1024 * 1024,
 
@@ -64,6 +65,7 @@ exports = module.exports = {
     TYPE_LIST: 'list',
     TYPE_ALIAS: 'alias',
 
+    _validateName: validateName,
     _delByDomain: delByDomain,
     _updateDomain: updateDomain
 };
@@ -75,6 +77,7 @@ const assert = require('assert'),
     crypto = require('crypto'),
     database = require('./database.js'),
     debug = require('debug')('box:mail'),
+    dig = require('./dig.js'),
     dns = require('./dns.js'),
     docker = require('./docker.js'),
     domains = require('./domains.js'),
@@ -106,6 +109,8 @@ const assert = require('assert'),
 const DNS_OPTIONS = { timeout: 5000 };
 const REMOVE_MAILBOX_CMD = path.join(__dirname, 'scripts/rmmailbox.sh');
 
+const OWNERTYPES = [ exports.OWNERTYPE_USER, exports.OWNERTYPE_GROUP, exports.OWNERTYPE_APP ];
+
 // if you add a field here, listMailboxes has to be updated
 const MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'ownerType', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain', 'active', 'enablePop3' ].join(',');
 const MAILDB_FIELDS = [ 'domain', 'enabled', 'mailFromValidation', 'catchAllJson', 'relayJson', 'dkimKeyJson', 'dkimSelector', 'bannerJson' ].join(',');
@@ -158,19 +163,13 @@ function validateName(name) {
     if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox name must be atleast 1 char');
     if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'mailbox name too long');
 
-    // also need to consider valid LDAP characters here (e.g '+' is reserved)
-    if (/[^a-zA-Z0-9.-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox name can only contain alphanumerals and dot');
+    // also need to consider valid LDAP characters here (e.g '+' is reserved). keep hyphen at the end so it doesn't become a range.
+    if (/[^a-zA-Z0-9._-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox name can only contain alphanumerals, dot, hyphen or underscore');
 
     return null;
 }
 
 async function checkOutboundPort25() {
-    const smtpServer = _.sample([
-        'smtp.gmail.com',
-        'smtp.live.com',
-        'smtp.1und1.de',
-    ]);
-
     const relay = {
         value: 'OK',
         status: false,
@@ -180,7 +179,7 @@ async function checkOutboundPort25() {
     return await new Promise((resolve) => {
         const client = new net.Socket();
         client.setTimeout(5000);
-        client.connect(25, smtpServer);
+        client.connect(25, constants.PORT25_CHECK_SERVER);
         client.on('connect', function () {
             relay.status = true;
             relay.value = 'OK';
@@ -189,16 +188,16 @@ async function checkOutboundPort25() {
         });
         client.on('timeout', function () {
             relay.status = false;
-            relay.value = `Connect to ${smtpServer} timed out. Check if port 25 (outbound) is blocked`;
+            relay.value = `Connect to ${constants.PORT25_CHECK_SERVER} timed out. Check if port 25 (outbound) is blocked`;
             client.destroy();
-            relay.errorMessage = `Connect to ${smtpServer} timed out.`;
+            relay.errorMessage = `Connect to ${constants.PORT25_CHECK_SERVER} timed out.`;
             resolve(relay);
         });
         client.on('error', function (error) {
             relay.status = false;
-            relay.value = `Connect to ${smtpServer} failed: ${error.message}. Check if port 25 (outbound) is blocked`;
+            relay.value = `Connect to ${constants.PORT25_CHECK_SERVER} failed: ${error.message}. Check if port 25 (outbound) is blocked`;
             client.destroy();
-            relay.errorMessage = `Connect to ${smtpServer} failed.`;
+            relay.errorMessage = `Connect to ${constants.PORT25_CHECK_SERVER} failed.`;
             resolve(relay);
         });
     });
@@ -269,7 +268,7 @@ async function checkDkim(mailDomain) {
 
     dkim.expected = `v=DKIM1; t=s; p=${publicKey}`;
 
-    const [error, txtRecords] = await safe(dns.promises.resolve(dkim.domain, dkim.type, DNS_OPTIONS));
+    const [error, txtRecords] = await safe(dig.resolve(dkim.domain, dkim.type, DNS_OPTIONS));
     if (error) {
         dkim.errorMessage = error.message;
         return dkim;
@@ -298,7 +297,7 @@ async function checkSpf(domain, mailFqdn) {
         errorMessage: ''
     };
 
-    const [error, txtRecords] = await safe(dns.promises.resolve(spf.domain, spf.type, DNS_OPTIONS));
+    const [error, txtRecords] = await safe(dig.resolve(spf.domain, spf.type, DNS_OPTIONS));
     if (error) {
         spf.errorMessage = error.message;
         return spf;
@@ -336,7 +335,7 @@ async function checkMx(domain, mailFqdn) {
         errorMessage: ''
     };
 
-    const [error, mxRecords] = await safe(dns.promises.resolve(mx.domain, mx.type, DNS_OPTIONS));
+    const [error, mxRecords] = await safe(dig.resolve(mx.domain, mx.type, DNS_OPTIONS));
     if (error) {
         mx.errorMessage = error.message;
         return mx;
@@ -349,10 +348,10 @@ async function checkMx(domain, mailFqdn) {
     if (mx.status) return mx; // MX record is "my."
 
     // cloudflare might create a conflict subdomain (https://support.cloudflare.com/hc/en-us/articles/360020296512-DNS-Troubleshooting-FAQ)
-    const [error2, mxIps] = await safe(dns.promises.resolve(mxRecords[0].exchange, 'A', DNS_OPTIONS));
+    const [error2, mxIps] = await safe(dig.resolve(mxRecords[0].exchange, 'A', DNS_OPTIONS));
     if (error2 || mxIps.length !== 1) return mx;
 
-    const [error3, ip] = await safe(sysinfo.getServerIp());
+    const [error3, ip] = await safe(sysinfo.getServerIPv4());
     if (error3) return mx;
 
     mx.status = mxIps[0] === ip;
@@ -381,7 +380,7 @@ async function checkDmarc(domain) {
         errorMessage: ''
     };
 
-    const [error, txtRecords] = await safe(dns.promises.resolve(dmarc.domain, dmarc.type, DNS_OPTIONS));
+    const [error, txtRecords] = await safe(dig.resolve(dmarc.domain, dmarc.type, DNS_OPTIONS));
 
     if (error) {
         dmarc.errorMessage = error.message;
@@ -410,7 +409,7 @@ async function checkPtr(mailFqdn) {
         errorMessage: ''
     };
 
-    const [error, ip] = await safe(sysinfo.getServerIp());
+    const [error, ip] = await safe(sysinfo.getServerIPv4());
     if (error) {
         ptr.errorMessage = error.message;
         return ptr;
@@ -419,7 +418,7 @@ async function checkPtr(mailFqdn) {
     ptr.domain = ip.split('.').reverse().join('.') + '.in-addr.arpa';
     ptr.name = ip;
 
-    const [error2, ptrRecords] = await safe(dns.promises.resolve(ptr.domain, 'PTR', DNS_OPTIONS));
+    const [error2, ptrRecords] = await safe(dig.resolve(ptr.domain, 'PTR', DNS_OPTIONS));
     if (error2) {
         ptr.errorMessage = error2.message;
         return ptr;
@@ -495,21 +494,21 @@ const RBL_LIST = [
 
 // this function currently only looks for black lists based on IP. TODO: also look up by domain
 async function checkRblStatus(domain) {
-    const ip = await sysinfo.getServerIp();
+    const ip = await sysinfo.getServerIPv4();
 
     const flippedIp = ip.split('.').reverse().join('.');
 
     // https://tools.ietf.org/html/rfc5782
     const blacklistedServers = [];
     for (const rblServer of RBL_LIST) {
-        const [error, records] = await safe(dns.promises.resolve(flippedIp + '.' + rblServer.dns, 'A', DNS_OPTIONS));
+        const [error, records] = await safe(dig.resolve(flippedIp + '.' + rblServer.dns, 'A', DNS_OPTIONS));
         if (error || !records) continue;    // not listed
 
         debug(`checkRblStatus: ${domain} (ip: ${flippedIp}) is in the blacklist of ${JSON.stringify(rblServer)}`);
 
         const result = _.extend({ }, rblServer);
 
-        const [error2, txtRecords] = await safe(dns.promises.resolve(flippedIp + '.' + rblServer.dns, 'TXT', DNS_OPTIONS));
+        const [error2, txtRecords] = await safe(dig.resolve(flippedIp + '.' + rblServer.dns, 'TXT', DNS_OPTIONS));
         result.txtRecords = error2 || !txtRecords ? 'No txt record' : txtRecords.map(x => x.join(''));
 
         debug(`checkRblStatus: ${domain} (error: ${error2.message}) (txtRecords: ${JSON.stringify(txtRecords)})`);
@@ -610,11 +609,10 @@ async function checkConfiguration() {
     return markdownMessage; // empty message means all status checks succeeded
 }
 
-async function createMailConfig(mailFqdn, mailDomain) {
+async function createMailConfig(mailFqdn) {
     assert.strictEqual(typeof mailFqdn, 'string');
-    assert.strictEqual(typeof mailDomain, 'string');
 
-    debug('createMailConfig: generating mail config');
+    debug(`createMailConfig: generating mail config with ${mailFqdn}`);
 
     const mailDomains = await listDomains();
 
@@ -623,7 +621,7 @@ async function createMailConfig(mailFqdn, mailDomain) {
 
     // mail_domain is used for SRS
     if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/mail.ini`,
-        `mail_in_domains=${mailInDomains}\nmail_out_domains=${mailOutDomains}\nmail_server_name=${mailFqdn}\nmail_domain=${mailDomain}\n\n`, 'utf8')) {
+        `mail_in_domains=${mailInDomains}\nmail_out_domains=${mailOutDomains}\nmail_server_name=${mailFqdn}\n\n`, 'utf8')) {
         throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
     }
 
@@ -706,7 +704,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) {
     await shell.promises.exec('stopMail', 'docker stop mail || true');
     await shell.promises.exec('removeMail', 'docker rm -f mail || true');
 
-    const allowInbound = await createMailConfig(mailFqdn, mailDomain);
+    const allowInbound = await createMailConfig(mailFqdn);
 
     const ports = allowInbound ? '-p 587:2587 -p 993:9993 -p 4190:4190 -p 25:2587 -p 465:2465 -p 995:9995' : '';
     const readOnly = !serviceConfig.recoveryMode ? '--read-only' : '';
@@ -762,8 +760,8 @@ async function restartMail() {
     const servicesConfig = await settings.getServicesConfig();
     const mailConfig = servicesConfig['mail'] || {};
 
-    debug(`restartMail: restarting mail container with mailFqdn:${settings.mailFqdn()} dashboardDomain:${settings.dashboardDomain()}`);
-    await configureMail(settings.mailFqdn(), settings.dashboardDomain(), mailConfig);
+    debug(`restartMail: restarting mail container with mailFqdn:${settings.mailFqdn()} mailDomain:${settings.mailDomain()}`);
+    await configureMail(settings.mailFqdn(), settings.mailDomain(), mailConfig);
 }
 
 async function startMail(existingInfra) {
@@ -936,7 +934,7 @@ async function changeLocation(auditSource, progressCallback) {
     let progress = 20;
     progressCallback({ percent: progress, message: `Setting up DNS of certs of mail server ${fqdn}` });
 
-    await cloudron.setupDnsAndCert(subdomain, domain, auditSource, progressCallback);
+    await cloudron.setupDnsAndCert(subdomain, domain, auditSource, (progress) => progressCallback({ message: progress.message })); // remove the percent
     const allDomains = await domains.list();
 
     for (let idx = 0; idx < allDomains.length; idx++) {
@@ -1166,13 +1164,14 @@ async function addMailbox(name, domain, data, auditSource) {
     let error = validateName(name);
     if (error) throw error;
 
-    if (ownerType !== exports.OWNERTYPE_USER && ownerType !== exports.OWNERTYPE_GROUP) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');
+    if (!OWNERTYPES.includes(ownerType)) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');
 
     [error] = await safe(database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, active) VALUES (?, ?, ?, ?, ?, ?)', [ name, exports.TYPE_MAILBOX, domain, ownerId, ownerType, active ]));
     if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
+    if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.includes('mailboxes_domain_constraint')) throw new BoxError(BoxError.NOT_FOUND, `no such domain '${domain}'`);
     if (error) throw error;
 
-    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active });
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active });
 }
 
 async function updateMailbox(name, domain, data, auditSource) {
@@ -1189,7 +1188,7 @@ async function updateMailbox(name, domain, data, auditSource) {
 
     name = name.toLowerCase();
 
-    if (ownerType !== exports.OWNERTYPE_USER && ownerType !== exports.OWNERTYPE_GROUP) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');
+    if (!OWNERTYPES.includes(ownerType)) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');
 
     const mailbox = await getMailbox(name, domain);
     if (!mailbox) throw new BoxError(BoxError.NOT_FOUND, 'No such mailbox');
@@ -1197,7 +1196,7 @@ async function updateMailbox(name, domain, data, auditSource) {
     const result = await database.query('UPDATE mailboxes SET ownerId = ?, ownerType = ?, active = ?, enablePop3 = ? WHERE name = ? AND domain = ?', [ ownerId, ownerType, active, enablePop3, name, domain ]);
     if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
 
-    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: mailbox.userId, ownerId, ownerType, active });
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: mailbox.userId, ownerId, ownerType, active });
 }
 
 async function removeSolrIndex(mailbox) {
@@ -1205,9 +1204,8 @@ async function removeSolrIndex(mailbox) {
 
     const addonDetails = await services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN');
 
-    const [error, response] = await safe(superagent.post(`https://${addonDetails.ip}:3000/solr_delete_index?access_token=${addonDetails.token}`)
+    const [error, response] = await safe(superagent.post(`http://${addonDetails.ip}:3000/solr_delete_index?access_token=${addonDetails.token}`)
         .timeout(2000)
-        .disableTLSCerts()
         .send({ mailbox })
         .ok(() => true));
 
@@ -1235,7 +1233,7 @@ async function delMailbox(name, domain, options, auditSource) {
     const [error] = await safe(removeSolrIndex(mailbox));
     if (error) debug(`delMailbox: failed to remove solr index: ${error.message}`);
 
-    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
 }
 
 async function getAlias(name, domain) {
@@ -1259,32 +1257,35 @@ async function getAliases(name, domain) {
     return await database.query('SELECT name, domain FROM mailboxes WHERE type = ? AND aliasName = ? AND aliasDomain = ? ORDER BY name', [ exports.TYPE_ALIAS, name, domain ]);
 }
 
-async function setAliases(name, domain, aliases) {
+async function setAliases(name, domain, aliases, auditSource) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert(Array.isArray(aliases));
+    assert.strictEqual(typeof auditSource, 'object');
 
     for (let i = 0; i < aliases.length; i++) {
-        let name = aliases[i].name.toLowerCase();
-        let domain = aliases[i].domain.toLowerCase();
+        const name = aliases[i].name.toLowerCase();
+        const domain = aliases[i].domain.toLowerCase();
 
-        let error = validateName(name);
+        const error = validateName(name);
         if (error) throw error;
 
-        if (!validator.isEmail(`${name}@${domain}`)) throw new BoxError(BoxError.BAD_FIELD, `Invalid email: ${name}@${domain}`);
+        const mailDomain = await getDomain(domain);
+        if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, `mail domain ${domain} not found`);
+
         aliases[i] = { name, domain };
     }
 
     const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ?', [ name, domain ]);
     if (results.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
 
-    let queries = [];
+    const queries = [];
     // clear existing aliases
     queries.push({ query: 'DELETE FROM mailboxes WHERE aliasName = ? AND aliasDomain = ? AND type = ?', args: [ name, domain, exports.TYPE_ALIAS ] });
-    aliases.forEach(function (alias) {
+    for (const alias of aliases) {
         queries.push({ query: 'INSERT INTO mailboxes (name, domain, type, aliasName, aliasDomain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?, ?, ?)',
             args: [ alias.name, alias.domain, exports.TYPE_ALIAS, name, domain, results[0].ownerId, results[0].ownerType ] });
-    });
+    }
 
     const [error] = await safe(database.transaction(queries));
     if (error && error.code === 'ER_DUP_ENTRY' && error.message.indexOf('mailboxes_name_domain_unique_index') !== -1) {
@@ -1294,6 +1295,8 @@ async function setAliases(name, domain, aliases) {
         throw new BoxError(BoxError.ALREADY_EXISTS, `Mailbox, mailinglist or alias for ${aliasMatch[1]} already exists`);
     }
     if (error) throw error;
+
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, aliases });
 }
 
 async function getLists(domain, search, page, perPage) {
@@ -1348,7 +1351,7 @@ async function addList(name, domain, data, auditSource) {
     if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
     if (error) throw error;
 
-    eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly, active });
+    await eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly, active });
 }
 
 async function updateList(name, domain, data, auditSource) {
@@ -1375,7 +1378,7 @@ async function updateList(name, domain, data, auditSource) {
         [ JSON.stringify(members), membersOnly, active, name, domain ]);
     if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
 
-    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly, active });
+    await eventlog.add(eventlog.ACTION_MAIL_LIST_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly, active });
 }
 
 async function delList(name, domain, auditSource) {
@@ -1387,7 +1390,7 @@ async function delList(name, domain, auditSource) {
     const result = await database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ]);
     if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
 
-    eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
+    await eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
 }
 
 // resolves the members of a list. i.e the lists and aliases

src/mounts.js

@@ -1,7 +1,7 @@
 'use strict';
 
 exports = module.exports = {
-    isMountProvider,
+    isManagedProvider,
     mountObjectFromBackupConfig,
     tryAddMount,
     removeMount,
@@ -40,6 +40,7 @@ function validateMountOptions(type, options) {
         if (typeof options.password !== 'string') return new BoxError(BoxError.BAD_FIELD, 'password is not a string');
         if (typeof options.host !== 'string') return new BoxError(BoxError.BAD_FIELD, 'host is not a string');
         if (typeof options.remoteDir !== 'string') return new BoxError(BoxError.BAD_FIELD, 'remoteDir is not a string');
+        if ('seal' in options && typeof options.seal !== 'boolean') return new BoxError(BoxError.BAD_FIELD, 'seal is not a boolean');
         return null;
     case 'nfs':
         if (typeof options.host !== 'string') return new BoxError(BoxError.BAD_FIELD, 'host is not a string');
@@ -60,14 +61,16 @@ function validateMountOptions(type, options) {
     }
 }
 
-function isMountProvider(provider) {
+function isManagedProvider(provider) {
     return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4';
 }
 
 function mountObjectFromBackupConfig(backupConfig) {
+    assert(isManagedProvider(backupConfig.provider));
+
     return {
         name: 'backup',
-        hostPath: backupConfig.mountPoint,
+        hostPath: paths.MANAGED_BACKUP_MOUNT_DIR,
         mountType: backupConfig.provider,
         mountOptions: backupConfig.mountOptions
     };
@@ -87,7 +90,7 @@ function renderMountFile(mount) {
     case 'cifs':
         type = 'cifs';
         what = `//${mountOptions.host}` + path.join('/', mountOptions.remoteDir);
-        options = `username=${mountOptions.username},password=${mountOptions.password},rw,iocharset=utf8,file_mode=0666,dir_mode=0777,uid=yellowtent,gid=yellowtent`;
+        options = `username=${mountOptions.username},password=${mountOptions.password},rw,${mountOptions.seal ? 'seal,' : ''}iocharset=utf8,file_mode=0666,dir_mode=0777,uid=yellowtent,gid=yellowtent`;
         break;
     case 'nfs':
         type = 'nfs';

src/network.js

@@ -30,6 +30,7 @@ async function setBlocklist(blocklist, auditSource) {
     for (const line of blocklist.split('\n')) {
         if (!line || line.startsWith('#')) continue;
         const rangeOrIP = line.trim();
+        // this checks for IPv4 and IPv6
         if (!validator.isIP(rangeOrIP) && !validator.isIPRange(rangeOrIP)) throw new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} is not a valid IP or range`);
 
         if (rangeOrIP.indexOf('/') === -1) {

src/nginxconfig.ejs

@@ -12,6 +12,7 @@ map $upstream_http_referrer_policy $hrp {
 
 # http server
 server {
+    # note listen [::]:80 only listens on ipv6 since ipv6only=on since nginx 1.3.4. listen 80 listens on ipv4 only
 <% if (endpoint === 'ip' || endpoint === 'setup') { -%>
     listen 80 default_server;
     server_name _;

src/paths.js

@@ -22,6 +22,7 @@ exports = module.exports = {
     SETUP_TOKEN_FILE: '/etc/cloudron/SETUP_TOKEN',
 
     VOLUMES_MOUNT_DIR: '/mnt/volumes',
+    MANAGED_BACKUP_MOUNT_DIR: '/mnt/cloudronbackup',
 
     PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'),
     APPS_DATA_DIR: path.join(baseDir(), 'appsdata'),
@@ -41,13 +42,13 @@ exports = module.exports = {
     DYNDNS_INFO_FILE: path.join(baseDir(), 'platformdata/dyndns-info.json'),
     DHPARAMS_FILE: path.join(baseDir(), 'platformdata/dhparams.pem'),
     FEATURES_INFO_FILE: path.join(baseDir(), 'platformdata/features-info.json'),
-    PROXY_AUTH_TOKEN_SECRET_FILE: path.join(baseDir(), 'platformdata/proxy-auth-token-secret'),
     VERSION_FILE: path.join(baseDir(), 'platformdata/VERSION'),
     SSHFS_KEYS_DIR: path.join(baseDir(), 'platformdata/sshfs'),
     SFTP_KEYS_DIR: path.join(baseDir(), 'platformdata/sftp/ssh'),
     SFTP_PUBLIC_KEY_FILE: path.join(baseDir(), 'platformdata/sftp/ssh/ssh_host_rsa_key.pub'),
     SFTP_PRIVATE_KEY_FILE: path.join(baseDir(), 'platformdata/sftp/ssh/ssh_host_rsa_key'),
     FIREWALL_BLOCKLIST_FILE: path.join(baseDir(), 'platformdata/firewall/blocklist.txt'),
+    LDAP_ALLOWLIST_FILE: path.join(baseDir(), 'platformdata/firewall/ldap_allowlist.txt'),
 
     BOX_DATA_DIR: path.join(baseDir(), 'boxdata/box'),
     MAIL_DATA_DIR: path.join(baseDir(), 'boxdata/mail'),

src/platform.js

@@ -51,7 +51,10 @@ async function start(options) {
 
     for (let attempt = 0; attempt < 5; attempt++) {
         try {
-            if (existingInfra.version !== infra.version) await removeAllContainers();
+            if (existingInfra.version !== infra.version) {
+                await removeAllContainers();
+                await createDockerNetwork();
+            }
             if (existingInfra.version === 'none') await volumes.mountAll(); // when restoring, mount all volumes
             await markApps(existingInfra, options); // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
             await services.startServices(existingInfra);
@@ -62,7 +65,7 @@ async function start(options) {
             // LOST is when existing connection breaks. REFUSED is when new connection cannot connect at all
             const retry = error.reason === BoxError.DATABASE_ERROR && (error.code === 'PROTOCOL_CONNECTION_LOST' || error.code === 'ECONNREFUSED');
             debug(`Failed to start services. retry=${retry} (attempt ${attempt}): ${error.message}`);
-            if (!retry) break;
+            if (!retry) throw error; // refuse to start
             await delay(10000);
         }
     }
@@ -111,6 +114,14 @@ async function pruneInfraImages() {
     }
 }
 
+async function createDockerNetwork() {
+    debug('createDockerNetwork: recreating docker network');
+
+    await shell.promises.exec('createDockerNetwork', 'docker network rm cloudron || true');
+    // the --ipv6 option will work even in ipv6 is disabled. fd00 is IPv6 ULA
+    await shell.promises.exec('createDockerNetwork', 'docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 --gateway 172.18.0.1 --ipv6 --subnet=fd00:c107:d509::/64 cloudron');
+}
+
 async function removeAllContainers() {
     debug('removeAllContainers: removing all containers for infra upgrade');
 

src/promise-retry.js

@@ -14,9 +14,11 @@ async function promiseRetry(options, asyncFunction) {
 
     for (let i = 0; i < times; i++) {
         try {
-            return await asyncFunction();
+            return await asyncFunction(i+1 /* attempt */);
         } catch (error) {
             if (i === times - 1) throw error;
+            if (options.retry && !options.retry(error)) throw error; // no more retry
+            if (options.debug) options.debug(`Attempt ${i+1} failed. Will retry: ${error.message}`);
             await delay(interval);
         }
     }

src/provision.js

@@ -83,8 +83,8 @@ async function setupTask(domain, auditSource) {
     gProvisionStatus.setup.active = false;
 }
 
-async function setup(dnsConfig, sysinfoConfig, auditSource) {
-    assert.strictEqual(typeof dnsConfig, 'object');
+async function setup(domainConfig, sysinfoConfig, auditSource) {
+    assert.strictEqual(typeof domainConfig, 'object');
     assert.strictEqual(typeof sysinfoConfig, 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
@@ -98,17 +98,17 @@ async function setup(dnsConfig, sysinfoConfig, auditSource) {
 
         await unprovision();
 
-        const domain = dnsConfig.domain.toLowerCase();
-        const zoneName = dnsConfig.zoneName ? dnsConfig.zoneName : (tld.getDomain(domain) || domain);
+        const domain = domainConfig.domain.toLowerCase();
+        const zoneName = domainConfig.zoneName ? domainConfig.zoneName : (tld.getDomain(domain) || domain);
 
         debug(`setup: Setting up Cloudron with domain ${domain} and zone ${zoneName}`);
 
         const data = {
             zoneName: zoneName,
-            provider: dnsConfig.provider,
-            config: dnsConfig.config,
-            fallbackCertificate: dnsConfig.fallbackCertificate || null,
-            tlsConfig: dnsConfig.tlsConfig || { provider: 'letsencrypt-prod' },
+            provider: domainConfig.provider,
+            config: domainConfig.config,
+            fallbackCertificate: domainConfig.fallbackCertificate || null,
+            tlsConfig: domainConfig.tlsConfig || { provider: 'letsencrypt-prod' },
             dkimSelector: 'cloudron'
         };
 
@@ -142,7 +142,7 @@ async function activate(username, password, email, displayName, ip, auditSource)
     const token = { clientId: tokens.ID_WEBADMIN, identifier: ownerId, expires: Date.now() + constants.DEFAULT_TOKEN_EXPIRATION_MSECS };
     const result = await tokens.add(token);
 
-    eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, {});
+    await eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, {});
 
     setImmediate(() => safe(cloudron.onActivated({}), { debug }));
 
@@ -195,7 +195,7 @@ async function restore(backupConfig, backupId, version, sysinfoConfig, options,
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
-    if (!semver.valid(version)) throw new BoxError(BoxError.BAD_FIELD, 'version is not a valid semver', { field: 'version' });
+    if (!semver.valid(version)) throw new BoxError(BoxError.BAD_FIELD, 'version is not a valid semver');
     if (constants.VERSION !== version) throw new BoxError(BoxError.BAD_STATE, `Run "cloudron-setup --version ${version}" on a fresh Ubuntu installation to restore from this backup`);
 
     if (gProvisionStatus.setup.active || gProvisionStatus.restore.active) throw new BoxError(BoxError.BAD_STATE, 'Already setting up or restoring');
@@ -206,13 +206,13 @@ async function restore(backupConfig, backupId, version, sysinfoConfig, options,
         const activated = await users.isActivated();
         if (activated) throw new BoxError(BoxError.CONFLICT, 'Already activated. Restore with a fresh Cloudron installation.');
 
-        if (mounts.isMountProvider(backupConfig.provider)) {
-            error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions);
+        if (mounts.isManagedProvider(backupConfig.provider)) {
+            const error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions);
             if (error) throw error;
 
             const newMount = {
                 name: 'backup',
-                hostPath: backupConfig.mountPoint,
+                hostPath: paths.MANAGED_BACKUP_MOUNT_DIR,
                 mountType: backupConfig.provider,
                 mountOptions: backupConfig.mountOptions
             };
@@ -230,7 +230,7 @@ async function restore(backupConfig, backupId, version, sysinfoConfig, options,
             backupConfig.encryption = null;
         }
 
-        error = await sysinfo.testConfig(sysinfoConfig);
+        error = await sysinfo.testIPv4Config(sysinfoConfig);
         if (error) throw error;
 
         safe(restoreTask(backupConfig, backupId, sysinfoConfig, options, auditSource), { debug }); // now that args are validated run the task in the background

src/proxyauth.js

@@ -10,11 +10,11 @@ exports = module.exports = {
 const apps = require('./apps.js'),
     assert = require('assert'),
     basicAuth = require('basic-auth'),
+    blobs = require('./blobs.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:proxyAuth'),
     ejs = require('ejs'),
     express = require('express'),
-    fs = require('fs'),
     hat = require('./hat.js'),
     http = require('http'),
     HttpError = require('connect-lastmile').HttpError,
@@ -31,21 +31,21 @@ const apps = require('./apps.js'),
     util = require('util');
 
 let gHttpServer = null;
-let TOKEN_SECRET = null;
+let gTokenSecret = null;
 
 function jwtVerify(req, res, next) {
     const token = req.cookies.authToken;
 
     if (!token) return next();
 
-    jwt.verify(token, TOKEN_SECRET, function (error, decoded) {
+    jwt.verify(token, gTokenSecret, function (error, decoded) {
         if (error) {
-            debug('clearing token', error);
-            res.clearCookie('authToken');
-            return next(new HttpError(403, 'Malformed token or bad signature'));
+            debug('jwtVerify: malformed token or bad signature', error.message);
+            req.user = null;
+        } else {
+            req.user = decoded.user || null;
         }
 
-        req.user = decoded.user || null;
         next();
     });
 }
@@ -95,7 +95,7 @@ async function loginPage(req, res, next) {
     try {
         finalContent = ejs.render(translatedContent, { title, icon, dashboardOrigin });
     } catch (e) {
-        debug('Error rendering proxyauth-login.ejs', e);
+        debug('loginPage: Error rendering proxyauth-login.ejs', e);
         return next(new HttpError(500, 'Login template error'));
     }
 
@@ -115,6 +115,7 @@ function isBrowser(req) {
 // called by nginx to authorize any protected route. this route must return only 2xx or 401/403 (http://nginx.org/en/docs/http/ngx_http_auth_request_module.html)
 function auth(req, res, next) {
     if (!req.user) {
+        res.clearCookie('authToken');
         if (isBrowser(req)) return next(new HttpError(401, 'Unauthorized'));
 
         // the header has to be generated here and cannot be set in nginx config - https://forum.nginx.org/read.php?2,171461,171469#msg-171469
@@ -123,7 +124,7 @@ function auth(req, res, next) {
     }
 
     // user is already authenticated, refresh cookie
-    const token = jwt.sign({ user: req.user }, TOKEN_SECRET, { expiresIn: `${constants.DEFAULT_TOKEN_EXPIRATION_DAYS}d` });
+    const token = jwt.sign({ user: req.user }, gTokenSecret, { expiresIn: `${constants.DEFAULT_TOKEN_EXPIRATION_DAYS}d` });
 
     res.cookie('authToken', token, {
         httpOnly: true,
@@ -171,7 +172,7 @@ async function authorize(req, res, next) {
 
     if (!apps.canAccess(app, req.user)) return next(new HttpError(403, 'Forbidden' ));
 
-    const token = jwt.sign({ user: users.removePrivateFields(req.user) }, TOKEN_SECRET, { expiresIn: `${constants.DEFAULT_TOKEN_EXPIRATION_DAYS}d` });
+    const token = jwt.sign({ user: users.removePrivateFields(req.user) }, gTokenSecret, { expiresIn: `${constants.DEFAULT_TOKEN_EXPIRATION_DAYS}d` });
 
     res.cookie('authToken', token, {
         httpOnly: true,
@@ -240,7 +241,7 @@ function initializeAuthwallExpressSync() {
         .use(middleware.lastMile());
 
     router.get ('/login',     loginPage);
-    router.get ('/auth',      jwtVerify, basicAuthVerify, auth);
+    router.get ('/auth',      jwtVerify, basicAuthVerify, auth); // called by nginx before accessing protected page
     router.post('/login',     json, passwordAuth, authorize);
     router.get ('/logout',    logoutPage);
     router.post('/logout',    json, logout);
@@ -251,11 +252,11 @@ function initializeAuthwallExpressSync() {
 async function start() {
     assert.strictEqual(gHttpServer, null, 'Authwall is already up and running.');
 
-    if (!fs.existsSync(paths.PROXY_AUTH_TOKEN_SECRET_FILE)) {
-        TOKEN_SECRET = hat(64);
-        fs.writeFileSync(paths.PROXY_AUTH_TOKEN_SECRET_FILE, TOKEN_SECRET, 'utf8');
-    } else {
-        TOKEN_SECRET = fs.readFileSync(paths.PROXY_AUTH_TOKEN_SECRET_FILE, 'utf8').trim();
+    gTokenSecret = await blobs.getString(blobs.PROXY_AUTH_TOKEN_SECRET);
+    if (!gTokenSecret) {
+        debug('start: generating new token secret');
+        gTokenSecret = hat(64);
+        await blobs.setString(blobs.PROXY_AUTH_TOKEN_SECRET, gTokenSecret);
     }
 
     gHttpServer = initializeAuthwallExpressSync();

src/reverseproxy.js

@@ -152,29 +152,29 @@ function validateCertificate(location, domainObject, certificate) {
     const cert = certificate.cert, key = certificate.key;
 
     // check for empty cert and key strings
-    if (!cert && key) return new BoxError(BoxError.BAD_FIELD, 'missing cert', { field: 'cert' });
-    if (cert && !key) return new BoxError(BoxError.BAD_FIELD, 'missing key', { field: 'key' });
+    if (!cert && key) return new BoxError(BoxError.BAD_FIELD, 'missing cert');
+    if (cert && !key) return new BoxError(BoxError.BAD_FIELD, 'missing key');
 
     // -checkhost checks for SAN or CN exclusively. SAN takes precedence and if present, ignores the CN.
     const fqdn = dns.fqdn(location, domainObject);
 
     let result = safe.child_process.execSync(`openssl x509 -noout -checkhost "${fqdn}"`, { encoding: 'utf8', input: cert });
-    if (result === null) return new BoxError(BoxError.BAD_FIELD, 'Unable to get certificate subject:' + safe.error.message, { field: 'cert' });
+    if (result === null) return new BoxError(BoxError.BAD_FIELD, 'Unable to get certificate subject:' + safe.error.message);
 
-    if (result.indexOf('does match certificate') === -1) return new BoxError(BoxError.BAD_FIELD, `Certificate is not valid for this domain. Expecting ${fqdn}`, { field: 'cert' });
+    if (result.indexOf('does match certificate') === -1) return new BoxError(BoxError.BAD_FIELD, `Certificate is not valid for this domain. Expecting ${fqdn}`);
 
     // check if public key in the cert and private key matches. pkey below works for RSA and ECDSA keys
     const pubKeyFromCert = safe.child_process.execSync('openssl x509 -noout -pubkey', { encoding: 'utf8', input: cert });
-    if (pubKeyFromCert === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from cert: ${safe.error.message}`, { field: 'cert' });
+    if (pubKeyFromCert === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from certificate: ${safe.error.message}`);
 
     const pubKeyFromKey = safe.child_process.execSync('openssl pkey -pubout', { encoding: 'utf8', input: key });
-    if (pubKeyFromKey === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from private key: ${safe.error.message}`, { field: 'cert' });
+    if (pubKeyFromKey === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from private key: ${safe.error.message}`);
 
-    if (pubKeyFromCert !== pubKeyFromKey) return new BoxError(BoxError.BAD_FIELD, 'Public key does not match the certificate.', { field: 'cert' });
+    if (pubKeyFromCert !== pubKeyFromKey) return new BoxError(BoxError.BAD_FIELD, 'Public key does not match the certificate.');
 
     // check expiration
     result = safe.child_process.execSync('openssl x509 -checkend 0', { encoding: 'utf8', input: cert });
-    if (!result) return new BoxError(BoxError.BAD_FIELD, 'Certificate has expired.', { field: 'cert' });
+    if (!result) return new BoxError(BoxError.BAD_FIELD, 'Certificate has expired.');
 
     return null;
 }
@@ -429,10 +429,9 @@ async function ensureCertificate(vhost, domain, auditSource) {
     return { bundle: getFallbackCertificatePathSync(domain), renewed: false };
 }
 
-async function writeDashboardNginxConfig(bundle, configFileName, vhost) {
-    assert.strictEqual(typeof bundle, 'object');
-    assert.strictEqual(typeof configFileName, 'string');
+async function writeDashboardNginxConfig(vhost, bundle) {
     assert.strictEqual(typeof vhost, 'string');
+    assert.strictEqual(typeof bundle, 'object');
 
     const data = {
         sourceDir: path.resolve(__dirname, '..'),
@@ -446,104 +445,99 @@ async function writeDashboardNginxConfig(bundle, configFileName, vhost) {
         ocsp: await isOcspEnabled(bundle.certFilePath)
     };
     const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
-    const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, configFileName);
+    const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${vhost}.conf`);
 
     if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) throw new BoxError(BoxError.FS_ERROR, safe.error);
 
     await reload();
 }
 
-async function writeDashboardConfig(domain) {
-    assert.strictEqual(typeof domain, 'string');
+async function writeDashboardConfig(domainObject) {
+    assert.strictEqual(typeof domainObject, 'object');
 
-    debug(`writeDashboardConfig: writing admin config for ${domain}`);
-
-    const domainObject = await domains.get(domain);
+    debug(`writeDashboardConfig: writing admin config for ${domainObject.domain}`);
 
     const dashboardFqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
-
     const bundle = await getCertificatePath(dashboardFqdn, domainObject.domain);
 
-    await writeDashboardNginxConfig(bundle, `${dashboardFqdn}.conf`, dashboardFqdn);
+    await writeDashboardNginxConfig(dashboardFqdn, bundle);
 }
 
-async function writeAppNginxConfig(app, fqdn, bundle) {
+function getNginxConfigFilename(app, fqdn, type) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof fqdn, 'string');
-    assert.strictEqual(typeof bundle, 'object');
+    assert.strictEqual(typeof type, 'string');
 
-    var sourceDir = path.resolve(__dirname, '..');
-    var endpoint = 'app';
+    let nginxConfigFilenameSuffix = '';
 
-    let robotsTxtQuoted = null, hideHeaders = [], cspQuoted = null;
-    const reverseProxyConfig = app.reverseProxyConfig || {}; // some of our code uses fake app objects
-    if (reverseProxyConfig.robotsTxt) robotsTxtQuoted = JSON.stringify(app.reverseProxyConfig.robotsTxt);
-    if (reverseProxyConfig.csp) {
-        cspQuoted = `"${app.reverseProxyConfig.csp}"`;
-        hideHeaders = [ 'Content-Security-Policy' ];
-        if (reverseProxyConfig.csp.includes('frame-ancestors ')) hideHeaders.push('X-Frame-Options');
+    if (type === apps.LOCATION_TYPE_ALIAS) {
+        nginxConfigFilenameSuffix = `-alias-${fqdn.replace('*', '_')}`;
+    } else if (type === apps.LOCATION_TYPE_SECONDARY) {
+        nginxConfigFilenameSuffix = `-secondary-${fqdn}`;
+    } else if (type === apps.LOCATION_TYPE_REDIRECT) {
+        nginxConfigFilenameSuffix = `-redirect-${fqdn}`;
     }
 
-    const data = {
-        sourceDir: sourceDir,
-        vhost: fqdn,
-        hasIPv6: sysinfo.hasIPv6(),
-        ip: app.containerIp,
-        port: app.manifest.httpPort,
-        endpoint,
-        certFilePath: bundle.certFilePath,
-        keyFilePath: bundle.keyFilePath,
-        robotsTxtQuoted,
-        cspQuoted,
-        hideHeaders,
-        proxyAuth: {
-            enabled: app.sso && app.manifest.addons && app.manifest.addons.proxyAuth,
-            id: app.id,
-            location: nginxLocation(safe.query(app.manifest, 'addons.proxyAuth.path') || '/')
-        },
-        ocsp: await isOcspEnabled(bundle.certFilePath)
-    };
-    const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
-
-    const aliasSuffix = app.fqdn === fqdn ? '' : `-alias-${fqdn.replace('*', '_')}`;
-    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}${aliasSuffix}.conf`);
-    debug('writeAppNginxConfig: writing config for "%s" to %s with options %j', fqdn, nginxConfigFilename, data);
-
-    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
-        debug('Error creating nginx config for "%s" : %s', app.fqdn, safe.error.message);
-        throw new BoxError(BoxError.FS_ERROR, safe.error);
-    }
-
-    await reload();
+    return path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}${nginxConfigFilenameSuffix}.conf`);
 }
 
-async function writeAppRedirectNginxConfig(app, fqdn, bundle) {
+async function writeAppNginxConfig(app, fqdn, type, bundle) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof fqdn, 'string');
+    assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof bundle, 'object');
 
     const data = {
         sourceDir: path.resolve(__dirname, '..'),
         vhost: fqdn,
-        redirectTo: app.fqdn,
         hasIPv6: sysinfo.hasIPv6(),
-        endpoint: 'redirect',
+        ip: null,
+        port: null,
+        endpoint: null,
+        redirectTo: null,
         certFilePath: bundle.certFilePath,
         keyFilePath: bundle.keyFilePath,
         robotsTxtQuoted: null,
         cspQuoted: null,
         hideHeaders: [],
-        proxyAuth: { enabled: false, id: app.id, location: nginxLocation('/') },
+        proxyAuth: { enabled: false },
         ocsp: await isOcspEnabled(bundle.certFilePath)
     };
+
+    if (type === apps.LOCATION_TYPE_PRIMARY || type === apps.LOCATION_TYPE_ALIAS || type === apps.LOCATION_TYPE_SECONDARY) {
+        data.endpoint = 'app';
+        // maybe these should become per domain at some point
+        const reverseProxyConfig = app.reverseProxyConfig || {}; // some of our code uses fake app objects
+        if (reverseProxyConfig.robotsTxt) data.robotsTxtQuoted = JSON.stringify(app.reverseProxyConfig.robotsTxt);
+        if (reverseProxyConfig.csp) {
+            data.cspQuoted = `"${app.reverseProxyConfig.csp}"`;
+            data.hideHeaders = [ 'Content-Security-Policy' ];
+            if (reverseProxyConfig.csp.includes('frame-ancestors ')) data.hideHeaders.push('X-Frame-Options');
+        }
+        if (type === apps.LOCATION_TYPE_PRIMARY || type == apps.LOCATION_TYPE_ALIAS) {
+            data.proxyAuth = {
+                enabled: app.sso && app.manifest.addons && app.manifest.addons.proxyAuth,
+                id: app.id,
+                location: nginxLocation(safe.query(app.manifest, 'addons.proxyAuth.path') || '/')
+            };
+            data.ip = app.containerIp;
+            data.port = app.manifest.httpPort;
+        } else if (type === apps.LOCATION_TYPE_SECONDARY) {
+            data.ip = app.containerIp;
+            const secondaryDomain = app.secondaryDomains.find(sd => sd.fqdn === fqdn);
+            data.port = app.manifest.httpPorts[secondaryDomain.environmentVariable].containerPort;
+        }
+    } else if (type === apps.LOCATION_TYPE_REDIRECT) {
+        data.proxyAuth = { enabled: false, id: app.id, location: nginxLocation('/') };
+        data.endpoint = 'redirect';
+        data.redirectTo = app.fqdn;
+    }
+
     const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
-
-    // if we change the filename, also change it in unconfigureApp()
-    const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-redirect-${fqdn}.conf`);
-    debug('writing config for "%s" redirecting to "%s" to %s with options %j', app.fqdn, fqdn, nginxConfigFilename, data);
-
-    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
-        debug('Error creating nginx redirect config for "%s" : %s', app.fqdn, safe.error.message);
+    const filename = getNginxConfigFilename(app, fqdn, type);
+    debug(`writeAppNginxConfig: writing config for "${fqdn}" to ${filename} with options ${JSON.stringify(data)}`);
+    if (!safe.fs.writeFileSync(filename, nginxConf)) {
+        debug(`Error creating nginx config for "${app.fqdn}" : ${safe.error.message}`);
         throw new BoxError(BoxError.FS_ERROR, safe.error);
     }
 
@@ -553,27 +547,14 @@ async function writeAppRedirectNginxConfig(app, fqdn, bundle) {
 async function writeAppConfig(app) {
     assert.strictEqual(typeof app, 'object');
 
-    let appDomains = [];
-    appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary' });
-
-    app.alternateDomains.forEach(function (alternateDomain) {
-        appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate' });
-    });
-
-    app.aliasDomains.forEach(function (aliasDomain) {
-        appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias' });
-    });
+    const appDomains = [{ domain: app.domain, fqdn: app.fqdn, type: apps.LOCATION_TYPE_PRIMARY }]
+        .concat(app.secondaryDomains.map(sd => { return { domain: sd.domain, fqdn: sd.fqdn, type: apps.LOCATION_TYPE_SECONDARY }; }))
+        .concat(app.redirectDomains.map(rd => { return { domain: rd.domain, fqdn: rd.fqdn, type: apps.LOCATION_TYPE_REDIRECT }; }))
+        .concat(app.aliasDomains.map(ad => { return { domain: ad.domain, fqdn: ad.fqdn, type: apps.LOCATION_TYPE_ALIAS }; }));
 
     for (const appDomain of appDomains) {
         const bundle = await getCertificatePath(appDomain.fqdn, appDomain.domain);
-
-        if (appDomain.type === 'primary') {
-            await writeAppNginxConfig(app, appDomain.fqdn, bundle);
-        } else if (appDomain.type === 'alternate') {
-            await writeAppRedirectNginxConfig(app, appDomain.fqdn, bundle);
-        } else if (appDomain.type === 'alias') {
-            await writeAppNginxConfig(app, appDomain.fqdn, bundle);
-        }
+        await writeAppNginxConfig(app, appDomain.fqdn, appDomain.type, bundle);
     }
 }
 
@@ -581,28 +562,16 @@ async function configureApp(app, auditSource) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
-    let appDomains = [];
-    appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary' });
-
-    app.alternateDomains.forEach(function (alternateDomain) {
-        appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate' });
-    });
-
-    app.aliasDomains.forEach(function (aliasDomain) {
-        appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias' });
-    });
+    const appDomains = [{ domain: app.domain, fqdn: app.fqdn }]
+        .concat(app.secondaryDomains.map(sd => { return { domain: sd.domain, fqdn: sd.fqdn }; }))
+        .concat(app.redirectDomains.map(rd => { return { domain: rd.domain, fqdn: rd.fqdn }; }))
+        .concat(app.aliasDomains.map(ad => { return { domain: ad.domain, fqdn: ad.fqdn }; }));
 
     for (const appDomain of appDomains) {
-        const { bundle } = await ensureCertificate(appDomain.fqdn, appDomain.domain, auditSource);
-
-        if (appDomain.type === 'primary') {
-            await writeAppNginxConfig(app, appDomain.fqdn, bundle);
-        } else if (appDomain.type === 'alternate') {
-            await writeAppRedirectNginxConfig(app, appDomain.fqdn, bundle);
-        } else if (appDomain.type === 'alias') {
-            await writeAppNginxConfig(app, appDomain.fqdn, bundle);
-        }
+        await ensureCertificate(appDomain.fqdn, appDomain.domain, auditSource);
     }
+
+    await writeAppConfig(app);
 }
 
 async function unconfigureApp(app) {
@@ -640,17 +609,10 @@ async function renewCerts(options, auditSource, progressCallback) {
     for (const app of allApps) {
         if (app.runState === apps.RSTATE_STOPPED) continue; // do not renew certs of stopped apps
 
-        appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary', app: app, nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf') });
-
-        app.alternateDomains.forEach(function (alternateDomain) {
-            const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-redirect-${alternateDomain.fqdn}.conf`);
-            appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate', app: app, nginxConfigFilename });
-        });
-
-        app.aliasDomains.forEach(function (aliasDomain) {
-            const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-alias-${aliasDomain.fqdn.replace('*', '_')}.conf`);
-            appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias', app: app, nginxConfigFilename });
-        });
+        appDomains = appDomains.concat([{ app, domain: app.domain, fqdn: app.fqdn, type: apps.LOCATION_TYPE_PRIMARY }])
+            .concat(app.secondaryDomains.map(sd => { return { app, domain: sd.domain, fqdn: sd.fqdn, type: apps.LOCATION_TYPE_SECONDARY, nginxConfigFilename: getNginxConfigFilename(app, sd.fqdn, apps.LOCATION_TYPE_SECONDARY) }; }))
+            .concat(app.redirectDomains.map(rd => { return { app, domain: rd.domain, fqdn: rd.fqdn, type: apps.LOCATION_TYPE_REDIRECT, nginxConfigFilename: getNginxConfigFilename(app, rd.fqdn, apps.LOCATION_TYPE_REDIRECT) }; }))
+            .concat(app.aliasDomains.map(ad => { return { app, domain: ad.domain, fqdn: ad.fqdn, type: apps.LOCATION_TYPE_ALIAS, nginxConfigFilename: getNginxConfigFilename(app, ad.fqdn, apps.LOCATION_TYPE_ALIAS) }; }));
     }
 
     if (options.domain) appDomains = appDomains.filter(function (appDomain) { return appDomain.domain === options.domain; });
@@ -675,15 +637,9 @@ async function renewCerts(options, auditSource, progressCallback) {
 
         // reconfigure since the cert changed
         if (appDomain.type === 'webadmin' || appDomain.type === 'webadmin+mail') {
-            await writeDashboardNginxConfig(bundle, `${settings.dashboardFqdn()}.conf`, settings.dashboardFqdn());
-        } else if (appDomain.type === 'primary') {
-            await writeAppNginxConfig(appDomain.app, appDomain.fqdn, bundle);
-        } else if (appDomain.type === 'alternate') {
-            await writeAppRedirectNginxConfig(appDomain.app, appDomain.fqdn, bundle);
-        } else if (appDomain.type === 'alias') {
-            await writeAppNginxConfig(appDomain.app, appDomain.fqdn, bundle);
+            await writeDashboardNginxConfig(settings.dashboardFqdn(), bundle);
         } else {
-            throw new BoxError(BoxError.INTERNAL_ERROR, `Unknown domain type for ${appDomain.fqdn}. This should never happen`);
+            await writeAppNginxConfig(appDomain.app, appDomain.fqdn, appDomain.type, bundle);
         }
     }
 
@@ -701,13 +657,17 @@ async function renewCerts(options, auditSource, progressCallback) {
     }
 }
 
-async function cleanupCerts() {
+async function cleanupCerts(auditSource) {
+    assert.strictEqual(typeof auditSource, 'object');
+
     const filenames = await fs.promises.readdir(paths.NGINX_CERT_DIR);
     const certFilenames = filenames.filter(f => f.endsWith('.cert'));
     const now = new Date();
 
     debug('cleanupCerts: start');
 
+    const fqdns = [];
+
     for (const certFilename of certFilenames) {
         const certFilePath = path.join(paths.NGINX_CERT_DIR, certFilename);
         const notAfter = getExpiryDate(certFilePath);
@@ -717,6 +677,7 @@ async function cleanupCerts() {
             const fqdn = certFilename.replace(/\.cert$/, '');
             debug(`cleanupCerts: deleting certs of ${fqdn}`);
 
+            // it is safe to delete the certs of stopped apps because their nginx configs are removed
             safe.fs.unlinkSync(certFilePath);
             safe.fs.unlinkSync(path.join(paths.NGINX_CERT_DIR, `${fqdn}.key`));
             safe.fs.unlinkSync(path.join(paths.NGINX_CERT_DIR, `${fqdn}.csr`));
@@ -724,9 +685,13 @@ async function cleanupCerts() {
             await blobs.del(`${blobs.CERT_PREFIX}-${fqdn}.key`);
             await blobs.del(`${blobs.CERT_PREFIX}-${fqdn}.cert`);
             await blobs.del(`${blobs.CERT_PREFIX}-${fqdn}.csr`);
+
+            fqdns.push(fqdn);
         }
     }
 
+    if (fqdns.length) await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_CLEANUP, auditSource, { domains: fqdns }));
+
     debug('cleanupCerts: done');
 }
 
@@ -736,12 +701,14 @@ async function checkCerts(options, auditSource, progressCallback) {
     assert.strictEqual(typeof progressCallback, 'function');
 
     await renewCerts(options, auditSource, progressCallback);
-    await cleanupCerts();
+    await cleanupCerts(auditSource);
 }
 
 function removeAppConfigs() {
     const dashboardConfigFilename = `${settings.dashboardFqdn()}.conf`;
 
+    debug('removeAppConfigs: reomving nginx configs of apps');
+
     // remove all configs which are not the default or current dashboard
     for (let appConfigFile of fs.readdirSync(paths.NGINX_APPCONFIG_DIR)) {
         if (appConfigFile !== constants.NGINX_DEFAULT_CONFIG_FILE_NAME && appConfigFile !== dashboardConfigFilename) {

src/routes/apps.js

@@ -123,7 +123,7 @@ async function install(req, res, next) {
     if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
 
     // required
-    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
+    if (typeof data.subdomain !== 'string') return next(new HttpError(400, 'subdomain is required'));
     if (typeof data.domain !== 'string') return next(new HttpError(400, 'domain is required'));
     if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
 
@@ -141,9 +141,14 @@ async function install(req, res, next) {
 
     if (('debugMode' in data) && typeof data.debugMode !== 'object') return next(new HttpError(400, 'debugMode must be an object'));
 
-    if ('alternateDomains' in data) {
-        if (!Array.isArray(data.alternateDomains)) return next(new HttpError(400, 'alternateDomains must be an array'));
-        if (data.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
+    if ('secondaryDomains' in data) {
+        if (!data.secondaryDomains || typeof data.secondaryDomains !== 'object') return next(new HttpError(400, 'secondaryDomains must be an object'));
+        if (Object.keys(data.secondaryDomains).some(function (key) { return typeof data.secondaryDomains[key].domain !== 'string' || typeof data.secondaryDomains[key].subdomain !== 'string'; })) return next(new HttpError(400, 'secondaryDomain object must contain domain and subdomain strings'));
+    }
+
+    if ('redirectDomains' in data) {
+        if (!Array.isArray(data.redirectDomains)) return next(new HttpError(400, 'redirectDomains must be an array'));
+        if (data.redirectDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'redirectDomains array must contain objects with domain and subdomain strings'));
     }
 
     if ('aliasDomains' in data) {
@@ -314,7 +319,7 @@ async function setCertificate(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.app, 'object');
 
-    if (typeof req.body.location !== 'string') return next(new HttpError(400, 'location must be string')); // location may be an empty string
+    if (typeof req.body.subdomain !== 'string') return next(new HttpError(400, 'subdomain must be string')); // subdomain may be an empty string
     if (!req.body.domain) return next(new HttpError(400, 'domain is required'));
     if (typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be string'));
 
@@ -390,15 +395,20 @@ async function setLocation(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.app, 'object');
 
-    if (typeof req.body.location !== 'string') return next(new HttpError(400, 'location must be string')); // location may be an empty string
+    if (typeof req.body.subdomain !== 'string') return next(new HttpError(400, 'subdomain must be string')); // subdomain may be an empty string
     if (!req.body.domain) return next(new HttpError(400, 'domain is required'));
     if (typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be string'));
 
     if ('portBindings' in req.body && typeof req.body.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
 
-    if ('alternateDomains' in req.body) {
-        if (!Array.isArray(req.body.alternateDomains)) return next(new HttpError(400, 'alternateDomains must be an array'));
-        if (req.body.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
+    if ('secondaryDomains' in req.body) {
+        if (!req.body.secondaryDomains || typeof req.body.secondaryDomains !== 'object') return next(new HttpError(400, 'secondaryDomains must be an object'));
+        if (Object.keys(req.body.secondaryDomains).some(function (key) { return typeof req.body.secondaryDomains[key].domain !== 'string' || typeof req.body.secondaryDomains[key].subdomain !== 'string'; })) return next(new HttpError(400, 'secondaryDomain object must contain domain and subdomain strings'));
+    }
+
+    if ('redirectDomains' in req.body) {
+        if (!Array.isArray(req.body.redirectDomains)) return next(new HttpError(400, 'redirectDomains must be an array'));
+        if (req.body.redirectDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'redirectDomains array must contain objects with domain and subdomain strings'));
     }
 
     if ('aliasDomains' in req.body) {
@@ -510,10 +520,15 @@ async function clone(req, res, next) {
     var data = req.body;
 
     if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be a string'));
-    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
+    if (typeof data.subdomain !== 'string') return next(new HttpError(400, 'subdomain is required'));
     if (typeof data.domain !== 'string') return next(new HttpError(400, 'domain is required'));
     if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
 
+    if ('secondaryDomains' in data) {
+        if (!data.secondaryDomains || typeof data.secondaryDomains !== 'object') return next(new HttpError(400, 'secondaryDomains must be an object'));
+        if (Object.keys(data.secondaryDomains).some(function (key) { return typeof data.secondaryDomains[key].domain !== 'string' || typeof data.secondaryDomains[key].subdomain !== 'string'; })) return next(new HttpError(400, 'secondaryDomain object must contain domain and subdomain strings'));
+    }
+
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
     if ('skipDnsSetup' in req.body && typeof req.body.skipDnsSetup !== 'boolean') return next(new HttpError(400, 'skipDnsSetup must be boolean'));
 

src/routes/branding.js

@@ -41,7 +41,7 @@ async function setCloudronName(req, res, next) {
     const [error] = await safe(settings.setCloudronName(req.body.name));
     if (error) return next(BoxError.toHttpError(error));
 
-    next(new HttpSuccess(202, {}));
+    next(new HttpSuccess(200, {}));
 }
 
 async function getCloudronName(req, res, next) {
@@ -72,7 +72,7 @@ async function setAppstoreListingConfig(req, res, next) {
     const [error] = await safe(settings.setAppstoreListingConfig(listingConfig));
     if (error) return next(BoxError.toHttpError(error));
 
-    next(new HttpSuccess(202, {}));
+    next(new HttpSuccess(200, {}));
 }
 
 async function getAppstoreListingConfig(req, res, next) {
@@ -92,7 +92,7 @@ async function setCloudronAvatar(req, res, next) {
     const [error] = await safe(settings.setCloudronAvatar(avatar));
     if (error) return next(BoxError.toHttpError(error));
 
-    next(new HttpSuccess(202, {}));
+    next(new HttpSuccess(200, {}));
 }
 
 async function getCloudronAvatar(req, res, next) {

src/routes/cloudron.js

@@ -19,7 +19,8 @@ exports = module.exports = {
     updateDashboardDomain,
     prepareDashboardDomain,
     renewCerts,
-    getServerIp,
+    getServerIpv4,
+    getServerIpv6,
     getLanguages,
     syncExternalLdap,
     syncDnsRecords
@@ -62,7 +63,7 @@ async function login(req, res, next) {
     [error, token] = await safe(tokens.add({ clientId: type, identifier: req.user.id, expires: Date.now() + constants.DEFAULT_TOKEN_EXPIRATION_MSECS }));
     if (error) return next(new HttpError(500, error));
 
-    eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+    await eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
     if (!req.user.ghost) safe(users.notifyLoginLocation(req.user, ip, userAgent, auditSource), { debug });
 
@@ -72,7 +73,7 @@ async function login(req, res, next) {
 async function logout(req, res) {
     assert.strictEqual(typeof req.access_token, 'string');
 
-    eventlog.add(eventlog.ACTION_USER_LOGOUT, AuditSource.fromRequest(req), { userId: req.user.id, user: users.removePrivateFields(req.user) });
+    await eventlog.add(eventlog.ACTION_USER_LOGOUT, AuditSource.fromRequest(req), { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
     await safe(tokens.delByAccessToken(req.access_token));
     res.redirect('/login.html');
@@ -134,9 +135,6 @@ async function setupAccount(req, res, next) {
     if (error) return next(new HttpError(401, 'Invalid inviteToken'));
     if (!userObject) return next(new HttpError(401, 'Invalid inviteToken'));
 
-    // if you fix the duration here, the emails and UI have to be fixed as well
-    if (Date.now() - userObject.resetTokenCreationTime > 7 * 24 * 60 * 60 * 1000) return next(new HttpError(401, 'Token expired'));
-
     const [setupAccountError, accessToken] = await safe(users.setupAccount(userObject, req.body, AuditSource.fromRequest(req)));
     if (setupAccountError) return next(BoxError.toHttpError(setupAccountError));
 
@@ -296,11 +294,18 @@ async function syncExternalLdap(req, res, next) {
     next(new HttpSuccess(202, { taskId }));
 }
 
-async function getServerIp(req, res, next) {
-    const [error, ip] = await safe(sysinfo.getServerIp());
+async function getServerIpv4(req, res, next) {
+    const [error, ipv4] = await safe(sysinfo.getServerIPv4());
     if (error) return next(BoxError.toHttpError(error));
 
-    next(new HttpSuccess(200, { ip }));
+    next(new HttpSuccess(200, { ipv4 }));
+}
+
+async function getServerIpv6(req, res, next) {
+    const [error, ipv6] = await safe(sysinfo.getServerIPv6()); // ignore any error
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, { ipv6 }));
 }
 
 async function getLanguages(req, res, next) {

src/routes/domains.js

@@ -4,7 +4,8 @@ exports = module.exports = {
     add,
     get,
     list,
-    update,
+    setConfig,
+    setWellKnown,
     del,
 
     checkDnsRecords,
@@ -76,7 +77,7 @@ async function list(req, res, next) {
     next(new HttpSuccess(200, { domains: results }));
 }
 
-async function update(req, res, next) {
+async function setConfig(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.body, 'object');
 
@@ -98,26 +99,33 @@ async function update(req, res, next) {
         if (!req.body.tlsConfig.provider || typeof req.body.tlsConfig.provider !== 'string') return next(new HttpError(400, 'tlsConfig.provider must be a string'));
     }
 
-    if ('wellKnown' in req.body) {
-        if (typeof req.body.wellKnown !== 'object') return next(new HttpError(400, 'wellKnown must be an object'));
-        if (req.body.wellKnown) {
-            if (Object.keys(req.body.wellKnown).some(k => typeof req.body.wellKnown[k] !== 'string')) return next(new HttpError(400, 'wellKnown is a map of strings'));
-        }
-    }
-
     // some DNS providers like DigitalOcean take a really long time to verify credentials (https://github.com/expressjs/timeout/issues/26)
     req.clearTimeout();
 
-    let data = {
+    const data = {
         zoneName: req.body.zoneName || '',
         provider: req.body.provider,
         config: req.body.config,
         fallbackCertificate: req.body.fallbackCertificate || null,
-        tlsConfig: req.body.tlsConfig || { provider: 'letsencrypt-prod' },
-        wellKnown: req.body.wellKnown || null
+        tlsConfig: req.body.tlsConfig || { provider: 'letsencrypt-prod' }
     };
 
-    const [error] = await safe(domains.update(req.params.domain, data, AuditSource.fromRequest(req)));
+    const [error] = await safe(domains.setConfig(req.params.domain, data, AuditSource.fromRequest(req)));
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(204, {}));
+}
+
+async function setWellKnown(req, res, next) {
+    assert.strictEqual(typeof req.params.domain, 'string');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.wellKnown !== 'object') return next(new HttpError(400, 'wellKnown must be an object'));
+    if (req.body.wellKnown) {
+        if (Object.keys(req.body.wellKnown).some(k => typeof req.body.wellKnown[k] !== 'string')) return next(new HttpError(400, 'wellKnown is a map of strings'));
+    }
+
+    const [error] = await safe(domains.setWellKnown(req.params.domain, req.body.wellKnown || null, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(204, {}));

src/routes/filemanager.js

@@ -33,7 +33,7 @@ function proxy(kind) {
 
         req.url = url.format({ pathname: `/files/${id}/${encodeURIComponent(req.params[0])}`, query: parsedUrl.query }); // params[0] already contains leading '/'
 
-        const proxyOptions = url.parse(`https://${result.ip}:3000`);
+        const proxyOptions = url.parse(`http://${result.ip}:3000`);
         proxyOptions.rejectUnauthorized = false;
         const fileManagerProxy = middleware.proxy(proxyOptions);
 

src/routes/mail.js

@@ -233,7 +233,7 @@ async function setAliases(req, res, next) {
         if (typeof alias.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
     }
 
-    const [error] = await safe(mail.setAliases(req.params.name, req.params.domain, req.body.aliases));
+    const [error] = await safe(mail.setAliases(req.params.name, req.params.domain, req.body.aliases, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(202));

src/routes/mailserver.js

@@ -20,8 +20,9 @@ const assert = require('assert'),
     services = require('../services.js'),
     url = require('url');
 
-function restart(req, res, next) {
-    mail.restartMail((error) => debug('Error restarting mail container', error));
+// because of how the proxy middleware works, the http response is already sent by the time this function is called
+async function restart(req, res, next) {
+    await safe(mail.restartMail(), { debug });
     next();
 }
 
@@ -40,8 +41,7 @@ async function proxy(req, res, next) {
     parsedUrl.query['access_token'] = addonDetails.token;
     req.url = url.format({ pathname: pathname, query: parsedUrl.query });
 
-    const proxyOptions = url.parse(`https://${addonDetails.ip}:3000`);
-    proxyOptions.rejectUnauthorized = false;
+    const proxyOptions = url.parse(`http://${addonDetails.ip}:3000`);
     const mailserverProxy = middleware.proxy(proxyOptions);
 
     req.clearTimeout(); // TODO: add timeout to mail server proxy logic instead of this
@@ -72,4 +72,4 @@ async function setLocation(req, res, next) {
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(202, { taskId }));
-}
+}

src/routes/profile.js

@@ -26,10 +26,10 @@ const assert = require('assert'),
 async function authorize(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    const [error, directoryConfig] = await safe(settings.getDirectoryConfig());
+    const [error, profileConfig] = await safe(settings.getProfileConfig());
     if (error) return next(BoxError.toHttpError(error));
 
-    if (directoryConfig.lockUserProfiles) return next(new HttpError(403, 'admin has disallowed users from editing profiles'));
+    if (profileConfig.lockUserProfiles) return next(new HttpError(403, 'admin has disallowed users from editing profiles'));
 
     next();
 }

src/routes/provision.js

@@ -38,11 +38,10 @@ async function providerTokenAuth(req, res, next) {
     if (settings.provider() === 'ami') {
         if (typeof req.body.providerToken !== 'string' || !req.body.providerToken) return next(new HttpError(400, 'providerToken must be a non empty string'));
 
-        const [error, response] = await superagent.get('http://169.254.169.254/latest/meta-data/instance-id').timeout(30 * 1000).ok(() => true);
+        const [error, response] = await safe(superagent.get('http://169.254.169.254/latest/meta-data/instance-id').timeout(30 * 1000).ok(() => true));
         if (error) return next(new HttpError(422, `Network error getting EC2 metadata: ${error.message}`));
-        if (response.statusCode !== 200) return next(new HttpError(422, `Unable to get EC2 meta data. statusCode: ${response.status}`));
-
-        if (response.body !== req.body.providerToken) return next(new HttpError(422, 'Instance ID does not match'));
+        if (response.status !== 200) return next(new HttpError(422, `Unable to get EC2 meta data. statusCode: ${response.status}`));
+        if (response.text !== req.body.providerToken) return next(new HttpError(422, 'Instance ID does not match'));
 
         next();
     } else {
@@ -53,25 +52,25 @@ async function providerTokenAuth(req, res, next) {
 async function setup(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (!req.body.dnsConfig || typeof req.body.dnsConfig !== 'object') return next(new HttpError(400, 'dnsConfig is required'));
+    if (!req.body.domainConfig || typeof req.body.domainConfig !== 'object') return next(new HttpError(400, 'domainConfig is required'));
 
-    const dnsConfig = req.body.dnsConfig;
+    const domainConfig = req.body.domainConfig;
 
-    if (typeof dnsConfig.provider !== 'string' || !dnsConfig.provider) return next(new HttpError(400, 'provider is required'));
-    if (typeof dnsConfig.domain !== 'string' || !dnsConfig.domain) return next(new HttpError(400, 'domain is required'));
+    if (typeof domainConfig.provider !== 'string' || !domainConfig.provider) return next(new HttpError(400, 'provider is required'));
+    if (typeof domainConfig.domain !== 'string' || !domainConfig.domain) return next(new HttpError(400, 'domain is required'));
 
-    if ('zoneName' in dnsConfig && typeof dnsConfig.zoneName !== 'string') return next(new HttpError(400, 'zoneName must be a string'));
-    if (!dnsConfig.config || typeof dnsConfig.config !== 'object') return next(new HttpError(400, 'config must be an object'));
+    if ('zoneName' in domainConfig && typeof domainConfig.zoneName !== 'string') return next(new HttpError(400, 'zoneName must be a string'));
+    if (!domainConfig.config || typeof domainConfig.config !== 'object') return next(new HttpError(400, 'config must be an object'));
 
-    if ('tlsConfig' in dnsConfig && typeof dnsConfig.tlsConfig !== 'object') return next(new HttpError(400, 'tlsConfig must be an object'));
-    if (dnsConfig.tlsConfig && (!dnsConfig.tlsConfig.provider || typeof dnsConfig.tlsConfig.provider !== 'string')) return next(new HttpError(400, 'tlsConfig.provider must be a string'));
+    if ('tlsConfig' in domainConfig && typeof domainConfig.tlsConfig !== 'object') return next(new HttpError(400, 'tlsConfig must be an object'));
+    if (domainConfig.tlsConfig && (!domainConfig.tlsConfig.provider || typeof domainConfig.tlsConfig.provider !== 'string')) return next(new HttpError(400, 'tlsConfig.provider must be a string'));
 
     if ('sysinfoConfig' in req.body && typeof req.body.sysinfoConfig !== 'object') return next(new HttpError(400, 'sysinfoConfig must be an object'));
 
     // it can take sometime to setup DNS, register cloudron
     req.clearTimeout();
 
-    const [error] = await safe(provision.setup(dnsConfig, req.body.sysinfoConfig || { provider: 'generic' }, AuditSource.fromRequest(req)));
+    const [error] = await safe(provision.setup(domainConfig, req.body.sysinfoConfig || { provider: 'generic' }, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(200, {}));

src/routes/settings.js

@@ -5,7 +5,7 @@ exports = module.exports = {
     get,
 
     // owner only settings
-    setBackupConfig
+    setBackupConfig,
 };
 
 const assert = require('assert'),
@@ -137,6 +137,26 @@ async function setExternalLdapConfig(req, res, next) {
     next(new HttpSuccess(200, {}));
 }
 
+async function getUserDirectoryConfig(req, res, next) {
+    const [error, config] = await safe(settings.getUserDirectoryConfig());
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, config));
+}
+
+async function setUserDirectoryConfig(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.enabled !== 'boolean') return next(new HttpError(400, 'enabled must be a boolean'));
+    if (typeof req.body.secret !== 'string') return next(new HttpError(400, 'secret must be a string'));
+    if ('allowlist' in req.body && typeof req.body.allowlist !== 'string') return next(new HttpError(400, 'allowlist must be a string'));
+
+    const [error] = await safe(settings.setUserDirectoryConfig(req.body));
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, {}));
+}
+
 async function getDynamicDnsConfig(req, res, next) {
     const [error, enabled] = await safe(settings.getDynamicDnsConfig());
     if (error) return next(BoxError.toHttpError(error));
@@ -155,6 +175,24 @@ async function setDynamicDnsConfig(req, res, next) {
     next(new HttpSuccess(200, {}));
 }
 
+async function getIPv6Config(req, res, next) {
+    const [error, ipv6Config] = await safe(settings.getIPv6Config());
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, ipv6Config));
+}
+
+async function setIPv6Config(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.provider || typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider is required'));
+
+    const [error] = await safe(settings.setIPv6Config(req.body));
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, {}));
+}
+
 async function getUnstableAppsConfig(req, res, next) {
     const [error, enabled] = await safe(settings.getUnstableAppsConfig());
     if (error) return next(BoxError.toHttpError(error));
@@ -197,20 +235,20 @@ async function setRegistryConfig(req, res, next) {
     next(new HttpSuccess(200));
 }
 
-async function getDirectoryConfig(req, res, next) {
-    const [error, directoryConfig] = await safe(settings.getDirectoryConfig());
+async function getProfileConfig(req, res, next) {
+    const [error, directoryConfig] = await safe(settings.getProfileConfig());
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(200, directoryConfig));
 }
 
-async function setDirectoryConfig(req, res, next) {
+async function setProfileConfig(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
     if (typeof req.body.lockUserProfiles !== 'boolean') return next(new HttpError(400, 'lockUserProfiles is required'));
     if (typeof req.body.mandatory2FA !== 'boolean') return next(new HttpError(400, 'mandatory2FA is required'));
 
-    const [error] = await safe(settings.setDirectoryConfig(req.body));
+    const [error] = await safe(settings.setProfileConfig(req.body));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(200, {}));
@@ -257,8 +295,10 @@ function get(req, res, next) {
 
     switch (req.params.setting) {
     case settings.DYNAMIC_DNS_KEY: return getDynamicDnsConfig(req, res, next);
+    case settings.IPV6_CONFIG_KEY: return getIPv6Config(req, res, next);
     case settings.BACKUP_CONFIG_KEY: return getBackupConfig(req, res, next);
     case settings.EXTERNAL_LDAP_KEY: return getExternalLdapConfig(req, res, next);
+    case settings.USER_DIRECTORY_KEY: return getUserDirectoryConfig(req, res, next);
     case settings.UNSTABLE_APPS_KEY: return getUnstableAppsConfig(req, res, next);
     case settings.REGISTRY_CONFIG_KEY: return getRegistryConfig(req, res, next);
     case settings.SYSINFO_CONFIG_KEY: return getSysinfoConfig(req, res, next);
@@ -267,7 +307,7 @@ function get(req, res, next) {
     case settings.AUTOUPDATE_PATTERN_KEY: return getAutoupdatePattern(req, res, next);
     case settings.TIME_ZONE_KEY: return getTimeZone(req, res, next);
 
-    case settings.DIRECTORY_CONFIG_KEY: return getDirectoryConfig(req, res, next);
+    case settings.PROFILE_CONFIG_KEY: return getProfileConfig(req, res, next);
     case settings.SUPPORT_CONFIG_KEY: return getSupportConfig(req, res, next);
 
     default: return next(new HttpError(404, 'No such setting'));
@@ -279,7 +319,9 @@ function set(req, res, next) {
 
     switch (req.params.setting) {
     case settings.DYNAMIC_DNS_KEY: return setDynamicDnsConfig(req, res, next);
+    case settings.IPV6_CONFIG_KEY: return setIPv6Config(req, res, next);
     case settings.EXTERNAL_LDAP_KEY: return setExternalLdapConfig(req, res, next);
+    case settings.USER_DIRECTORY_KEY: return setUserDirectoryConfig(req, res, next);
     case settings.UNSTABLE_APPS_KEY: return setUnstableAppsConfig(req, res, next);
     case settings.REGISTRY_CONFIG_KEY: return setRegistryConfig(req, res, next);
     case settings.SYSINFO_CONFIG_KEY: return setSysinfoConfig(req, res, next);
@@ -288,7 +330,7 @@ function set(req, res, next) {
     case settings.AUTOUPDATE_PATTERN_KEY: return setAutoupdatePattern(req, res, next);
     case settings.TIME_ZONE_KEY: return setTimeZone(req, res, next);
 
-    case settings.DIRECTORY_CONFIG_KEY: return setDirectoryConfig(req, res, next);
+    case settings.PROFILE_CONFIG_KEY: return setProfileConfig(req, res, next);
 
     default: return next(new HttpError(404, 'No such setting'));
     }

src/routes/support.js

@@ -62,7 +62,9 @@ async function canEnableRemoteSupport(req, res, next) {
     const SSHD_CONFIG_FILE = '/etc/ssh/sshd_config';
     const sshdConfig = safe.fs.readFileSync(SSHD_CONFIG_FILE, 'utf8');
     if (!sshdConfig) return next(new HttpError(412, `Failed to read file ${SSHD_CONFIG_FILE}`));
-    if (!sshdConfig.split('\n').find(function (line) { return line.search(/^PermitRootLogin.*yes/) !== -1; })) return next(new HttpError(417, `Set "PermitRootLogin yes" in ${SSHD_CONFIG_FILE}`));
+
+    // only check for PermitRootLogin if we want to enable remote support
+    if (req.body.enable && !sshdConfig.split('\n').find(function (line) { return line.search(/^PermitRootLogin.*yes/) !== -1; })) return next(new HttpError(417, `Set "PermitRootLogin yes" in ${SSHD_CONFIG_FILE}`));
 
     next();
 }

src/routes/test/apps-test.js

@@ -49,8 +49,8 @@ const DOMAIN_0 = {
 };
 
 var APP_STORE_ID = 'test', APP_ID;
-var APP_LOCATION = 'appslocation';
-var APP_LOCATION_NEW = 'appslocationnew';
+var APP_SUBDOMAIN = 'appssubdomain';
+var APP_SUBDOMAIN_NEW = 'appssubdomainnew';
 
 var APP_MANIFEST = JSON.parse(fs.readFileSync(__dirname + '/../../../../test-app/CloudronManifest.json', 'utf8'));
 APP_MANIFEST.dockerImage = TEST_IMAGE;
@@ -143,7 +143,7 @@ function startBox(done) {
 
         function (callback) {
             superagent.post(SERVER_URL + '/api/v1/cloudron/setup')
-                .send({ dnsConfig: DOMAIN_0 })
+                .send({ domainConfig: DOMAIN_0 })
                 .end(function (error, result) {
                     expect(result).to.be.ok();
                     expect(result.statusCode).to.eql(200);
@@ -226,7 +226,7 @@ function stopBox(done) {
     ], done);
 }
 
-describe('App API', function () {
+xdescribe('App API', function () {
     let taskId = '';
 
     before(startBox);
@@ -288,7 +288,7 @@ describe('App API', function () {
         it('app install fails - missing domain', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ manifest: APP_MANIFEST, location: 'some', accessRestriction: null })
+                .send({ manifest: APP_MANIFEST, subdomain: 'some', accessRestriction: null })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     expect(res.body.message).to.eql('domain is required');
@@ -299,7 +299,7 @@ describe('App API', function () {
         it('app install fails - non-existing domain', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ manifest: APP_MANIFEST, location: 'some', accessRestriction: null, domain: 'doesnotexist.com' })
+                .send({ manifest: APP_MANIFEST, subdomain: 'some', accessRestriction: null, domain: 'doesnotexist.com' })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     expect(res.body.message).to.eql('No such domain');
@@ -307,21 +307,21 @@ describe('App API', function () {
                 });
         });
 
-        it('app install fails - invalid location type', function (done) {
+        it('app install fails - invalid subdomain type', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ manifest: APP_MANIFEST, location: 42, accessRestriction: null, domain: DOMAIN_0.domain })
+                .send({ manifest: APP_MANIFEST, subdomain: 42, accessRestriction: null, domain: DOMAIN_0.domain })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
-                    expect(res.body.message).to.eql('location is required');
+                    expect(res.body.message).to.eql('subdomain is required');
                     done();
                 });
         });
 
-        it('app install fails - reserved admin location', function (done) {
+        it('app install fails - reserved admin subdomain', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ manifest: APP_MANIFEST, location: 'my', accessRestriction: null, domain: DOMAIN_0.domain })
+                .send({ manifest: APP_MANIFEST, subdomain: 'my', accessRestriction: null, domain: DOMAIN_0.domain })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     expect(res.body.message).to.contain('my is reserved');
@@ -329,10 +329,10 @@ describe('App API', function () {
                 });
         });
 
-        it('app install fails - reserved smtp location', function (done) {
+        it('app install fails - reserved smtp subdomain', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ manifest: APP_MANIFEST, location: constants.SMTP_LOCATION, accessRestriction: null, domain: DOMAIN_0.domain })
+                .send({ manifest: APP_MANIFEST, subdomain: constants.SMTP_LOCATION, accessRestriction: null, domain: DOMAIN_0.domain })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     expect(res.body.message).to.contain(constants.SMTP_LOCATION + ' is reserved');
@@ -343,7 +343,7 @@ describe('App API', function () {
         it('app install fails - portBindings must be object', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ manifest: APP_MANIFEST, location: APP_LOCATION, portBindings: 23, accessRestriction: null, domain: DOMAIN_0.domain })
+                .send({ manifest: APP_MANIFEST, subdomain: APP_SUBDOMAIN, portBindings: 23, accessRestriction: null, domain: DOMAIN_0.domain })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     expect(res.body.message).to.contain('portBindings must be an object');
@@ -354,7 +354,7 @@ describe('App API', function () {
         it('app install fails - accessRestriction is required', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ manifest: APP_MANIFEST, location: APP_LOCATION, portBindings: {}, domain: DOMAIN_0.domain })
+                .send({ manifest: APP_MANIFEST, subdomain: APP_SUBDOMAIN, portBindings: {}, domain: DOMAIN_0.domain })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     expect(res.body.message).to.contain('accessRestriction is required');
@@ -365,7 +365,7 @@ describe('App API', function () {
         it('app install fails - accessRestriction type is wrong', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ manifest: APP_MANIFEST, location: APP_LOCATION, portBindings: {}, accessRestriction: '', domain: DOMAIN_0.domain })
+                .send({ manifest: APP_MANIFEST, subdomain: APP_SUBDOMAIN, portBindings: {}, accessRestriction: '', domain: DOMAIN_0.domain })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     expect(res.body.message).to.contain('accessRestriction is required');
@@ -376,7 +376,7 @@ describe('App API', function () {
         it('app install fails for non admin', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token_1 })
-                .send({ manifest: APP_MANIFEST, location: APP_LOCATION, portBindings: null, accessRestriction: null, domain: DOMAIN_0.domain })
+                .send({ manifest: APP_MANIFEST, subdomain: APP_SUBDOMAIN, portBindings: null, accessRestriction: null, domain: DOMAIN_0.domain })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(403);
                     done();
@@ -388,7 +388,7 @@ describe('App API', function () {
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ appStoreId: APP_STORE_ID, location: APP_LOCATION, portBindings: null, domain: DOMAIN_0.domain, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                .send({ appStoreId: APP_STORE_ID, subdomain: APP_SUBDOMAIN, portBindings: null, domain: DOMAIN_0.domain, accessRestriction: { users: [ 'someuser' ], groups: [] } })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(404);
                     expect(fake.isDone()).to.be.ok();
@@ -401,7 +401,7 @@ describe('App API', function () {
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ appStoreId: APP_STORE_ID, location: APP_LOCATION, domain: DOMAIN_0.domain, portBindings: null, accessRestriction: null })
+                .send({ appStoreId: APP_STORE_ID, subdomain: APP_SUBDOMAIN, domain: DOMAIN_0.domain, portBindings: null, accessRestriction: null })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(402);
                     expect(fake1.isDone()).to.be.ok();
@@ -417,7 +417,7 @@ describe('App API', function () {
 
             const res = await superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ appStoreId: APP_STORE_ID, location: APP_LOCATION, domain: DOMAIN_0.domain, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                .send({ appStoreId: APP_STORE_ID, subdomain: APP_SUBDOMAIN, domain: DOMAIN_0.domain, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: { users: [ 'someuser' ], groups: [] } })
 
             expect(res.statusCode).to.equal(202);
             expect(res.body.id).to.be.a('string');
@@ -427,10 +427,10 @@ describe('App API', function () {
             taskId = res.body.taskId;
         });
 
-        it('app install fails because of conflicting location', function (done) {
+        it('app install fails because of conflicting subdomain', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
-                .send({ manifest: APP_MANIFEST, location: APP_LOCATION, domain: DOMAIN_0.domain, portBindings: null, accessRestriction: null })
+                .send({ manifest: APP_MANIFEST, subdomain: APP_SUBDOMAIN, domain: DOMAIN_0.domain, portBindings: null, accessRestriction: null })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(409);
                     done();
@@ -446,7 +446,7 @@ describe('App API', function () {
                     expect(res.statusCode).to.equal(200);
                     expect(res.body.id).to.eql(APP_ID);
                     expect(res.body.installationState).to.be.ok();
-                    expect(res.body.mailboxName).to.be(APP_LOCATION + '.app');
+                    expect(res.body.mailboxName).to.be(APP_SUBDOMAIN + '.app');
                     expect(res.body.mailboxDomain).to.be(DOMAIN_0.domain);
                     done();
                 });
@@ -522,10 +522,10 @@ describe('App API', function () {
                 expect(data.Config.Env).to.contain('CLOUDRON_WEBADMIN_ORIGIN=' + settings.dashboardOrigin());
                 expect(data.Config.Env).to.contain('CLOUDRON_API_ORIGIN=' + settings.dashboardOrigin());
                 expect(data.Config.Env).to.contain('CLOUDRON=1');
-                expect(data.Config.Env).to.contain('CLOUDRON_APP_ORIGIN=https://' + APP_LOCATION + '.' + DOMAIN_0.domain);
-                expect(data.Config.Env).to.contain('CLOUDRON_APP_DOMAIN=' + APP_LOCATION + '.' + DOMAIN_0.domain);
-                // Hostname must not be set of app fqdn or app location!
-                expect(data.Config.Hostname).to.not.contain(APP_LOCATION);
+                expect(data.Config.Env).to.contain('CLOUDRON_APP_ORIGIN=https://' + APP_SUBDOMAIN + '.' + DOMAIN_0.domain);
+                expect(data.Config.Env).to.contain('CLOUDRON_APP_DOMAIN=' + APP_SUBDOMAIN + '.' + DOMAIN_0.domain);
+                // Hostname must not be set of app fqdn or app subdomain!
+                expect(data.Config.Hostname).to.not.contain(APP_SUBDOMAIN);
                 expect(data.Config.Env).to.contain('ECHO_SERVER_PORT=7171');
                 expect(data.HostConfig.PortBindings['7778/tcp'][0].HostPort).to.eql('7171');
                 done();
@@ -533,7 +533,7 @@ describe('App API', function () {
         });
 
         it('nginx config', function (done) {
-            expect(fs.existsSync(paths.NGINX_APPCONFIG_DIR + '/' + APP_LOCATION + '.conf'));
+            expect(fs.existsSync(paths.NGINX_APPCONFIG_DIR + '/' + APP_SUBDOMAIN + '.conf'));
             done();
         });
 
@@ -857,7 +857,7 @@ describe('App API', function () {
         it('cannot set only the cert, no key', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/cert')
                 .query({ access_token: token })
-                .send({ location: APP_LOCATION, domain: DOMAIN_0.domain, cert: CERT })
+                .send({ subdomain: APP_SUBDOMAIN, domain: DOMAIN_0.domain, cert: CERT })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     done();
@@ -867,7 +867,7 @@ describe('App API', function () {
         it('cannot reconfigure app with only the key, no cert', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/cert')
                 .query({ access_token: token })
-                .send({ location: APP_LOCATION, domain: DOMAIN_0.domain, key: KEY })
+                .send({ subdomain: APP_SUBDOMAIN, domain: DOMAIN_0.domain, key: KEY })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     done();
@@ -877,7 +877,7 @@ describe('App API', function () {
         it('cannot set invalid cert', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/cert')
                 .query({ access_token: token })
-                .send({ location: APP_LOCATION, domain: DOMAIN_0.domain, cert: 'x' + CERT, key: KEY })
+                .send({ subdomain: APP_SUBDOMAIN, domain: DOMAIN_0.domain, cert: 'x' + CERT, key: KEY })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     done();
@@ -887,7 +887,7 @@ describe('App API', function () {
         it('can set cert', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/cert')
                 .query({ access_token: token })
-                .send({ location: APP_LOCATION, domain: DOMAIN_0.domain, cert: CERT, key: KEY })
+                .send({ subdomain: APP_SUBDOMAIN, domain: DOMAIN_0.domain, cert: CERT, key: KEY })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(200);
                     done();
@@ -897,7 +897,7 @@ describe('App API', function () {
         it('can reset cert', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/cert')
                 .query({ access_token: token })
-                .send({ location: APP_LOCATION, domain: DOMAIN_0.domain, cert: null, key: null })
+                .send({ subdomain: APP_SUBDOMAIN, domain: DOMAIN_0.domain, cert: null, key: null })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(200);
                     done();
@@ -1015,21 +1015,21 @@ describe('App API', function () {
         });
     });
 
-    describe('configure location', function () {
+    describe('configure subdomain', function () {
         it('cannot reconfigure app with missing domain', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/location')
                 .query({ access_token: token })
-                .send({ location: 'hellothre' })
+                .send({ subdomain: 'hellothre' })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     done();
                 });
         });
 
-        it('cannot reconfigure app with bad location', function (done) {
+        it('cannot reconfigure app with bad subdomain', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/location')
                 .query({ access_token: token })
-                .send({ location: 1234, domain: DOMAIN_0.domain })
+                .send({ subdomain: 1234, domain: DOMAIN_0.domain })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     done();
@@ -1039,17 +1039,17 @@ describe('App API', function () {
         it('non admin cannot reconfigure app', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/location')
                 .query({ access_token: token_1 })
-                .send({ location: APP_LOCATION_NEW, domain: DOMAIN_0.domain, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                .send({ subdomain: APP_SUBDOMAIN_NEW, domain: DOMAIN_0.domain, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(403);
                     done();
                 });
         });
 
-        xit('can change location', function (done) {
+        xit('can change subdomain', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/location')
                 .query({ access_token: token })
-                .send({ location: APP_LOCATION_NEW, domain: DOMAIN_0.domain, portBindings: { ECHO_SERVER_PORT: 7172 } })
+                .send({ subdomain: APP_SUBDOMAIN_NEW, domain: DOMAIN_0.domain, portBindings: { ECHO_SERVER_PORT: 7172 } })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(202);
                     taskId = res.body.taskId;
@@ -1061,18 +1061,18 @@ describe('App API', function () {
             waitForTask(taskId, done);
         });
 
-        xit('did change location', function (done) {
+        xit('did change subdomain', function (done) {
             apps.get(APP_ID, function (error, app) {
                 if (error) return done(error);
 
-                expect(app.mailboxName).to.be(APP_LOCATION_NEW + '.app'); // must follow location change
+                expect(app.mailboxName).to.be(APP_SUBDOMAIN_NEW + '.app'); // must follow subdomain change
                 expect(app.mailboxDomain).to.be(DOMAIN_0.domain);
 
                 docker.getContainer(app.containerId).inspect(function (error, data) {
                     expect(error).to.not.be.ok();
-                    expect(data.Config.Env).to.contain('CLOUDRON_APP_ORIGIN=https://' + APP_LOCATION_NEW + '.' + DOMAIN_0.domain);
-                    expect(data.Config.Env).to.contain('CLOUDRON_APP_DOMAIN=' + APP_LOCATION_NEW + '.' + DOMAIN_0.domain);
-                    expect(data.Config.Hostname).to.not.contain(APP_LOCATION_NEW);
+                    expect(data.Config.Env).to.contain('CLOUDRON_APP_ORIGIN=https://' + APP_SUBDOMAIN_NEW + '.' + DOMAIN_0.domain);
+                    expect(data.Config.Env).to.contain('CLOUDRON_APP_DOMAIN=' + APP_SUBDOMAIN_NEW + '.' + DOMAIN_0.domain);
+                    expect(data.Config.Hostname).to.not.contain(APP_SUBDOMAIN_NEW);
                     expect(data.Config.Env).to.contain('ECHO_SERVER_PORT=7172');
                     expect(data.HostConfig.PortBindings['7778/tcp'][0].HostPort).to.eql('7172');
                     done();
@@ -1314,7 +1314,7 @@ describe('App API', function () {
 
                 docker.getContainer(app.containerId).inspect(function (error, data) {
                     expect(error).to.not.be.ok();
-                    expect(data.Config.Env).to.not.contain('CLOUDRON_MAIL_FROM=' + app.location + '@' + app.domain + '.app');
+                    expect(data.Config.Env).to.not.contain('CLOUDRON_MAIL_FROM=' + app.subdomain + '@' + app.domain + '.app');
                     done();
                 });
             });
@@ -1553,7 +1553,7 @@ describe('App API', function () {
         });
 
         it('removed nginx', function (done) {
-            expect(!fs.existsSync(paths.NGINX_APPCONFIG_DIR + '/' + APP_LOCATION + '.conf'));
+            expect(!fs.existsSync(paths.NGINX_APPCONFIG_DIR + '/' + APP_SUBDOMAIN + '.conf'));
             done();
         });
 

src/routes/test/branding-test.js

@@ -51,7 +51,7 @@ describe('Branding API', function () {
                 .query({ access_token: owner.token })
                 .send({ name: name });
 
-            expect(response.statusCode).to.equal(202);
+            expect(response.statusCode).to.equal(200);
         });
 
         it('get succeeds', async function () {
@@ -85,7 +85,7 @@ describe('Branding API', function () {
                 .query({ access_token: owner.token })
                 .attach('avatar', paths.CLOUDRON_DEFAULT_AVATAR_FILE);
 
-            expect(response.statusCode).to.equal(202);
+            expect(response.statusCode).to.equal(200);
         });
 
         it('get succeeds', async function () {
@@ -138,7 +138,7 @@ describe('Branding API', function () {
                 .query({ access_token: owner.token })
                 .send({ blacklist: [ 'id1', 'id2' ] });
 
-            expect(response.statusCode).to.equal(202);
+            expect(response.statusCode).to.equal(200);
         });
 
         it('get bl succeeds', async function () {
@@ -155,7 +155,7 @@ describe('Branding API', function () {
                 .query({ access_token: owner.token })
                 .send({ whitelist: [ 'id1', 'id2' ] });
 
-            expect(response.statusCode).to.equal(202);
+            expect(response.statusCode).to.equal(200);
         });
 
         it('get wl succeeds', async function () {

src/routes/test/cloudron-test.js

@@ -52,6 +52,151 @@ describe('Cloudron API', function () {
         });
     });
 
+    describe('account setup', function () {
+        it('succeeds without pre-set username and display name', async function () {
+            const USER = {
+                email: 'setup1@account.com',
+                password: 'test?!3434543534',
+                username: 'setupuser1',
+                displayName: 'setup user1',
+            };
+
+            const response = await superagent.post(`${serverUrl}/api/v1/users`)
+                .query({ access_token: owner.token })
+                .send({ email: USER.email });
+            expect(response.statusCode).to.equal(201);
+            USER.id = response.body.id;
+
+            const response2 = await superagent.get(`${serverUrl}/api/v1/users/${USER.id}/invite_link`)
+                .query({ access_token: owner.token })
+                .ok(() => true);
+            expect(response2.statusCode).to.equal(200);
+
+            const response3 = await superagent.post(`${serverUrl}/api/v1/cloudron/setup_account`)
+                .send({
+                    inviteToken: require('url').parse(response2.body.inviteLink, true).query.inviteToken,
+                    password: USER.password,
+                    username: USER.username,
+                    displayName: USER.displayName
+                })
+                .ok(() => true);
+            expect(response3.statusCode).to.equal(201);
+            expect(response3.body.accessToken).to.be.a('string');
+
+            const response4 = await superagent.get(`${serverUrl}/api/v1/users/${USER.id}`)
+                .query({ access_token: owner.token })
+                .ok(() => true);
+
+            expect(response4.statusCode).to.equal(200);
+            expect(response4.body.username).to.equal(USER.username);
+            expect(response4.body.displayName).to.equal(USER.displayName);
+
+            const response5 = await superagent.post(`${serverUrl}/api/v1/cloudron/login`)
+                .send({ username: USER.username, password: USER.password });
+            expect(response5.statusCode).to.equal(200);
+        });
+
+        it('succeeds and overwrites with pre-set username and display name', async function () {
+            const USER = {
+                email: 'setup2@account.com',
+                password: 'test?!3434543534',
+                username: 'presetup2',
+                displayName: 'setup user2',
+            };
+
+            const response = await superagent.post(`${serverUrl}/api/v1/users`)
+                .query({ access_token: owner.token })
+                .send({ email: USER.email, username: 'presetup2', displayName: 'pre setup' });
+            expect(response.statusCode).to.equal(201);
+            USER.id = response.body.id;
+
+            const response2 = await superagent.get(`${serverUrl}/api/v1/users/${USER.id}/invite_link`)
+                .query({ access_token: owner.token })
+                .ok(() => true);
+            expect(response2.statusCode).to.equal(200);
+
+            const response3 = await superagent.post(`${serverUrl}/api/v1/cloudron/setup_account`)
+                .send({
+                    inviteToken: require('url').parse(response2.body.inviteLink, true).query.inviteToken,
+                    password: USER.password,
+                    username: 'setupuser2', // this will cause a conflict. cannot change username
+                    displayName: USER.displayName
+                })
+                .ok(() => true);
+            expect(response3.statusCode).to.equal(409);
+
+            const response4 = await superagent.post(`${serverUrl}/api/v1/cloudron/setup_account`)
+                .send({
+                    inviteToken: require('url').parse(response2.body.inviteLink, true).query.inviteToken,
+                    password: USER.password,
+                    displayName: USER.displayName
+                })
+                .ok(() => true);
+            expect(response4.statusCode).to.equal(201);
+            expect(response4.body.accessToken).to.be.a('string');
+
+            const response5 = await superagent.get(`${serverUrl}/api/v1/users/${USER.id}`)
+                .query({ access_token: owner.token })
+                .ok(() => true);
+
+            expect(response5.statusCode).to.equal(200);
+            expect(response5.body.username).to.equal(USER.username);
+            expect(response5.body.displayName).to.equal(USER.displayName);
+
+            const response6 = await superagent.post(`${serverUrl}/api/v1/cloudron/login`)
+                .send({ username: USER.username, password: USER.password });
+            expect(response6.statusCode).to.equal(200);
+        });
+
+        it('succeeds and does not overwrite pre-set username and display name if profiles are locked', async function () {
+            const USER = {
+                email: 'setup3@account.com',
+                password: 'test?!3434543534',
+                username: 'setupuser3',
+                displayName: 'setup user3',
+            };
+
+            const response0 = await superagent.post(`${serverUrl}/api/v1/settings/profile_config`)
+                .query({ access_token: owner.token })
+                .send({ lockUserProfiles: true, mandatory2FA: false });
+            expect(response0.statusCode).to.equal(200);
+
+            const response = await superagent.post(`${serverUrl}/api/v1/users`)
+                .query({ access_token: owner.token })
+                .send({ email: USER.email, username: 'presetup3', displayName: 'pre setup3' });
+            expect(response.statusCode).to.equal(201);
+            USER.id = response.body.id;
+
+            const response2 = await superagent.get(`${serverUrl}/api/v1/users/${USER.id}/invite_link`)
+                .query({ access_token: owner.token })
+                .ok(() => true);
+            expect(response2.statusCode).to.equal(200);
+
+            const response3 = await superagent.post(`${serverUrl}/api/v1/cloudron/setup_account`)
+                .send({
+                    inviteToken: require('url').parse(response2.body.inviteLink, true).query.inviteToken,
+                    password: USER.password,
+                    username: USER.username, // ignored
+                    displayName: USER.displayName // ignored
+                })
+                .ok(() => true);
+            expect(response3.statusCode).to.equal(201);
+            expect(response3.body.accessToken).to.be.a('string');
+
+            const response4 = await superagent.get(`${serverUrl}/api/v1/users/${USER.id}`)
+                .query({ access_token: owner.token })
+                .ok(() => true);
+
+            expect(response4.statusCode).to.equal(200);
+            expect(response4.body.username).to.equal('presetup3'); // what the admin provided
+            expect(response4.body.displayName).to.equal('pre setup3'); // what the admin provided
+
+            const response5 = await superagent.post(`${serverUrl}/api/v1/cloudron/login`)
+                .send({ username: 'presetup3', password: USER.password });
+            expect(response5.statusCode).to.equal(200);
+        });
+    });
+
     describe('login', function () {
         it('cannot login without body', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/login`)

src/routes/test/common.js

@@ -59,7 +59,7 @@ async function setup() {
 
     // setup
     let response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-        .send({ dnsConfig: { provider: 'noop', domain: exports.dashboardDomain, config: {}, tlsConfig: { provider: 'fallback' } } });
+        .send({ domainConfig: { provider: 'noop', domain: exports.dashboardDomain, config: {}, tlsConfig: { provider: 'fallback' } } });
     expect(response.status).to.eql(200);
 
     await delay(2000);

src/routes/test/domains-test.js

@@ -152,6 +152,40 @@ describe('Domains API', function () {
         });
     });
 
+    describe('update', function () {
+        it('config fails for non-existing domain', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/whatever/update`)
+                .query({ access_token: owner.token })
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(404);
+        });
+
+        it('config succeeds', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}/config`)
+                .query({ access_token: owner.token })
+                .send(DOMAIN_0);
+
+            expect(response.statusCode).to.equal(204);
+        });
+
+        it('wellknown succeeds', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}/wellknown`)
+                .query({ access_token: owner.token })
+                .send({ wellKnown: null });
+
+            expect(response.statusCode).to.equal(204);
+        });
+
+        it('wellknown succeeds', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}/wellknown`)
+                .query({ access_token: owner.token })
+                .send({ wellKnown: { service: 'some.service' } });
+
+            expect(response.statusCode).to.equal(204);
+        });
+    });
+
     describe('Certificates API', function () {
         let validCert0, validKey0, // example.com
             validCert1, validKey1; // *.example.com
@@ -170,7 +204,7 @@ describe('Domains API', function () {
             let d = Object.assign({}, DOMAIN_0);
             d.fallbackCertificate = { key: validKey1 };
 
-            const response = await superagent.put(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}`)
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}/config`)
                 .query({ access_token: owner.token })
                 .send(d)
                 .ok(() => true);
@@ -182,7 +216,7 @@ describe('Domains API', function () {
             let d = Object.assign({}, DOMAIN_0);
             d.fallbackCertificate = { cert: validCert1 };
 
-            const response = await superagent.put(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}`)
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}/config`)
                 .query({ access_token: owner.token })
                 .send(d)
                 .ok(() => true);
@@ -194,7 +228,7 @@ describe('Domains API', function () {
             let d = Object.assign({}, DOMAIN_0);
             d.fallbackCertificate = { cert: 1234, key: validKey1 };
 
-            const response = await superagent.put(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}`)
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}/config`)
                 .query({ access_token: owner.token })
                 .send(d)
                 .ok(() => true);
@@ -206,7 +240,7 @@ describe('Domains API', function () {
             let d = Object.assign({}, DOMAIN_0);
             d.fallbackCertificate = { cert: validCert1, key: true };
 
-            const response = await superagent.put(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}`)
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}/config`)
                 .query({ access_token: owner.token })
                 .send(d)
                 .ok(() => true);
@@ -218,7 +252,7 @@ describe('Domains API', function () {
             let d = Object.assign({}, DOMAIN_0);
             d.fallbackCertificate = { cert: validCert0, key: validKey0 };
 
-            const response = await superagent.put(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}`)
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}/config`)
                 .query({ access_token: owner.token })
                 .send(d)
                 .ok(() => true);
@@ -230,7 +264,7 @@ describe('Domains API', function () {
             let d = Object.assign({}, DOMAIN_0);
             d.fallbackCertificate = { cert: validCert1, key: validKey1 };
 
-            const response = await superagent.put(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}`)
+            const response = await superagent.post(`${serverUrl}/api/v1/domains/${DOMAIN_0.domain}/config`)
                 .query({ access_token: owner.token })
                 .send(d);
 

src/routes/test/mail-test.js

@@ -54,11 +54,11 @@ describe('Mail API', function () {
         let dkimDomain, spfDomain, mxDomain, dmarcDomain;
 
         before(function (done) {
-            const dns = require('../../dns.js');
+            const dig = require('../../dig.js');
 
             // replace dns resolveTxt()
-            resolve = dns.promises.resolve;
-            dns.promises.resolve = async function (hostname, type/*, options*/) {
+            resolve = dig.resolve;
+            dig.resolve = async function (hostname, type/*, options*/) {
                 expect(hostname).to.be.a('string');
 
                 if (!dnsAnswerQueue[hostname] || !(type in dnsAnswerQueue[hostname])) throw new Error('no mock answer');
@@ -84,9 +84,9 @@ describe('Mail API', function () {
         });
 
         after(function (done) {
-            const dns = require('../../dns.js');
+            const dig = require('../../dig.js');
 
-            dns.promises.resolve = resolve;
+            dig.resolve = resolve;
 
             done();
         });

src/routes/test/provision-test.js

@@ -36,7 +36,7 @@ describe('Provision', function () {
     describe('DNS Setup', async function () {
         it('fails without provider', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { domain: DOMAIN, config: {} } })
+                .send({ domainConfig: { domain: DOMAIN, config: {} } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(400);
@@ -44,7 +44,7 @@ describe('Provision', function () {
 
         it('fails with invalid provider', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'foobar', domain: DOMAIN, config: {} } })
+                .send({ domainConfig: { provider: 'foobar', domain: DOMAIN, config: {} } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(400);
@@ -52,7 +52,7 @@ describe('Provision', function () {
 
         it('fails with missing domain', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'noop', config: {} } })
+                .send({ domainConfig: { provider: 'noop', config: {} } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(400);
@@ -60,7 +60,7 @@ describe('Provision', function () {
 
         it('fails with invalid domain', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'noop', domain: '.foo', config: {} } })
+                .send({ domainConfig: { provider: 'noop', domain: '.foo', config: {} } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(400);
@@ -68,7 +68,7 @@ describe('Provision', function () {
 
         it('fails with invalid config', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, config: 'not an object' } })
+                .send({ domainConfig: { provider: 'noop', domain: DOMAIN, config: 'not an object' } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(400);
@@ -76,7 +76,7 @@ describe('Provision', function () {
 
         it('fails with invalid zoneName', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, config: {}, zoneName: 1337 } })
+                .send({ domainConfig: { provider: 'noop', domain: DOMAIN, config: {}, zoneName: 1337 } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(400);
@@ -84,7 +84,7 @@ describe('Provision', function () {
 
         it('fails with invalid tlsConfig', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, config: {}, tlsConfig: 'foobar' } })
+                .send({ domainConfig: { provider: 'noop', domain: DOMAIN, config: {}, tlsConfig: 'foobar' } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(400);
@@ -92,7 +92,7 @@ describe('Provision', function () {
 
         it('fails with invalid tlsConfig provider', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, config: {}, tlsConfig: { provider: 1337 } } })
+                .send({ domainConfig: { provider: 'noop', domain: DOMAIN, config: {}, tlsConfig: { provider: 1337 } } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(400);
@@ -100,7 +100,7 @@ describe('Provision', function () {
 
         it('succeeds', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, adminFqdn: 'my.' + DOMAIN, config: {}, tlsConfig: { provider: 'fallback' } } })
+                .send({ domainConfig: { provider: 'noop', domain: DOMAIN, adminFqdn: 'my.' + DOMAIN, config: {}, tlsConfig: { provider: 'fallback' } } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(200);
@@ -110,7 +110,7 @@ describe('Provision', function () {
 
         it('twice succeeds', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, adminFqdn: 'my.' + DOMAIN, config: {}, tlsConfig: { provider: 'fallback' } } })
+                .send({ domainConfig: { provider: 'noop', domain: DOMAIN, adminFqdn: 'my.' + DOMAIN, config: {}, tlsConfig: { provider: 'fallback' } } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(200);
@@ -224,7 +224,7 @@ describe('Provision', function () {
 
         it('after fails', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/cloudron/setup`)
-                .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, adminFqdn: 'my.' + DOMAIN, config: {}, tlsConfig: { provider: 'fallback' } } })
+                .send({ domainConfig: { provider: 'noop', domain: DOMAIN, adminFqdn: 'my.' + DOMAIN, config: {}, tlsConfig: { provider: 'fallback' } } })
                 .ok(() => true);
 
             expect(response.statusCode).to.eql(409);

src/routes/test/settings-test.js

@@ -79,6 +79,96 @@ describe('Settings API', function () {
         });
     });
 
+    describe('user_directory_config', function () {
+        // keep in sync with defaults in settings.js
+        let defaultConfig = {
+            enabled: false,
+            secret: '',
+            allowlist: ''
+        };
+
+        it('can get user_directory_config (default)', async function () {
+            const response = await superagent.get(`${serverUrl}/api/v1/settings/user_directory_config`)
+                .query({ access_token: owner.token });
+
+            expect(response.statusCode).to.equal(200);
+            expect(response.body).to.eql(defaultConfig);
+        });
+
+        it('cannot set user_directory_config without enabled boolean', async function () {
+            let tmp = JSON.parse(JSON.stringify(defaultConfig));
+            delete tmp.enabled;
+
+            const response = await superagent.post(`${serverUrl}/api/v1/settings/user_directory_config`)
+                .query({ access_token: owner.token })
+                .send(tmp)
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(400);
+        });
+
+        it('cannot set user_directory_config without secret', async function () {
+            let tmp = JSON.parse(JSON.stringify(defaultConfig));
+            delete tmp.secret;
+
+            const response = await superagent.post(`${serverUrl}/api/v1/settings/user_directory_config`)
+                .query({ access_token: owner.token })
+                .send(tmp)
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(400);
+        });
+
+        it('cannot enable user_directory_config with empty secret', async function () {
+            let tmp = JSON.parse(JSON.stringify(defaultConfig));
+            tmp.enabled = true;
+
+            const response = await superagent.post(`${serverUrl}/api/v1/settings/user_directory_config`)
+                .query({ access_token: owner.token })
+                .send(tmp)
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(400);
+        });
+
+        it('cannot enable user_directory_config with empty allowlist', async function () {
+            let tmp = JSON.parse(JSON.stringify(defaultConfig));
+            tmp.enabled = true;
+            tmp.secret = 'ldapsecret';
+
+            const response = await superagent.post(`${serverUrl}/api/v1/settings/user_directory_config`)
+                .query({ access_token: owner.token })
+                .send(tmp)
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(400);
+        });
+
+        it('can enable user_directory_config', async function () {
+            let tmp = JSON.parse(JSON.stringify(defaultConfig));
+            tmp.enabled = true;
+            tmp.secret = 'ldapsecret';
+            tmp.allowlist = '1.2.3.4';
+
+            const response = await superagent.post(`${serverUrl}/api/v1/settings/user_directory_config`)
+                .query({ access_token: owner.token })
+                .send(tmp);
+
+            expect(response.statusCode).to.equal(200);
+        });
+
+        it('can get user_directory_config', async function () {
+            let tmp = JSON.parse(JSON.stringify(defaultConfig));
+            tmp.enabled = true;
+
+            const response = await superagent.get(`${serverUrl}/api/v1/settings/user_directory_config`)
+                .query({ access_token: owner.token });
+
+            expect(response.statusCode).to.equal(200);
+            expect(response.body).to.eql({ enabled: true, secret: 'ldapsecret', allowlist: '1.2.3.4' });
+        });
+    });
+
     describe('backup_config', function () {
         // keep in sync with defaults in settings.js
         let defaultConfig = {

src/routes/test/users-test.js

@@ -11,7 +11,7 @@ const common = require('./common.js'),
     users = require('../../users.js');
 
 describe('Users API', function () {
-    const { setup, cleanup, serverUrl, owner, user } = common;
+    const { setup, cleanup, serverUrl, owner, user, dashboardDomain } = common;
 
     const user2 = {
         id: null,
@@ -21,6 +21,13 @@ describe('Users API', function () {
         token: null
     };
 
+    const user3 = {
+        id: null,
+        username: 'transientuser',
+        password: 'Foobar?1334',
+        email: 'transient@cloudron.LoCal',
+    };
+
     const unnamedUser = {
         id: null,
         email: 'unnameduser@cloudron.local',
@@ -135,7 +142,7 @@ describe('Users API', function () {
         it('create user with short name succeeds', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/users`)
                 .query({ access_token: owner.token })
-                .send({ username: 'n', email: 'reserved@cloudron.local' });
+                .send({ username: 'n', email: 'singleletter@cloudron.local' });
 
             expect(response.statusCode).to.equal(201);
         });
@@ -173,8 +180,8 @@ describe('Users API', function () {
     });
 
     describe('invite', function () {
-        it(' unknown user fails', async function () {
-            const response = await superagent.post(`${serverUrl}/api/v1/users/randomuserid/create_invite`)
+        it('creationg fails for unknown user', async function () {
+            const response = await superagent.get(`${serverUrl}/api/v1/users/randomuserid/invite_link`)
                 .query({ access_token: owner.token })
                 .send({})
                 .ok(() => true);
@@ -182,6 +189,16 @@ describe('Users API', function () {
             expect(response.statusCode).to.equal(404);
         });
 
+        it('creation succeeds', async function () {
+            const response = await superagent.get(`${serverUrl}/api/v1/users/${user.id}/invite_link`)
+                .query({ access_token: owner.token })
+                .send({})
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(200);
+            expect(response.body.inviteLink).to.be.a('string');
+        });
+
         it('sending succeeds', async function () {
             common.clearMailQueue();
 
@@ -248,7 +265,7 @@ describe('Users API', function () {
     });
 
     describe('groups', function () {
-        it('does not list groupIds when listing users', async function () {
+        it('lists groupIds when listing users', async function () {
             const response = await superagent.get(`${serverUrl}/api/v1/users`)
                 .query({ access_token: owner.token });
 
@@ -256,7 +273,7 @@ describe('Users API', function () {
             expect(response.body.users).to.be.an('array');
 
             response.body.users.forEach(function (user) {
-                expect('groupIds' in user).to.be(false);
+                expect('groupIds' in user).to.be(true);
             });
         });
     });
@@ -282,10 +299,10 @@ describe('Users API', function () {
                 expect(user).to.be.an('object');
                 expect(user.id).to.be.ok();
                 expect(user.email).to.be.ok();
+                expect(user.role).to.be.ok();
                 if (!user.email.startsWith('unnamed')) expect(user.username).to.be.ok();
                 expect(user.password).to.not.be.ok();
                 expect(user.salt).to.not.be.ok();
-                expect(user.groupIds).to.not.be.ok();
             });
         });
     });
@@ -400,7 +417,7 @@ describe('Users API', function () {
     });
 
     describe('role - user manager', function () {
-        it('can make second user a usermanager', async function () {
+        it('can make normal user a usermanager', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/users/${user.id}`)
                 .query({ access_token: owner.token })
                 .send({ role: users.ROLE_USER_MANAGER });
@@ -460,12 +477,101 @@ describe('Users API', function () {
             expect(response.statusCode).to.equal(403);
         });
 
-        it('can remove normal user', async function () {
-            const response = await superagent.del(`${serverUrl}/api/v1/users/${unnamedUser.id}`)
+        it('can create user as user manager', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/users`)
+                .query({ access_token: user.token })
+                .send({ username: user3.username, email: user3.email });
+
+            expect(response.statusCode).to.equal(201);
+
+            user3.id = response.body.id;
+        });
+
+        it('can remove normal user as user manager', async function () {
+            const response = await superagent.del(`${serverUrl}/api/v1/users/${user3.id}`)
                 .query({ access_token: user.token });
 
             expect(response.statusCode).to.equal(204);
         });
+
+        it('add mailbox fails', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/mail/${dashboardDomain}/mailboxes`)
+                .send({ name: 'support', ownerId: owner.id, ownerType: 'user', active: true })
+                .query({ access_token: user.token })
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(403);
+        });
+    });
+
+    describe('role - mail manager', function () {
+        it('can make normal user a usermanager', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/users/${user.id}`)
+                .query({ access_token: owner.token })
+                .send({ role: users.ROLE_MAIL_MANAGER });
+
+            expect(response.statusCode).to.equal(204);
+        });
+
+        it('can list users as mail manager', async function () {
+            const response = await superagent.get(`${serverUrl}/api/v1/users`)
+                .query({ access_token: user.token });
+
+            expect(response.statusCode).to.equal(200);
+            expect(response.body.users).to.be.an(Array);
+            expect(response.body.users.length).to.be.greaterThan(3);
+        });
+
+        it('cannot change admin bit of self', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/users/${user.id}`)
+                .query({ access_token: user.token })
+                .send({ role: users.ROLE_ADMIN })
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(409);
+        });
+
+        it('cannot remove admin', async function () {
+            const response = await superagent.del(`${serverUrl}/api/v1/users/${owner.id}`)
+                .query({ access_token: user.token })
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(403);
+        });
+
+        it('can create user as mail manager', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/users`)
+                .query({ access_token: user.token })
+                .send({ username: user3.username, email: user3.email });
+
+            expect(response.statusCode).to.equal(201);
+
+            user3.id = response.body.id;
+        });
+
+        it('can remove normal user as mail manager', async function () {
+            const response = await superagent.del(`${serverUrl}/api/v1/users/${user3.id}`)
+                .query({ access_token: user.token });
+
+            expect(response.statusCode).to.equal(204);
+        });
+
+        it('add mailbox succeeds as mail manager', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/mail/${dashboardDomain}/mailboxes`)
+                .send({ name: 'support', ownerId: owner.id, ownerType: 'user', active: true })
+                .query({ access_token: user.token });
+
+            expect(response.statusCode).to.equal(201);
+        });
+
+        it('list mailbox succeeds as mail manager', async function () {
+            const response = await superagent.get(`${serverUrl}/api/v1/mail/${dashboardDomain}/mailboxes`)
+                .query({ access_token: user.token });
+
+            expect(response.statusCode).to.equal(200);
+            expect(response.body.mailboxes.length).to.be(1);
+            expect(response.body.mailboxes[0].name).to.be('support');
+        });
     });
 
     describe('remove', function () {

src/routes/users.js

@@ -73,6 +73,7 @@ async function update(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
     assert.strictEqual(typeof req.body, 'object');
 
+    if ('username' in req.body && typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be string')); // when profile are locked, admin can set username
     if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
     if ('fallbackEmail' in req.body && typeof req.body.fallbackEmail !== 'string') return next(new HttpError(400, 'fallbackEmail must be string'));
     if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
@@ -102,15 +103,17 @@ async function list(req, res, next) {
     const page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
     if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
 
-    const perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    const perPage = typeof req.query.per_page !== 'undefined' ? parseInt(req.query.per_page) : 25;
     if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
 
     if (req.query.search && typeof req.query.search !== 'string') return next(new HttpError(400, 'search must be a string'));
 
-    let [error, results] = await safe(users.listPaged(req.query.search || null, page, perPage));
+    const active = typeof req.query.active !== 'undefined' ? ((req.query.active === '1' || req.query.active === 'true') ? true : false) : null;
+
+    let [error, results] = await safe(users.listPaged(req.query.search || null, active, page, perPage));
     if (error) return next(BoxError.toHttpError(error));
 
-    results = results.map(users.removeRestrictedFields);
+    results = results.map(users.removePrivateFields);
 
     next(new HttpSuccess(200, { users: results }));
 }

src/scripts/restart.sh

@@ -2,8 +2,6 @@
 
 set -eu -o pipefail
 
-readonly INFRA_VERSION_FILE=/home/yellowtent/platformdata/INFRA_VERSION
-
 if [[ ${EUID} -ne 0 ]]; then
     echo "This script should be run as root." > /dev/stderr
     exit 1

src/scripts/setblocklist.sh

@@ -23,6 +23,11 @@ if [[ -f "${user_firewall_json}" ]]; then
     while read -r line || [[ -n "$line" ]]; do
         [[ -z "${line}" ]] && continue # ignore empty lines
         [[ "$line" =~ ^#.*$ ]] && continue # ignore lines starting with #
-        ipset add -! cloudron_blocklist "${line}" # the -! ignore duplicates
+
+        if [[ "$line" == *":"* ]]; then
+            ipset add -! cloudron_blocklist6 "${line}" # the -! ignores duplicates
+        else
+            ipset add -! cloudron_blocklist "${line}" # the -! ignores duplicates
+        fi
     done < "${user_firewall_json}"
 fi

src/scripts/setldapallowlist.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
+if [[ $# == 1 && "$1" == "--check" ]]; then
+    echo "OK"
+    exit 0
+fi
+
+[[ "${BOX_ENV}" == "test" ]] && exit
+
+systemctl restart cloudron-firewall

src/server.js

@@ -86,6 +86,7 @@ function initializeExpressSync() {
     const authorizeAdmin = routes.accesscontrol.authorize(users.ROLE_ADMIN);
     const authorizeOperator = routes.accesscontrol.authorizeOperator;
     const authorizeUserManager = routes.accesscontrol.authorize(users.ROLE_USER_MANAGER);
+    const authorizeMailManager = routes.accesscontrol.authorize(users.ROLE_MAIL_MANAGER);
 
     // public routes
     router.post('/api/v1/cloudron/setup',    json, routes.provision.setupTokenAuth, routes.provision.providerTokenAuth, routes.provision.setup);    // only available until no-domain
@@ -120,7 +121,8 @@ function initializeExpressSync() {
     router.get ('/api/v1/cloudron/eventlog',                       token, authorizeAdmin, routes.eventlog.list);
     router.get ('/api/v1/cloudron/eventlog/:eventId',              token, authorizeAdmin, routes.eventlog.get);
     router.post('/api/v1/cloudron/sync_external_ldap',       json, token, authorizeAdmin, routes.cloudron.syncExternalLdap);
-    router.get ('/api/v1/cloudron/server_ip',                      token, authorizeAdmin, routes.cloudron.getServerIp);
+    router.get ('/api/v1/cloudron/server_ipv4',                      token, authorizeAdmin, routes.cloudron.getServerIpv4);
+    router.get ('/api/v1/cloudron/server_ipv6',                      token, authorizeAdmin, routes.cloudron.getServerIpv6);
 
     // task routes
     router.get ('/api/v1/tasks',                         token, authorizeAdmin, routes.tasks.list);
@@ -265,8 +267,8 @@ function initializeExpressSync() {
 
     // email routes
     router.get ('/api/v1/mailserver/eventlog',                token, authorizeOwner, routes.mailserver.proxy);
-    router.get ('/api/v1/mailserver/usage',                   token, authorizeAdmin, routes.mailserver.proxy);
     router.post('/api/v1/mailserver/clear_eventlog',          token, authorizeOwner, routes.mailserver.proxy);
+    router.use ('/api/v1/mailserver/files/*',                 token, authorizeOwner, routes.filemanager.proxy('mail'));
     router.get ('/api/v1/mailserver/location',                token, authorizeAdmin, routes.mailserver.getLocation);
     router.post('/api/v1/mailserver/location',          json, token, authorizeAdmin, routes.mailserver.setLocation);
     router.get ('/api/v1/mailserver/max_email_size',          token, authorizeAdmin, routes.mailserver.proxy);
@@ -279,29 +281,31 @@ function initializeExpressSync() {
     router.post('/api/v1/mailserver/dnsbl_config',            token, authorizeAdmin, routes.mailserver.proxy);
     router.get ('/api/v1/mailserver/solr_config',             token, authorizeAdmin, routes.mailserver.proxy);
     router.post('/api/v1/mailserver/solr_config',             token, authorizeAdmin, routes.mailserver.proxy, routes.mailserver.restart);
-    router.use ('/api/v1/mailserver/files/*',                 token, authorizeOwner, routes.filemanager.proxy('mail'));
+    router.get ('/api/v1/mailserver/mailbox_sharing',         token, authorizeAdmin, routes.mailserver.proxy);
+    router.post('/api/v1/mailserver/mailbox_sharing',         token, authorizeAdmin, routes.mailserver.proxy, routes.mailserver.restart);
+    router.get ('/api/v1/mailserver/usage',                   token, authorizeMailManager, routes.mailserver.proxy);
 
-    router.get ('/api/v1/mail/:domain',                               token, authorizeAdmin, routes.mail.getDomain);
-    router.get ('/api/v1/mail/:domain/status',                        token, authorizeAdmin, routes.mail.getStatus);
-    router.post('/api/v1/mail/:domain/mail_from_validation',    json, token, authorizeAdmin, routes.mail.setMailFromValidation);
-    router.post('/api/v1/mail/:domain/catch_all',               json, token, authorizeAdmin, routes.mail.setCatchAllAddress);
-    router.post('/api/v1/mail/:domain/relay',                   json, token, authorizeAdmin, routes.mail.setMailRelay);
+    router.get ('/api/v1/mail/:domain',                               token, authorizeMailManager, routes.mail.getDomain);
     router.post('/api/v1/mail/:domain/enable',                  json, token, authorizeAdmin, routes.mail.setMailEnabled);
-    router.post('/api/v1/mail/:domain/banner',                  json, token, authorizeAdmin, routes.mail.setBanner);
-    router.post('/api/v1/mail/:domain/send_test_mail',          json, token, authorizeAdmin, routes.mail.sendTestMail);
-    router.get ('/api/v1/mail/:domain/mailbox_count',                 token, authorizeAdmin, routes.mail.getMailboxCount);
-    router.get ('/api/v1/mail/:domain/mailboxes',                     token, authorizeAdmin, routes.mail.listMailboxes);
-    router.get ('/api/v1/mail/:domain/mailboxes/:name',               token, authorizeAdmin, routes.mail.getMailbox);
-    router.post('/api/v1/mail/:domain/mailboxes',               json, token, authorizeAdmin, routes.mail.addMailbox);
-    router.post('/api/v1/mail/:domain/mailboxes/:name',         json, token, authorizeAdmin, routes.mail.updateMailbox);
-    router.del ('/api/v1/mail/:domain/mailboxes/:name',         json, token, authorizeAdmin, routes.mail.delMailbox);
-    router.get ('/api/v1/mail/:domain/mailboxes/:name/aliases',       token, authorizeAdmin, routes.mail.getAliases);
-    router.put ('/api/v1/mail/:domain/mailboxes/:name/aliases', json, token, authorizeAdmin, routes.mail.setAliases);
-    router.get ('/api/v1/mail/:domain/lists',                         token, authorizeAdmin, routes.mail.getLists);
-    router.post('/api/v1/mail/:domain/lists',                   json, token, authorizeAdmin, routes.mail.addList);
-    router.get ('/api/v1/mail/:domain/lists/:name',                   token, authorizeAdmin, routes.mail.getList);
-    router.post('/api/v1/mail/:domain/lists/:name',             json, token, authorizeAdmin, routes.mail.updateList);
-    router.del ('/api/v1/mail/:domain/lists/:name',                   token, authorizeAdmin, routes.mail.delList);
+    router.get ('/api/v1/mail/:domain/status',                        token, authorizeMailManager, routes.mail.getStatus);
+    router.post('/api/v1/mail/:domain/mail_from_validation',    json, token, authorizeMailManager, routes.mail.setMailFromValidation);
+    router.post('/api/v1/mail/:domain/catch_all',               json, token, authorizeMailManager, routes.mail.setCatchAllAddress);
+    router.post('/api/v1/mail/:domain/relay',                   json, token, authorizeAdmin, routes.mail.setMailRelay);
+    router.post('/api/v1/mail/:domain/banner',                  json, token, authorizeMailManager, routes.mail.setBanner);
+    router.post('/api/v1/mail/:domain/send_test_mail',          json, token, authorizeMailManager, routes.mail.sendTestMail);
+    router.get ('/api/v1/mail/:domain/mailbox_count',                 token, authorizeMailManager, routes.mail.getMailboxCount);
+    router.get ('/api/v1/mail/:domain/mailboxes',                     token, authorizeMailManager, routes.mail.listMailboxes);
+    router.get ('/api/v1/mail/:domain/mailboxes/:name',               token, authorizeMailManager, routes.mail.getMailbox);
+    router.post('/api/v1/mail/:domain/mailboxes',               json, token, authorizeMailManager, routes.mail.addMailbox);
+    router.post('/api/v1/mail/:domain/mailboxes/:name',         json, token, authorizeMailManager, routes.mail.updateMailbox);
+    router.del ('/api/v1/mail/:domain/mailboxes/:name',         json, token, authorizeMailManager, routes.mail.delMailbox);
+    router.get ('/api/v1/mail/:domain/mailboxes/:name/aliases',       token, authorizeMailManager, routes.mail.getAliases);
+    router.put ('/api/v1/mail/:domain/mailboxes/:name/aliases', json, token, authorizeMailManager, routes.mail.setAliases);
+    router.get ('/api/v1/mail/:domain/lists',                         token, authorizeMailManager, routes.mail.getLists);
+    router.post('/api/v1/mail/:domain/lists',                   json, token, authorizeMailManager, routes.mail.addList);
+    router.get ('/api/v1/mail/:domain/lists/:name',                   token, authorizeMailManager, routes.mail.getList);
+    router.post('/api/v1/mail/:domain/lists/:name',             json, token, authorizeMailManager, routes.mail.updateList);
+    router.del ('/api/v1/mail/:domain/lists/:name',                   token, authorizeMailManager, routes.mail.delList);
 
     // support routes
     router.post('/api/v1/support/ticket',         json, token, authorizeAdmin, routes.support.canCreateTicket, routes.support.createTicket);
@@ -312,7 +316,8 @@ function initializeExpressSync() {
     router.post('/api/v1/domains',                    json, token, authorizeAdmin, routes.domains.add);
     router.get ('/api/v1/domains',                          token,                 routes.domains.list);
     router.get ('/api/v1/domains/:domain',                  token, authorizeAdmin, routes.domains.get);  // this is manage scope because it returns non-restricted fields
-    router.put ('/api/v1/domains/:domain',            json, token, authorizeAdmin, routes.domains.update);
+    router.post('/api/v1/domains/:domain/config',      json, token, authorizeAdmin, routes.domains.setConfig);
+    router.post('/api/v1/domains/:domain/wellknown',   json, token, authorizeAdmin, routes.domains.setWellKnown);
     router.del ('/api/v1/domains/:domain',                  token, authorizeAdmin, routes.domains.del);
     router.get ('/api/v1/domains/:domain/dns_check',        token, authorizeAdmin, routes.domains.checkDnsRecords);
 

src/services.js

@@ -33,7 +33,6 @@ exports = module.exports = {
 const addonConfigs = require('./addonconfigs.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
-    async = require('async'),
     blobs = require('./blobs.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
@@ -43,12 +42,13 @@ const addonConfigs = require('./addonconfigs.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
     hat = require('./hat.js'),
+    http = require('http'),
     infra = require('./infra_version.js'),
     mail = require('./mail.js'),
-    once = require('once'),
     os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
+    { pipeline } = require('stream'),
     promiseRetry = require('./promise-retry.js'),
     safe = require('safetydance'),
     semver = require('semver'),
@@ -58,7 +58,6 @@ const addonConfigs = require('./addonconfigs.js'),
     spawn = require('child_process').spawn,
     split = require('split'),
     superagent = require('superagent'),
-    request = require('request'),
     system = require('./system.js');
 
 const NOOP = async function (/*app, options*/) {};
@@ -291,8 +290,7 @@ async function containerStatus(containerName, tokenEnvName) {
     if (error && (error.reason === BoxError.NOT_FOUND || error.reason === BoxError.INACTIVE)) return { status: exports.SERVICE_STATUS_STOPPED };
     if (error) throw error;
 
-    const [networkError, response] = await safe(superagent.get(`https://${addonDetails.ip}:3000/healthcheck?access_token=${addonDetails.token}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.get(`http://${addonDetails.ip}:3000/healthcheck?access_token=${addonDetails.token}`)
         .timeout(20000)
         .ok(() => true));
 
@@ -589,10 +587,9 @@ async function waitForContainer(containerName, tokenEnvName) {
 
     const result = await getContainerDetails(containerName, tokenEnvName);
 
-    await promiseRetry({ times: 10, interval: 15000 }, async () => {
-        const [networkError, response] = await safe(superagent.get(`https://${result.ip}:3000/healthcheck?access_token=${result.token}`)
+    await promiseRetry({ times: 10, interval: 15000, debug }, async () => {
+        const [networkError, response] = await safe(superagent.get(`http://${result.ip}:3000/healthcheck?access_token=${result.token}`)
             .timeout(5000)
-            .disableTLSCerts()
             .ok(() => true));
 
         if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error waiting for ${containerName}: ${networkError.message}`);
@@ -685,18 +682,15 @@ async function restoreAddons(app, addons) {
     }
 }
 
-function importAppDatabase(app, addon, callback) {
+async function importAppDatabase(app, addon) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof addon, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    if (!(addon in ADDONS)) return callback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
+    if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`);
 
-    async.series([
-        ADDONS[addon].setup.bind(null, app, app.manifest.addons[addon]),
-        ADDONS[addon].clear.bind(null, app, app.manifest.addons[addon]), // clear in case we crashed in a restore
-        ADDONS[addon].restore.bind(null, app, app.manifest.addons[addon])
-    ], callback);
+    await ADDONS[addon].setup(app, app.manifest.addons[addon]);
+    await ADDONS[addon].clear(app, app.manifest.addons[addon]); // clear in case we crashed in a restore
+    await ADDONS[addon].restore(app, app.manifest.addons[addon]);
 }
 
 async function importDatabase(addon) {
@@ -892,7 +886,7 @@ async function setupTurn(app, options) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
 
-    const turnSecret = await blobs.get(blobs.ADDON_TURN_SECRET);
+    const turnSecret = await blobs.getString(blobs.ADDON_TURN_SECRET);
     if (!turnSecret) throw new BoxError(BoxError.ADDONS_ERROR, 'Turn secret is missing');
 
     const env = [
@@ -919,17 +913,19 @@ async function startTurn(existingInfra) {
     const memory = system.getMemoryAllocation(memoryLimit);
     const realm = settings.dashboardFqdn();
 
-    let turnSecret = await blobs.get(blobs.ADDON_TURN_SECRET);
+    let turnSecret = await blobs.getString(blobs.ADDON_TURN_SECRET);
     if (!turnSecret) {
         debug('startTurn: generting turn secret');
         turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter
-        await blobs.set(blobs.ADDON_TURN_SECRET, Buffer.from(turnSecret));
+        await blobs.setString(blobs.ADDON_TURN_SECRET, turnSecret);
     }
 
     const readOnly = !serviceConfig.recoveryMode ? '--read-only' : '';
     const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : '';
 
-    // this exports 3478/tcp, 5349/tls and 50000-51000/udp. note that this runs on the host network!
+    // this exports 3478/tcp, 5349/tls and 50000-51000/udp. note that this runs on the host network because docker's userland proxy
+    // is spun for every port. we can disable this in some future release with --userland-proxy=false
+    // https://github.com/moby/moby/issues/8356 and https://github.com/moby/moby/issues/14856
     const runCmd = `docker run --restart=always -d --name="turn" \
                 --hostname turn \
                 --net host \
@@ -967,20 +963,21 @@ async function setupEmail(app, options) {
     const mailDomains = await mail.listDomains();
     const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(',');
 
-    const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
     // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation)
     const env = [
-        { name: `${envPrefix}MAIL_SMTP_SERVER`, value: 'mail' },
-        { name: `${envPrefix}MAIL_SMTP_PORT`, value: '2525' },
-        { name: `${envPrefix}MAIL_IMAP_SERVER`, value: 'mail' },
-        { name: `${envPrefix}MAIL_IMAP_PORT`, value: '9993' },
-        { name: `${envPrefix}MAIL_SIEVE_SERVER`, value: 'mail' },
-        { name: `${envPrefix}MAIL_SIEVE_PORT`, value: '4190' },
-        { name: `${envPrefix}MAIL_DOMAIN`, value: app.domain },
-        { name: `${envPrefix}MAIL_DOMAINS`, value: mailInDomains },
-        { name: 'CLOUDRON_MAIL_SERVER_HOST', value: settings.mailFqdn() }, // this is also a hint to reconfigure on mail server name change
-        { name: `${envPrefix}LDAP_MAILBOXES_BASE_DN`, value: 'ou=mailboxes,dc=cloudron' }
+        { name: 'CLOUDRON_EMAIL_SMTP_SERVER', value: 'mail' },
+        { name: 'CLOUDRON_EMAIL_SMTP_PORT', value: '2525' },
+        { name: 'CLOUDRON_EMAIL_SMTPS_PORT', value: '2465' },
+        { name: 'CLOUDRON_EMAIL_STARTTLS_PORT', value: '2587' },
+        { name: 'CLOUDRON_EMAIL_IMAP_SERVER', value: 'mail' },
+        { name: 'CLOUDRON_EMAIL_IMAPS_PORT', value: '9993' },
+        { name: 'CLOUDRON_EMAIL_IMAP_PORT', value: '9393' },
+        { name: 'CLOUDRON_EMAIL_SIEVE_SERVER', value: 'mail' },
+        { name: 'CLOUDRON_EMAIL_SIEVE_PORT', value: '4190' }, // starttls
+        { name: 'CLOUDRON_EMAIL_DOMAIN', value: app.domain },
+        { name: 'CLOUDRON_EMAIL_DOMAINS', value: mailInDomains },
+        { name: 'CLOUDRON_EMAIL_SERVER_HOST', value: settings.mailFqdn() }, // this is also a hint to reconfigure on mail server name change
+        { name: 'CLOUDRON_EMAIL_LDAP_MAILBOXES_BASE_DN', value: 'ou=mailboxes,dc=cloudron' }
     ];
 
     debug('Setting up Email');
@@ -1003,17 +1000,15 @@ async function setupLdap(app, options) {
 
     if (!app.sso) return;
 
-    const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
     const env = [
-        { name: `${envPrefix}LDAP_SERVER`, value: '172.18.0.1' },
+        { name: 'CLOUDRON_LDAP_SERVER', value: '172.18.0.1' },
         { name: 'CLOUDRON_LDAP_HOST', value: '172.18.0.1' }, // to keep things in sync with the database _HOST vars
-        { name: `${envPrefix}LDAP_PORT`, value: '' + constants.LDAP_PORT },
-        { name: `${envPrefix}LDAP_URL`, value: 'ldap://172.18.0.1:' + constants.LDAP_PORT },
-        { name: `${envPrefix}LDAP_USERS_BASE_DN`, value: 'ou=users,dc=cloudron' },
-        { name: `${envPrefix}LDAP_GROUPS_BASE_DN`, value: 'ou=groups,dc=cloudron' },
-        { name: `${envPrefix}LDAP_BIND_DN`, value: 'cn='+ app.id + ',ou=apps,dc=cloudron' },
-        { name: `${envPrefix}LDAP_BIND_PASSWORD`, value: hat(4 * 128) } // this is ignored
+        { name: 'CLOUDRON_LDAP_PORT', value: '' + constants.LDAP_PORT },
+        { name: 'CLOUDRON_LDAP_URL', value: 'ldap://172.18.0.1:' + constants.LDAP_PORT },
+        { name: 'CLOUDRON_LDAP_USERS_BASE_DN', value: 'ou=users,dc=cloudron' },
+        { name: 'CLOUDRON_LDAP_GROUPS_BASE_DN', value: 'ou=groups,dc=cloudron' },
+        { name: 'CLOUDRON_LDAP_BIND_DN', value: 'cn='+ app.id + ',ou=apps,dc=cloudron' },
+        { name: 'CLOUDRON_LDAP_BIND_PASSWORD', value: hat(4 * 128) } // this is ignored
     ];
 
     debug('Setting up LDAP');
@@ -1043,17 +1038,15 @@ async function setupSendMail(app, options) {
 
     const password = existingPassword || hat(4 * 48); // see box#565 for password length
 
-    const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
     const env = [
-        { name: `${envPrefix}MAIL_SMTP_SERVER`, value: 'mail' },
-        { name: `${envPrefix}MAIL_SMTP_PORT`, value: '2525' },
-        { name: `${envPrefix}MAIL_SMTPS_PORT`, value: '2465' },
-        { name: `${envPrefix}MAIL_STARTTLS_PORT`, value: '2587' },
-        { name: `${envPrefix}MAIL_SMTP_USERNAME`, value: app.mailboxName + '@' + app.mailboxDomain },
-        { name: `${envPrefix}MAIL_SMTP_PASSWORD`, value: password },
-        { name: `${envPrefix}MAIL_FROM`, value: app.mailboxName + '@' + app.mailboxDomain },
-        { name: `${envPrefix}MAIL_DOMAIN`, value: app.mailboxDomain }
+        { name: 'CLOUDRON_MAIL_SMTP_SERVER', value: 'mail' },
+        { name: 'CLOUDRON_MAIL_SMTP_PORT', value: '2525' },
+        { name: 'CLOUDRON_MAIL_SMTPS_PORT', value: '2465' },
+        { name: 'CLOUDRON_MAIL_STARTTLS_PORT', value: '2587' },
+        { name: 'CLOUDRON_MAIL_SMTP_USERNAME', value: app.mailboxName + '@' + app.mailboxDomain },
+        { name: 'CLOUDRON_MAIL_SMTP_PASSWORD', value: password },
+        { name: 'CLOUDRON_MAIL_FROM', value: app.mailboxName + '@' + app.mailboxDomain },
+        { name: 'CLOUDRON_MAIL_DOMAIN', value: app.mailboxDomain }
     ];
     debug('Setting sendmail addon config to %j', env);
     await addonConfigs.set(app.id, 'sendmail', env);
@@ -1080,18 +1073,16 @@ async function setupRecvMail(app, options) {
 
     const password = existingPassword || hat(4 * 48);  // see box#565 for password length
 
-    const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
     const env = [
-        { name: `${envPrefix}MAIL_IMAP_SERVER`, value: 'mail' },
-        { name: `${envPrefix}MAIL_IMAP_PORT`, value: '9393' },
-        { name: `${envPrefix}MAIL_IMAPS_PORT`, value: '9993' },
-        { name: `${envPrefix}MAIL_POP3_PORT`, value: '9595' },
-        { name: `${envPrefix}MAIL_POP3S_PORT`, value: '9995' },
-        { name: `${envPrefix}MAIL_IMAP_USERNAME`, value: app.inboxName + '@' + app.inboxDomain },
-        { name: `${envPrefix}MAIL_IMAP_PASSWORD`, value: password },
-        { name: `${envPrefix}MAIL_TO`, value: app.inboxName + '@' + app.inboxDomain },
-        { name: `${envPrefix}MAIL_TO_DOMAIN`, value: app.inboxDomain },
+        { name: 'CLOUDRON_MAIL_IMAP_SERVER', value: 'mail' },
+        { name: 'CLOUDRON_MAIL_IMAP_PORT', value: '9393' },
+        { name: 'CLOUDRON_MAIL_IMAPS_PORT', value: '9993' },
+        { name: 'CLOUDRON_MAIL_POP3_PORT', value: '9595' },
+        { name: 'CLOUDRON_MAIL_POP3S_PORT', value: '9995' },
+        { name: 'CLOUDRON_MAIL_IMAP_USERNAME', value: app.inboxName + '@' + app.inboxDomain },
+        { name: 'CLOUDRON_MAIL_IMAP_PASSWORD', value: password },
+        { name: 'CLOUDRON_MAIL_TO', value: app.inboxName + '@' + app.inboxDomain },
+        { name: 'CLOUDRON_MAIL_TO_DOMAIN', value: app.inboxDomain },
     ];
 
     debug('setupRecvMail: setting recvmail addon config to %j', env);
@@ -1182,29 +1173,26 @@ async function setupMySql(app, options) {
 
     const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN');
 
-    const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `?access_token=${result.token}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `?access_token=${result.token}`)
         .send(data)
         .ok(() => true));
 
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mysql: ${networkError.message}`);
     if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up mysql. Status code: ${response.status} message: ${response.body.message}`);
 
-    const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
     let env = [
-        { name: `${envPrefix}MYSQL_USERNAME`, value: data.username },
-        { name: `${envPrefix}MYSQL_PASSWORD`, value: data.password },
-        { name: `${envPrefix}MYSQL_HOST`, value: 'mysql' },
-        { name: `${envPrefix}MYSQL_PORT`, value: '3306' }
+        { name: 'CLOUDRON_MYSQL_USERNAME', value: data.username },
+        { name: 'CLOUDRON_MYSQL_PASSWORD', value: data.password },
+        { name: 'CLOUDRON_MYSQL_HOST', value: 'mysql' },
+        { name: 'CLOUDRON_MYSQL_PORT', value: '3306' }
     ];
 
     if (options.multipleDatabases) {
-        env = env.concat({ name: `${envPrefix}MYSQL_DATABASE_PREFIX`, value: `${data.prefix}_` });
+        env = env.concat({ name: 'CLOUDRON_MYSQL_DATABASE_PREFIX', value: `${data.prefix}_` });
     } else {
         env = env.concat(
-            { name: `${envPrefix}MYSQL_URL`, value: `mysql://${data.username}:${data.password}@mysql/${data.database}` },
-            { name: `${envPrefix}MYSQL_DATABASE`, value: data.database }
+            { name: 'CLOUDRON_MYSQL_URL', value: `mysql://${data.username}:${data.password}@mysql/${data.database}` },
+            { name: 'CLOUDRON_MYSQL_DATABASE', value: data.database }
         );
     }
 
@@ -1220,8 +1208,7 @@ async function clearMySql(app, options) {
 
     const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN');
 
-    const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`)
         .ok(() => true));
 
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mysql: ${networkError.message}`);
@@ -1237,8 +1224,7 @@ async function teardownMySql(app, options) {
 
     const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN');
 
-    const [networkError, response] = await safe(superagent.del(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.del(`http://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`)
         .ok(() => true));
 
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql: ${networkError.message}`);
@@ -1251,26 +1237,43 @@ async function pipeRequestToFile(url, filename) {
     assert.strictEqual(typeof url, 'string');
     assert.strictEqual(typeof filename, 'string');
 
-    await new Promise((resolve, reject) => {
+    return new Promise((resolve, reject) => {
         const writeStream = fs.createWriteStream(filename);
+        const request = http.request(url, { method: 'POST' }); // ClientRequest
+        request.setTimeout(60000, () => request.destroy(new Error('Request timedout'))); // connect OR post-connect idle timeout
 
-        const done = once(function (error) { // the writeStream and the request can both error
-            if (error) writeStream.close();
-            if (error) reject(error); else resolve();
+        request.on('error', (error) => reject(new BoxError(BoxError.NETWORK_ERROR, `Could not pipe ${url} to ${filename}: ${error.message}`))); // network error, dns error
+        request.on('response', (response) => {
+            if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code or HTTP error when piping ${url} to ${filename}: status ${response.statusCode}`));
+
+            pipeline(response, writeStream, (error) => {
+                if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error piping ${url} to ${filename}: ${error.message}`));
+
+                if (!response.complete) return reject(new BoxError(BoxError.ADDONS_ERROR, `Response not complete when piping ${url} to ${filename}`));
+                resolve();
+            });
+        });
+        request.end(); // make the request
+    });
+}
+
+async function pipeFileToRequest(filename, url) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof url, 'string');
+
+    return new Promise((resolve, reject) => {
+        const readStream = fs.createReadStream(filename);
+        const request = http.request(url, { method: 'POST' }); // ClientRequest
+        request.setTimeout(60000, () => request.destroy(new Error('Request timedout'))); // connect OR post-connect idle timeout
+        request.on('response', (response) => {
+            response.resume(); // drain the response
+            if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code or HTTP error when piping ${filename} to ${url}: status ${response.statusCode} complete ${response.complete}`));
+
+            resolve();
         });
 
-        writeStream.on('error', (error) => done(new BoxError(BoxError.FS_ERROR, `Error writing to ${filename}: ${error.message}`)));
-
-        writeStream.on('open', function () {
-            // note: do not attach to post callback handler because this will buffer the entire reponse!
-            // see https://github.com/request/request/issues/2270
-            const req = request.post(url, { rejectUnauthorized: false });
-            req.on('error', (error) => done(new BoxError(BoxError.NETWORK_ERROR, `Request error writing to ${filename}: ${error.message}`))); // network error, dns error, request errored in middle etc
-            req.on('response', function (response) {
-                if (response.statusCode !== 200) return done(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code when piping ${url}: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`));
-
-                response.pipe(writeStream).on('finish', done); // this is hit after data written to disk
-            });
+        pipeline(readStream, request, function (error) {
+            if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error piping file ${filename} to request ${url}`));
         });
     });
 }
@@ -1285,7 +1288,7 @@ async function backupMySql(app, options) {
 
     const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN');
 
-    const url = `https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/backup?access_token=${result.token}`;
+    const url = `http://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/backup?access_token=${result.token}`;
     await pipeRequestToFile(url, dumpPath('mysql', app.id));
 }
 
@@ -1299,21 +1302,8 @@ async function restoreMySql(app, options) {
 
     const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN');
 
-    await new Promise((resolve, reject) => {
-        reject = once(reject); // protect from multiple returns with streams
-
-        const input = fs.createReadStream(dumpPath('mysql', app.id));
-        input.on('error', (error) => reject(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mysql: ${error.message}`)));
-
-        const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
-            if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql: ${error.message}`));
-            if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
-
-            resolve();
-        });
-
-        input.pipe(restoreReq);
-    });
+    const url = `http://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`;
+    await pipeFileToRequest(dumpPath('mysql', app.id), url);
 }
 
 function postgreSqlNames(appId) {
@@ -1387,22 +1377,19 @@ async function setupPostgreSql(app, options) {
 
     const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN');
 
-    const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/databases?access_token=${result.token}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/databases?access_token=${result.token}`)
         .send(data)
         .ok(() => true));
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up postgresql: ${networkError.message}`);
     if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up postgresql. Status code: ${response.status} message: ${response.body.message}`);
 
-    const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
     const env = [
-        { name: `${envPrefix}POSTGRESQL_URL`, value: `postgres://${data.username}:${data.password}@postgresql/${data.database}` },
-        { name: `${envPrefix}POSTGRESQL_USERNAME`, value: data.username },
-        { name: `${envPrefix}POSTGRESQL_PASSWORD`, value: data.password },
-        { name: `${envPrefix}POSTGRESQL_HOST`, value: 'postgresql' },
-        { name: `${envPrefix}POSTGRESQL_PORT`, value: '5432' },
-        { name: `${envPrefix}POSTGRESQL_DATABASE`, value: data.database }
+        { name: 'CLOUDRON_POSTGRESQL_URL', value: `postgres://${data.username}:${data.password}@postgresql/${data.database}` },
+        { name: 'CLOUDRON_POSTGRESQL_USERNAME', value: data.username },
+        { name: 'CLOUDRON_POSTGRESQL_PASSWORD', value: data.password },
+        { name: 'CLOUDRON_POSTGRESQL_HOST', value: 'postgresql' },
+        { name: 'CLOUDRON_POSTGRESQL_PORT', value: '5432' },
+        { name: 'CLOUDRON_POSTGRESQL_DATABASE', value: data.database }
     ];
 
     debug('Setting postgresql addon config to %j', env);
@@ -1420,8 +1407,7 @@ async function clearPostgreSql(app, options) {
 
     const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN');
 
-    const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}&locale=${locale}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}&locale=${locale}`)
         .ok(() => true));
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing postgresql: ${networkError.message}`);
     if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing postgresql. Status code: ${response.status} message: ${response.body.message}`);
@@ -1435,8 +1421,7 @@ async function teardownPostgreSql(app, options) {
 
     const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN');
 
-    const [networkError, response] = await safe(superagent.del(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.del(`http://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`)
         .ok(() => true));
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error tearing down postgresql: ${networkError.message}`);
     if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down postgresql. Status code: ${response.status} message: ${response.body.message}`);
@@ -1453,9 +1438,7 @@ async function backupPostgreSql(app, options) {
     const { database } = postgreSqlNames(app.id);
 
     const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN');
-
-    const url = `https://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`;
-    await pipeRequestToFile(url, dumpPath('postgresql', app.id));
+    await pipeRequestToFile(`http://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`, dumpPath('postgresql', app.id));
 }
 
 async function restorePostgreSql(app, options) {
@@ -1468,21 +1451,7 @@ async function restorePostgreSql(app, options) {
 
     const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN');
 
-    await new Promise((resolve, reject) => {
-        resolve = once(resolve); // protect from multiple returns with streams
-
-        const input = fs.createReadStream(dumpPath('postgresql', app.id));
-        input.on('error', (error) => reject(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring postgresql: ${error.message}`)));
-
-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
-            if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql: ${error.message}`));
-            if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
-
-            resolve();
-        });
-
-        input.pipe(restoreReq);
-    });
+    await pipeFileToRequest(dumpPath('postgresql', app.id), `http://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`);
 }
 
 async function startMongodb(existingInfra) {
@@ -1550,27 +1519,24 @@ async function setupMongoDb(app, options) {
 
     const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN');
 
-    const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/databases?access_token=${result.token}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/databases?access_token=${result.token}`)
         .send(data)
         .ok(() => true));
 
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mongodb: ${networkError.message}`);
     if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up mongodb. Status code: ${response.status} message: ${response.body.message}`);
 
-    const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
     const env = [
-        { name: `${envPrefix}MONGODB_URL`, value : `mongodb://${data.username}:${data.password}@mongodb:27017/${data.database}` },
-        { name: `${envPrefix}MONGODB_USERNAME`, value : data.username },
-        { name: `${envPrefix}MONGODB_PASSWORD`, value: data.password },
-        { name: `${envPrefix}MONGODB_HOST`, value : 'mongodb' },
-        { name: `${envPrefix}MONGODB_PORT`, value : '27017' },
-        { name: `${envPrefix}MONGODB_DATABASE`, value : data.database }
+        { name: 'CLOUDRON_MONGODB_URL', value : `mongodb://${data.username}:${data.password}@mongodb:27017/${data.database}` },
+        { name: 'CLOUDRON_MONGODB_USERNAME', value : data.username },
+        { name: 'CLOUDRON_MONGODB_PASSWORD', value: data.password },
+        { name: 'CLOUDRON_MONGODB_HOST', value : 'mongodb' },
+        { name: 'CLOUDRON_MONGODB_PORT', value : '27017' },
+        { name: 'CLOUDRON_MONGODB_DATABASE', value : data.database }
     ];
 
     if (options.oplog) {
-        env.push({ name: `${envPrefix}MONGODB_OPLOG_URL`, value : `mongodb://${data.username}:${data.password}@mongodb:27017/local?authSource=${data.database}` });
+        env.push({ name: 'CLOUDRON_MONGODB_OPLOG_URL', value : `mongodb://${data.username}:${data.password}@mongodb:27017/local?authSource=${data.database}` });
     }
 
     debug('Setting mongodb addon config to %j', env);
@@ -1586,8 +1552,7 @@ async function clearMongodb(app, options) {
     const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE');
     if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error clearing mongodb. No database');
 
-    const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}`)
         .ok(() => true));
 
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mongodb: ${networkError.message}`);
@@ -1603,8 +1568,7 @@ async function teardownMongoDb(app, options) {
     const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE');
     if (!database) return;
 
-    const [networkError, response] = await safe(superagent.del(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.del(`http://${result.ip}:3000/databases/${database}?access_token=${result.token}`)
         .ok(() => true));
 
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb: ${networkError.message}`);
@@ -1624,8 +1588,7 @@ async function backupMongoDb(app, options) {
     const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE');
     if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error backing up mongodb. No database');
 
-    const url = `https://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`;
-    await pipeRequestToFile(url, dumpPath('mongodb', app.id));
+    await pipeRequestToFile(`http://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`, dumpPath('mongodb', app.id));
 }
 
 async function restoreMongoDb(app, options) {
@@ -1639,21 +1602,7 @@ async function restoreMongoDb(app, options) {
     const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE');
     if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error restoring mongodb. No database');
 
-    await new Promise((resolve, reject) => {
-        reject = once(reject); // protect from multiple returns with streams
-
-        const readStream = fs.createReadStream(dumpPath('mongodb', app.id));
-        readStream.on('error', (error) => reject(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mongodb: ${error.message}`)));
-
-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
-            if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb: ${error.message}`));
-            if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
-
-            resolve();
-        });
-
-        readStream.pipe(restoreReq);
-    });
+    await pipeFileToRequest(dumpPath('mongodb', app.id), `http://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}`);
 }
 
 async function startGraphite(existingInfra) {
@@ -1785,13 +1734,11 @@ async function setupRedis(app, options) {
                 --label isCloudronManaged=true \
                 ${readOnly} -v /tmp -v /run ${tag} ${cmd}`;
 
-    const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
     const env = [
-        { name: `${envPrefix}REDIS_URL`, value: 'redis://redisuser:' + redisPassword + '@redis-' + app.id },
-        { name: `${envPrefix}REDIS_PASSWORD`, value: redisPassword },
-        { name: `${envPrefix}REDIS_HOST`, value: redisName },
-        { name: `${envPrefix}REDIS_PORT`, value: '6379' }
+        { name: 'CLOUDRON_REDIS_URL', value: 'redis://redisuser:' + redisPassword + '@redis-' + app.id },
+        { name: 'CLOUDRON_REDIS_PASSWORD', value: redisPassword },
+        { name: 'CLOUDRON_REDIS_HOST', value: redisName },
+        { name: 'CLOUDRON_REDIS_PORT', value: '6379' }
     ];
 
     const [inspectError, result] = await safe(docker.inspect(redisName));
@@ -1815,8 +1762,7 @@ async function clearRedis(app, options) {
 
     const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN');
 
-    const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/clear?access_token=${result.token}`)
-        .disableTLSCerts()
+    const [networkError, response] = await safe(superagent.post(`http://${result.ip}:3000/clear?access_token=${result.token}`)
         .ok(() => true));
 
     if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing redis: ${networkError.message}`);
@@ -1845,9 +1791,7 @@ async function backupRedis(app, options) {
     debug('Backing up redis');
 
     const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN');
-
-    const url = `https://${result.ip}:3000/backup?access_token=${result.token}`;
-    await pipeRequestToFile(url, dumpPath('redis', app.id));
+    await pipeRequestToFile(`http://${result.ip}:3000/backup?access_token=${result.token}`, dumpPath('redis', app.id));
 }
 
 async function restoreRedis(app, options) {
@@ -1857,28 +1801,7 @@ async function restoreRedis(app, options) {
     debug('Restoring redis');
 
     const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN');
-
-    await new Promise((resolve, reject) => {
-        reject = once(reject); // protect from multiple returns with streams
-
-        let input;
-        const newDumpLocation = dumpPath('redis', app.id);
-        if (fs.existsSync(newDumpLocation)) {
-            input = fs.createReadStream(newDumpLocation);
-        } else { // old location of dumps
-            input = fs.createReadStream(path.join(paths.APPS_DATA_DIR, app.id, 'redis/dump.rdb'));
-        }
-        input.on('error', (error) => reject(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring redis: ${error.message}`)));
-
-        const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
-            if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis: ${error.message}`));
-            if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis. Status code: ${response.statusCode} message: ${response.body.message}`));
-
-            resolve();
-        });
-
-        input.pipe(restoreReq);
-    });
+    await pipeFileToRequest(dumpPath('redis', app.id), `http://${result.ip}:3000/restore?access_token=${result.token}`);
 }
 
 async function statusTurn() {