6.3.4 changes

(cherry picked from commit c052882de9)

.jshintrc

@@ -0,0 +1,5 @@
+{
+    "node": true,
+    "unused": true,
+    "esversion": 8
+}

CHANGES

@@ -2232,3 +2232,69 @@
 [6.2.4]
 * Another addon crash fix
 
+[6.2.5]
+* update: set memory limit properly
+* Fix bug where renew certs button did not work
+* sftp: fix rebuild condition
+* Fix display of user management/dashboard visiblity for email apps
+* graphite: disable tagdb and reduce log noise
+
+[6.2.6]
+* Fix issue where collectd is restarted too quickly before graphite
+
+[6.2.7]
+* redis: backup before upgrade
+
+[6.2.8]
+* linode object storage: update aws sdk to make it work again
+* Fix crash in blocklist setting when source and list have mixed ip versions
+* mysql: bump connection limit to 200
+* namecheap: fix issue where DNS updates and del were not working
+* turn: turn off verbose logging
+* Fix crash when parsing df output (set LC_ALL for box service)
+
+[6.3.0]
+* mail: allow TLS from internal hosts
+* tokens: add lastUsedTime
+* update: set memory limit properly
+* addons: better error handling
+* filemanager: various enhancements
+* sftp: fix rebuild condition
+* app mailbox is now optional
+* Fix display of user management/dashboard visiblity for email apps
+* graphite: disable tagdb and reduce log noise
+* hsts: change max-age to 2 years
+* clone: copy over redis memory limit
+* namecheap: fix bug where records were not removed
+* add UI to disable 2FA of a user
+* mail: add active flag to mailboxes and lists
+* Implement OCSP stapling
+* security: send new browser login location notification email
+* backups: add fqdn to the backup filename
+* import all boxdata settings into the database
+* volumes: generate systemd mount configs based on type
+* postgresql: set max conn limit per db
+* ubuntu 16: add alert about EOL
+* clone: save and restore app config
+* app import: restore icon, tag, label, proxy configs etc
+* sieve: fix redirects to not do SRS
+* notifications are now system level instead of per-user
+* vultr DNS
+* vultr object storage
+* mail: do not forward spam to mailing lists
+
+[6.3.1]
+* Fix cert migration issues
+
+[6.3.2]
+* Avatar was migrated as base64 instead of binary
+* Fix issue where filemanager came up empty for CIFS mounts
+
+[6.3.3]
+* volumes: add filesystem volume type for shared folders
+* mail: enable sieve extension editheader
+* mail: update solr to 8.9.0
+
+[6.3.4]
+* Fix issue where old nginx configs where not removed before upgrade
+

README.md

@@ -1,3 +1,5 @@
+![Translation status](https://translate.cloudron.io/widgets/cloudron/-/svg-badge.svg)
+
 # Cloudron
 
 [Cloudron](https://cloudron.io) is the best way to run apps on your server.
@@ -70,8 +72,13 @@ Just to give some heads up, we are a bit restrictive in merging changes. We are
 would like to keep our maintenance burden low. It might be best to discuss features first in the [forum](https://forum.cloudron.io),
 to also figure out how many other people will use it to justify maintenance for a feature.
 
+# Localization
+
+![Translation status](https://translate.cloudron.io/widgets/cloudron/-/287x66-white.png)
+
 ## Support
 
 * [Documentation](https://docs.cloudron.io/)
 * [Forum](https://forum.cloudron.io/)
 
+

baseimage/initializeBaseUbuntuImage.sh

@@ -48,9 +48,11 @@ apt-get -y install --no-install-recommends \
     linux-generic \
     logrotate \
     $mysql_package \
+    nfs-common \
     openssh-server \
     pwgen \
     resolvconf \
+    sshfs \
     swaks \
     tzdata \
     unattended-upgrades \

box.js

@@ -15,7 +15,7 @@ const NOOP_CALLBACK = function () { };
 function setupLogging(callback) {
     if (process.env.BOX_ENV === 'test') return callback();
 
-    var logfileStream = fs.createWriteStream(paths.BOX_LOG_FILE, { flags:'a' });
+    const logfileStream = fs.createWriteStream(paths.BOX_LOG_FILE, { flags:'a' });
     process.stdout.write = process.stderr.write = logfileStream.write.bind(logfileStream);
 
     callback();

migrations/20210315194623-tokens-add-lastUsedTime.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE tokens ADD COLUMN lastUsedTime TIMESTAMP NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE tokens DROP COLUMN lastUsedTime', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210317014101-apps-add-enableMailbox.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN enableMailbox BOOLEAN DEFAULT 1', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN enableMailbox', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20210415030645-mailboxes-add-active.js

@@ -0,0 +1,17 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes ADD COLUMN active BOOLEAN DEFAULT 1', function (error) {
+        if (error) return callback(error);
+
+        callback();
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN active', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20210429194328-users-add-avatar.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    path = require('path');
+
+const AVATAR_DIR = '/home/yellowtent/boxdata/profileicons';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN avatar MEDIUMBLOB', function (error) {
+        if (error) return callback(error);
+
+        fs.readdir(AVATAR_DIR, function (error, filenames) {
+            if (error && error.code === 'ENOENT') return callback();
+            if (error) return callback(error);
+
+            async.eachSeries(filenames, function (filename, iteratorCallback) {
+                const avatar = fs.readFileSync(path.join(AVATAR_DIR, filename));
+                const userId = filename;
+
+                db.runSql('UPDATE users SET avatar=? WHERE id=?', [ avatar, userId ], iteratorCallback);
+            }, function (error) {
+                if (error) return callback(error);
+
+                fs.rmdir(AVATAR_DIR, { recursive: true }, callback);
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN avatar', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20210429224354-settings-add-valueBlob.js

@@ -0,0 +1,20 @@
+'use strict';
+
+const fs = require('fs');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE settings ADD COLUMN valueBlob MEDIUMBLOB', function (error) {
+        if (error) return callback(error);
+
+        fs.readFile('/home/yellowtent/boxdata/avatar.png', function (error, avatar) {
+            if (error && error.code === 'ENOENT') return callback();
+            if (error) return callback(error);
+
+            db.runSql('INSERT INTO settings (name, valueBlob) VALUES (?, ?)', [ 'cloudron_avatar', avatar ], callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210430120936-users-add-loginLocationsJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN loginLocationsJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN loginLocationsJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210430200947-apps-add-icon.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    path = require('path');
+
+const APPICONS_DIR = '/home/yellowtent/boxdata/appicons';
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN icon MEDIUMBLOB'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN appStoreIcon MEDIUMBLOB'),
+        function migrateIcons(next) {
+            fs.readdir(APPICONS_DIR, function (error, filenames) {
+                if (error && error.code === 'ENOENT') return next();
+                if (error) return next(error);
+
+                async.eachSeries(filenames, function (filename, iteratorCallback) {
+                    const icon = fs.readFileSync(path.join(APPICONS_DIR, filename));
+                    const appId = filename.split('.')[0];
+
+                    if (filename.endsWith('.user.png')) {
+                        db.runSql('UPDATE apps SET icon=? WHERE id=?', [ icon, appId ], iteratorCallback);
+                    } else {
+                        db.runSql('UPDATE apps SET appStoreIcon=? WHERE id=?', [ icon, appId ], iteratorCallback);
+                    }
+                }, function (error) {
+                    if (error) return next(error);
+
+                    fs.rmdir(APPICONS_DIR, { recursive: true }, next);
+                });
+            });
+        }
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN icon'),
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN appStoreIcon'),
+    ], callback);
+};

migrations/20210430230901-apps-alter-ts-update.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210501044940-blobs-add-table.js

@@ -0,0 +1,20 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    const cmd = 'CREATE TABLE blobs(' +
+      'id VARCHAR(128) NOT NULL UNIQUE,' +
+      'value MEDIUMBLOB,' +
+      'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE blobs', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210503182308-blobs-migrate-secrets.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    safe = require('safetydance');
+
+const BOX_DATA_DIR = '/home/yellowtent/boxdata';
+const PLATFORM_DATA_DIR = '/home/yellowtent/platformdata';
+
+exports.up = function (db, callback) {
+    let funcs = [];
+
+    const acmeKey = safe.fs.readFileSync(`${BOX_DATA_DIR}/acme/acme.key`);
+    if (acmeKey) {
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'acme_account_key', acmeKey ]));
+        funcs.push(fs.rmdir.bind(fs, `${BOX_DATA_DIR}/acme`, { recursive: true }));
+    }
+    const dhparams = safe.fs.readFileSync(`${BOX_DATA_DIR}/dhparams.pem`);
+    if (dhparams) {
+        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/dhparams.pem`, dhparams);
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'dhparams', dhparams ]));
+        // leave the dhparms here for the moment because startup code regenerates box nginx config and reloads nginx. at that point,
+        // nginx config of apps has not been re-generated yet and the reload fails. post 6.3, this file can be removed in start.sh
+        // funcs.push(fs.unlink.bind(fs, `${BOX_DATA_DIR}/dhparams.pem`));
+    }
+    const turnSecret = safe.fs.readFileSync(`${BOX_DATA_DIR}/addon-turn-secret`);
+    if (turnSecret) {
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'addon_turn_secret', turnSecret ]));
+        funcs.push(fs.unlink.bind(fs, `${BOX_DATA_DIR}/addon-turn-secret`));
+    }
+
+    // sftp keys get moved to platformdata in start.sh
+    const sftpPublicKey = safe.fs.readFileSync(`${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key.pub`);
+    const sftpPrivateKey = safe.fs.readFileSync(`${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key`);
+    if (sftpPublicKey) {
+        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/sftp/ssh/ssh_host_rsa_key.pub`, sftpPublicKey);
+        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/sftp/ssh/ssh_host_rsa_key`, sftpPrivateKey);
+        safe.fs.chmodSync(`${PLATFORM_DATA_DIR}/sftp/ssh/ssh_host_rsa_key`, 0o600);
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'sftp_public_key', sftpPublicKey ]));
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'sftp_private_key', sftpPrivateKey ]));
+        funcs.push(fs.rmdir.bind(fs, `${BOX_DATA_DIR}/sftp`, { recursive: true }));
+    }
+
+    async.series(funcs, callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210504221634-settings-migrate-firewall.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    safe = require('safetydance');
+
+const BOX_DATA_DIR = '/home/yellowtent/boxdata';
+const PLATFORM_DATA_DIR = '/home/yellowtent/platformdata';
+
+exports.up = function (db, callback) {
+    if (!fs.existsSync(`${BOX_DATA_DIR}/firewall`)) return callback();
+
+    const ports = safe.fs.readFileSync(`${BOX_DATA_DIR}/firewall/ports.json`);
+    if (ports) {
+        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/firewall/ports.json`, ports);
+    }
+
+    const blocklist = safe.fs.readFileSync(`${BOX_DATA_DIR}/firewall/blocklist.txt`);
+    async.series([
+        (next) => {
+            if (!blocklist) return next();
+            db.runSql('INSERT INTO settings (name, valueBlob) VALUES (?, ?)', [ 'firewall_blocklist', blocklist ], next);
+        },
+        fs.writeFile.bind(fs, `${PLATFORM_DATA_DIR}/firewall/blocklist.txt`, blocklist || ''),
+        fs.rmdir.bind(fs, `${BOX_DATA_DIR}/firewall`, { recursive: true })
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210504230054-domains-add-fallbackCertificateJson.js

@@ -0,0 +1,38 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+const CERTS_DIR = '/home/yellowtent/boxdata/certs',
+    PLATFORM_CERTS_DIR = '/home/yellowtent/platformdata/nginx/cert';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE domains ADD COLUMN fallbackCertificateJson MEDIUMTEXT', function (error) {
+        if (error) return callback(error);
+
+        db.all('SELECT * FROM domains', [ ], function (error, domains) {
+            if (error) return callback(error);
+
+            async.eachSeries(domains, function (domain, iteratorDone) {
+                // b94dbf5fa33a6d68d784571721ff44348c2d88aa seems to have moved certs from platformdata to boxdata
+                let cert = safe.fs.readFileSync(`${CERTS_DIR}/${domain.domain}.host.cert`, 'utf8');
+                let key = safe.fs.readFileSync(`${CERTS_DIR}/${domain.domain}.host.key`, 'utf8');
+
+                if (!cert) {
+                    cert = safe.fs.readFileSync(`${PLATFORM_CERTS_DIR}/${domain.domain}.host.cert`, 'utf8');
+                    key = safe.fs.readFileSync(`${PLATFORM_CERTS_DIR}/${domain.domain}.host.key`, 'utf8');
+                }
+
+                const fallbackCertificate = { cert, key };
+
+                db.runSql('UPDATE domains SET fallbackCertificateJson=? WHERE domain=?', [ JSON.stringify(fallbackCertificate), domain.domain ], iteratorDone);
+            }, callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.run(db, 'ALTER TABLE domains DROP COLUMN fallbackCertificateJson')
+    ], callback);
+};

migrations/20210505165936-subdomains-add-certificateJson.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    safe = require('safetydance');
+
+const CERTS_DIR = '/home/yellowtent/boxdata/certs';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains ADD COLUMN certificateJson MEDIUMTEXT', function (error) {
+        if (error) return callback(error);
+
+        db.all('SELECT * FROM subdomains', [ ], function (error, subdomains) {
+            if (error) return callback(error);
+
+            async.eachSeries(subdomains, function (subdomain, iteratorDone) {
+                const cert = safe.fs.readFileSync(`${CERTS_DIR}/${subdomain.subdomain}.${subdomain.domain}.user.cert`, 'utf8');
+                const key = safe.fs.readFileSync(`${CERTS_DIR}/${subdomain.subdomain}.${subdomain.domain}.user.key`, 'utf8');
+
+                if (!cert || !key) return iteratorDone();
+
+                const certificate = { cert, key };
+
+                db.runSql('UPDATE subdomains SET certificateJson=? WHERE domain=? AND subdomain=?', [ JSON.stringify(certificate), subdomain.domain, subdomain.subdomain ], iteratorDone);
+            }, callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.run(db, 'ALTER TABLE subdomains DROP COLUMN certificateJson')
+    ], callback);
+};

migrations/20210505223829-blobs-migrate-certs.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const async = require('async'),
+    child_process = require('child_process'),
+    fs = require('fs'),
+    path = require('path'),
+    safe = require('safetydance');
+
+const OLD_CERTS_DIR = '/home/yellowtent/boxdata/certs';
+const NEW_CERTS_DIR = '/home/yellowtent/platformdata/nginx/cert';
+
+exports.up = function(db, callback) {
+    fs.readdir(OLD_CERTS_DIR, function (error, filenames) {
+        if (error && error.code === 'ENOENT') return callback();
+        if (error) return callback(error);
+
+        filenames = filenames.filter(f => f.endsWith('.key') && !f.endsWith('.host.key') && !f.endsWith('.user.key')); // ignore fallback and user keys
+
+        async.eachSeries(filenames, function (filename, iteratorCallback) {
+            const privateKeyFile = filename;
+            const privateKey = fs.readFileSync(path.join(OLD_CERTS_DIR, filename));
+            const certificateFile = filename.replace(/\.key$/, '.cert');
+            const certificate = safe.fs.readFileSync(path.join(OLD_CERTS_DIR, certificateFile));
+            if (!certificate) {
+                console.log(`${certificateFile} is missing. skipping migration`);
+                return iteratorCallback();
+            }
+            const csrFile = filename.replace(/\.key$/, '.csr');
+            const csr = safe.fs.readFileSync(path.join(OLD_CERTS_DIR, csrFile));
+            if (!csr) {
+                console.log(`${csrFile} is missing. skipping migration`);
+                return iteratorCallback();
+            }
+
+            async.series([
+                db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', `cert-${privateKeyFile}`, privateKey),
+                db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', `cert-${certificateFile}`, certificate),
+                db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', `cert-${csrFile}`, csr),
+            ], iteratorCallback);
+        }, function (error) {
+            if (error) return callback(error);
+
+            child_process.execSync(`cp ${OLD_CERTS_DIR}/* ${NEW_CERTS_DIR}`); // this way we copy the non-migrated ones like .host, .user etc as well
+            fs.rmdir(OLD_CERTS_DIR, { recursive: true }, callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
+

migrations/20210512210953-volumes-add-mountType-and-mountOptionsJson.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE volumes ADD COLUMN mountType VARCHAR(16) DEFAULT "noop"'),
+        db.runSql.bind(db, 'ALTER TABLE volumes ADD COLUMN mountOptionsJson TEXT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE volumes DROP COLUMN mountType'),
+        db.runSql.bind(db, 'ALTER TABLE volumes DROP COLUMN mountOptionsJson')
+    ], callback);
+};

migrations/20210517135636-add-order-by-indexes.js

@@ -0,0 +1,21 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE backups ADD INDEX creationTime_index (creationTime)'),
+        db.runSql.bind(db, 'ALTER TABLE eventlog ADD INDEX creationTime_index (creationTime)'),
+        db.runSql.bind(db, 'ALTER TABLE notifications ADD INDEX creationTime_index (creationTime)'),
+        db.runSql.bind(db, 'ALTER TABLE tasks ADD INDEX creationTime_index (creationTime)'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE backups DROP INDEX creationTime_index'),
+        db.runSql.bind(db, 'ALTER TABLE eventlog DROP INDEX creationTime_index'),
+        db.runSql.bind(db, 'ALTER TABLE notifications DROP INDEX creationTime_index'),
+        db.runSql.bind(db, 'ALTER TABLE tasks DROP INDEX creationTime_index'),
+    ], callback);
+};

migrations/20210517141957-users-rename-createdAt-to-creationTime.js

@@ -0,0 +1,33 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
+        if (error) return callback(error);
+
+        db.runSql('ALTER TABLE users ADD INDEX creationTime_index (creationTime)', function (error) {
+            if (error) return callback(error);
+
+            db.all('SELECT id, createdAt FROM users', function (error, results) {
+                if (error) return callback(error);
+
+                async.eachSeries(results, function (r, iteratorDone) {
+                    const creationTime = new Date(r.createdAt);
+                    db.runSql('UPDATE users SET creationTime=? WHERE id=?', [ creationTime, r.id ], iteratorDone);
+                }, function (error) {
+                    if (error) return callback(error);
+
+                    db.runSql('ALTER TABLE users DROP COLUMN createdAt', callback);
+                });
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN creationTime', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210517194116-backups-provider-mountpoint.js

@@ -0,0 +1,27 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
+        if (error || results.length === 0) return callback(error);
+
+        const backupConfig = JSON.parse(results[0].value);
+        if (backupConfig.provider === 'sshfs' || backupConfig.provider === 'cifs' || backupConfig.provider === 'nfs' || backupConfig.externalDisk) {
+            backupConfig.chown = backupConfig.provider === 'nfs' || backupConfig.provider === 'sshfs' || backupConfig.externalDisk;
+            backupConfig.preserveAttributes = !!backupConfig.externalDisk;
+            backupConfig.provider = 'mountpoint';
+            if (backupConfig.externalDisk) {
+                backupConfig.mountPoint = backupConfig.backupFolder;
+                backupConfig.prefix = '';
+                delete backupConfig.backupFolder;
+                delete backupConfig.externalDisk;
+            }
+            db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [JSON.stringify(backupConfig)], callback);
+        } else {
+            callback();
+        }
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210528205138-notifications-drop-userId.js

@@ -0,0 +1,13 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE notifications DROP COLUMN userId', function (error) {
+        if (error) return callback(error);
+
+        db.runSql('DELETE FROM notifications', callback); // just clear notifications table
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE notifications ADD COLUMN userId VARCHAR(128) NOT NULL', callback);
+};

migrations/20210625061443-volumes-change-noop-to-mountpoint.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM volumes', function (error, volumes) {
+        if (error || volumes.length === 0) return callback(error);
+
+        async.eachSeries(volumes, function (volume, iteratorDone) {
+            if (volume.mountType !== 'noop') return iteratorDone();
+
+            let mountType;
+            if (safe.child_process.execSync(`mountpoint -q -- ${volume.hostPath}`)) {
+                mountType = 'mountpoint';
+            } else {
+                mountType = 'filesystem';
+            }
+            db.runSql('UPDATE volumes SET mountType=? WHERE id=?', [ mountType, volume.id ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/schema.sql

@@ -6,7 +6,7 @@
 #### Strict mode is enabled
 #### VARCHAR - stored as part of table row (use for strings)
 #### TEXT - stored offline from table row (use for strings)
-#### BLOB - stored offline from table row (use for binary data)
+#### BLOB (64KB), MEDIUMBLOB (16MB), LONGBLOB (4GB) - stored offline from table row (use for binary data)
 #### https://dev.mysql.com/doc/refman/5.0/en/storage-requirements.html
 #### Times are stored in the database in UTC. And precision is seconds
 
@@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS users(
     email VARCHAR(254) NOT NULL UNIQUE,
     password VARCHAR(1024) NOT NULL,
     salt VARCHAR(512) NOT NULL,
-    createdAt VARCHAR(512) NOT NULL,
+    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     displayName VARCHAR(512) DEFAULT "",
     fallbackEmail VARCHAR(512) DEFAULT "",
@@ -31,7 +31,10 @@ CREATE TABLE IF NOT EXISTS users(
     resetToken VARCHAR(128) DEFAULT "",
     resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     active BOOLEAN DEFAULT 1,
+    avatar MEDIUMBLOB,
+    locationJson TEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
 
+    INDEX creationTime_index (creationTime),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS userGroups(
@@ -55,6 +58,7 @@ CREATE TABLE IF NOT EXISTS tokens(
     clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
     expires BIGINT NOT NULL, // FIXME: make this a timestamp
+    lastUsedTime TIMESTAMP NULL,
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS apps(
@@ -78,6 +82,7 @@ CREATE TABLE IF NOT EXISTS apps(
     reverseProxyConfigJson TEXT, // { robotsTxt, csp }
     enableBackup BOOLEAN DEFAULT 1, // misnomer: controls automatic daily backups
     enableAutomaticUpdate BOOLEAN DEFAULT 1,
+    enableMailbox BOOLEAN DEFAULT 1, // whether sendmail addon is enabled
     mailboxName VARCHAR(128), // mailbox of this app
     mailboxDomain VARCHAR(128), // mailbox domain of this apps
     label VARCHAR(128),     // display name
@@ -87,6 +92,8 @@ CREATE TABLE IF NOT EXISTS apps(
     errorJson TEXT,
     servicesConfigJson TEXT, // app services configuration
     containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair'
+    appStoreIcon MEDIUMBLOB,
+    icon MEDIUMBLOB,
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
@@ -103,6 +110,7 @@ CREATE TABLE IF NOT EXISTS appPortBindings(
 CREATE TABLE IF NOT EXISTS settings(
     name VARCHAR(128) NOT NULL UNIQUE,
     value TEXT,
+    valueBlob MEDIUMBLOB,
     PRIMARY KEY(name));
 
 CREATE TABLE IF NOT EXISTS appAddonConfigs(
@@ -131,6 +139,7 @@ CREATE TABLE IF NOT EXISTS backups(
     format VARCHAR(16) DEFAULT "tgz",
     preserveSecs INTEGER DEFAULT 0,
 
+    INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
 CREATE TABLE IF NOT EXISTS eventlog(
@@ -140,6 +149,7 @@ CREATE TABLE IF NOT EXISTS eventlog(
     data TEXT, /* free flowing json based on action */
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
 
+    INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
 CREATE TABLE IF NOT EXISTS domains(
@@ -150,6 +160,8 @@ CREATE TABLE IF NOT EXISTS domains(
     tlsConfigJson TEXT, /* JSON containing the tls provider config */
     wellKnownJson TEXT, /* JSON containing well known docs for this domain */
 
+    fallbackCertificateJson MEDIUMTEXT,
+
     PRIMARY KEY (domain))
 
     /* the default db collation is utf8mb4_unicode_ci but for the app table domain constraint we have to use the old one */
@@ -189,6 +201,7 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     membersOnly BOOLEAN DEFAULT false,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     domain VARCHAR(128),
+    active BOOLEAN DEFAULT 1,
 
     FOREIGN KEY(domain) REFERENCES mail(domain),
     FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
@@ -200,6 +213,8 @@ CREATE TABLE IF NOT EXISTS subdomains(
     subdomain VARCHAR(128) NOT NULL,
     type VARCHAR(128) NOT NULL, /* primary or redirect */
 
+    certificateJson MEDIUMTEXT,
+
     FOREIGN KEY(domain) REFERENCES domains(domain),
     FOREIGN KEY(appId) REFERENCES apps(id),
     UNIQUE (subdomain, domain));
@@ -213,17 +228,20 @@ CREATE TABLE IF NOT EXISTS tasks(
     resultJson TEXT,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+    INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
 CREATE TABLE IF NOT EXISTS notifications(
     id int NOT NULL AUTO_INCREMENT,
-    userId VARCHAR(128) NOT NULL,
     eventId VARCHAR(128), // reference to eventlog. can be null
     title VARCHAR(512) NOT NULL,
     message TEXT,
     acknowledged BOOLEAN DEFAULT false,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    UNIQUE KEY appPasswords_name_appId_identifier (name, userId, identifier),
+
+    INDEX creationTime_index (creationTime),
+    FOREIGN KEY(eventId) REFERENCES eventlog(id),
     PRIMARY KEY (id)
 );
 
@@ -234,6 +252,7 @@ CREATE TABLE IF NOT EXISTS appPasswords(
     identifier VARCHAR(128) NOT NULL, // resourceId: app id or mail or webadmin
     hashedPassword VARCHAR(1024) NOT NULL,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE KEY appPasswords_name_appId_identifier (name, userId, identifier)
     FOREIGN KEY(userId) REFERENCES users(id),
 
     PRIMARY KEY (id)
@@ -244,6 +263,8 @@ CREATE TABLE IF NOT EXISTS volumes(
     name VARCHAR(256) NOT NULL UNIQUE,
     hostPath VARCHAR(1024) NOT NULL UNIQUE,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    mountType VARCHAR(16) DEFAULT "noop",
+    mountOptionsJson TEXT,
     PRIMARY KEY (id)
 );
 
@@ -255,4 +276,9 @@ CREATE TABLE IF NOT EXISTS appMounts(
     FOREIGN KEY(appId) REFERENCES apps(id),
     FOREIGN KEY(volumeId) REFERENCES volumes(id));
 
+CREATE TABLE IF NOT EXISTS blobs(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    value TEXT,
+    PRIMARY KEY(id));
+
 CHARACTER SET utf8 COLLATE utf8_bin;

package-lock.json

@@ -54,9 +54,9 @@
       "integrity": "sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw=="
     },
     "@google-cloud/storage": {
-      "version": "5.8.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.8.0.tgz",
-      "integrity": "sha512-WOShvBPOfkDXUzXMO+3j8Bzus+PFI9r1Ey9dLG2Zf458/PVuFTtaRWntd9ZiDG8g90zl2LmnA1JkDCreGUKr5g==",
+      "version": "5.8.5",
+      "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.8.5.tgz",
+      "integrity": "sha512-i0gB9CRwQeOBYP7xuvn14M40LhHCwMjceBjxE4CTvsqL519sVY5yVKxLiAedHWGwUZHJNRa7Q2CmNfkdRwVNPg==",
       "requires": {
         "@google-cloud/common": "^3.6.0",
         "@google-cloud/paginator": "^3.0.0",
@@ -64,11 +64,11 @@
         "arrify": "^2.0.0",
         "async-retry": "^1.3.1",
         "compressible": "^2.0.12",
-        "date-and-time": "^0.14.2",
+        "date-and-time": "^1.0.0",
         "duplexify": "^4.0.0",
         "extend": "^3.0.2",
         "gaxios": "^4.0.0",
-        "gcs-resumable-upload": "^3.1.3",
+        "gcs-resumable-upload": "^3.1.4",
         "get-stream": "^6.0.0",
         "hash-stream-validation": "^0.2.2",
         "mime": "^2.2.0",
@@ -98,14 +98,14 @@
           }
         },
         "get-stream": {
-          "version": "6.0.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.0.tgz",
-          "integrity": "sha512-A1B3Bh1UmL0bidM/YX2NsCOTnGJePL9rO/M+Mw3m9f2gUpfokS0hi5Eah0WSUEWZdZhIZtMjkIYS7mDfOqNHbg=="
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
+          "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg=="
         },
         "google-auth-library": {
-          "version": "7.0.2",
-          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.0.2.tgz",
-          "integrity": "sha512-vjyNZR3pDLC0u7GHLfj+Hw9tGprrJwoMwkYGqURCXYITjCrP9HprOyxVV+KekdLgATtWGuDkQG2MTh0qpUPUgg==",
+          "version": "7.0.4",
+          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.0.4.tgz",
+          "integrity": "sha512-o8irYyeijEiecTXeoEe8UKNEzV1X+uhR4b2oNdapDMZixypp0J+eHimGOyx5Joa3UAeokGngdtDLXtq9vDqG2Q==",
           "requires": {
             "arrify": "^2.0.0",
             "base64-js": "^1.3.0",
@@ -232,9 +232,9 @@
       }
     },
     "anymatch": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
-      "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==",
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz",
+      "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==",
       "dev": true,
       "requires": {
         "normalize-path": "^3.0.0",
@@ -316,9 +316,9 @@
       "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
     },
     "aws-sdk": {
-      "version": "2.850.0",
-      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.850.0.tgz",
-      "integrity": "sha512-RqPeSKe1JlhTUL9+xUsp771ZtMY7JICoQEnFJuJ+JVqGyILhg95L4t8S5KnznUfWYc0pcpTiHKLmPteHyHS3pw==",
+      "version": "2.906.0",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.906.0.tgz",
+      "integrity": "sha512-u/kmVILew/9HFpHwVrc3VMK24m+XrazXEooMxkzbWXEBvtVm1xTYv8xPmdgiYvogWIkWTkeIF9ME4LBeHenYkw==",
       "requires": {
         "buffer": "4.9.2",
         "events": "1.1.1",
@@ -359,7 +359,7 @@
     },
     "backoff": {
       "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/backoff/-/backoff-2.5.0.tgz",
+      "resolved": false,
       "integrity": "sha1-9hbtqdPktmuMp/ynn2lXIsX44m8=",
       "requires": {
         "precond": "0.2"
@@ -574,6 +574,17 @@
         "is-glob": "~4.0.1",
         "normalize-path": "~3.0.0",
         "readdirp": "~3.5.0"
+      },
+      "dependencies": {
+        "readdirp": {
+          "version": "3.5.0",
+          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz",
+          "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==",
+          "dev": true,
+          "requires": {
+            "picomatch": "^2.2.1"
+          }
+        }
       }
     },
     "chownr": {
@@ -592,49 +603,16 @@
       }
     },
     "cloudron-manifestformat": {
-      "version": "5.10.1",
-      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.10.1.tgz",
-      "integrity": "sha512-NI4wkf3rioeoJZQJKtpBHH3gGvyIBpw7sOvU2lFPMlsWWb6nXvGYJWCJkZ/s7Sdm937LGE6ZkBcNRNfcmxVMIg==",
+      "version": "5.10.2",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.10.2.tgz",
+      "integrity": "sha512-RU+uwwdPDXJ+zoORBVCkA6+mDEgyYlqtHjZWnvomdcemfHnxsE54A1r7+spm2Tz3tS1aVUQN7Vd5l1PD21PUzg==",
       "requires": {
         "cron": "^1.8.2",
         "java-packagename-regex": "^1.0.0",
-        "safetydance": "1.0.0",
-        "semver": "^7.1.3",
+        "safetydance": "2.0.1",
+        "semver": "^7.3.5",
         "tv4": "^1.3.0",
-        "validator": "^12.2.0"
-      },
-      "dependencies": {
-        "lru-cache": {
-          "version": "6.0.0",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
-          "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
-          "requires": {
-            "yallist": "^4.0.0"
-          }
-        },
-        "safetydance": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-1.0.0.tgz",
-          "integrity": "sha512-ji9T/p5poiGgx4N3OFORGChAS9L8YL8S0/RpYvEPmQucCPrzmQMfdzbjFG/4+0BhJUvNA19KZUVj3YE8gkLpjA=="
-        },
-        "semver": {
-          "version": "7.3.4",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
-          "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
-          "requires": {
-            "lru-cache": "^6.0.0"
-          }
-        },
-        "validator": {
-          "version": "12.2.0",
-          "resolved": "https://registry.npmjs.org/validator/-/validator-12.2.0.tgz",
-          "integrity": "sha512-jJfE/DW6tIK1Ek8nCfNFqt8Wb3nzMoAbocBF6/Icgg1ZFSBpObdnwVY2jQj6qUqzhx5jc71fpvBWyLGO7Xl+nQ=="
-        },
-        "yallist": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
-        }
+        "validator": "^13.6.0"
       }
     },
     "code-point-at": {
@@ -727,11 +705,11 @@
       }
     },
     "connect-lastmile": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-2.0.0.tgz",
-      "integrity": "sha512-kMQlR0OVtV9x/p/8m/DhfVyB/7PnS88vg/OmHLRuPxnOhTQdsnZvovg7OiApqAaDnUKUSnevWYd9oEDFV2Bh3w==",
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-2.1.1.tgz",
+      "integrity": "sha512-723vDmZuy6KBUAmuXff1mb+l9ZMs+JqXJuAGHgWNI3fNYAu9DKXC+GYdxqY0+9oMXyVJNf5AscoONcq9Nqb0Ig==",
       "requires": {
-        "underscore": "^1.9.1"
+        "underscore": "^1.13.1"
       }
     },
     "connect-timeout": {
@@ -908,9 +886,9 @@
       }
     },
     "date-and-time": {
-      "version": "0.14.2",
-      "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz",
-      "integrity": "sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA=="
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-1.0.0.tgz",
+      "integrity": "sha512-477D7ypIiqlXBkxhU7YtG9wWZJEQ+RUpujt2quTfgf4+E8g5fNUkB0QIL0bVyP5/TKBg8y55Hfa1R/c4bt3dEw=="
     },
     "db-migrate": {
       "version": "0.11.12",
@@ -1059,6 +1037,13 @@
             "which-module": "^2.0.0",
             "y18n": "^4.0.0",
             "yargs-parser": "^18.1.2"
+          },
+          "dependencies": {
+            "y18n": {
+              "version": "4.0.3",
+              "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz",
+              "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ=="
+            }
           }
         },
         "yargs-parser": {
@@ -1119,6 +1104,11 @@
       "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
       "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
     },
+    "delay": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
+      "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw=="
+    },
     "delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -1157,9 +1147,9 @@
       "integrity": "sha512-8hkrtLbVNqCgnRQv8jjit8MnGzqYBouxoP/WMAObbN0aPr43hy/Ml+VxMXKC75lRz2DEwUFN2SNpVnrrQWobew=="
     },
     "docker-modem": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-2.1.4.tgz",
-      "integrity": "sha512-vDTzZjjO1sXMY7m0xKjGdFMMZL7vIUerkC3G4l6rnrpOET2M6AOufM8ajmQoOB+6RfSn6I/dlikCUq/Y91Q1sQ==",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-3.0.0.tgz",
+      "integrity": "sha512-WwFajJ8I5geZ/dDZ5FDMDA6TBkWa76xWwGIGw8uzUjNUGCN0to83wJ8Oi1AxrJTC0JBn+7fvIxUctnawtlwXeg==",
       "requires": {
         "debug": "^4.1.1",
         "readable-stream": "^3.5.0",
@@ -1198,11 +1188,11 @@
       }
     },
     "dockerode": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-3.2.1.tgz",
-      "integrity": "sha512-XsSVB5Wu5HWMg1aelV5hFSqFJaKS5x1aiV/+sT7YOzOq1IRl49I/UwV8Pe4x6t0iF9kiGkWu5jwfvbkcFVupBw==",
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-3.3.0.tgz",
+      "integrity": "sha512-St08lfOjpYCOXEM8XA0VLu3B3hRjtddODphNW5GFoA0AS3JHgoPQKOz0Qmdzg3P+hUPxhb02g1o1Cu1G+U3lRg==",
       "requires": {
-        "docker-modem": "^2.1.0",
+        "docker-modem": "^3.0.0",
         "tar-fs": "~2.0.1"
       },
       "dependencies": {
@@ -1347,9 +1337,9 @@
       "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0="
     },
     "env-paths": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.0.tgz",
-      "integrity": "sha512-6u0VYSCo/OW6IoD5WCLLy9JUGARbamfSavcNXry/eu8aHVFei6CD3Sw+VGX5alea1i9pgPHW0mbu6Xj0uBh7gA==",
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",
+      "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==",
       "dev": true
     },
     "error-ex": {
@@ -1747,9 +1737,9 @@
       }
     },
     "gcs-resumable-upload": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-3.1.3.tgz",
-      "integrity": "sha512-LjVrv6YVH0XqBr/iBW0JgRA1ndxhK6zfEFFJR4im51QVTj/4sInOXimY2evDZuSZ75D3bHxTaQAdXRukMc1y+w==",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-3.1.4.tgz",
+      "integrity": "sha512-5dyDfHrrVcIskiw/cPssVD4HRiwoHjhk1Nd6h5W3pQ/qffDvhfy4oNCr1f3ZXFPwTnxkCbibsB+73oOM+NvmJQ==",
       "requires": {
         "abort-controller": "^3.0.0",
         "configstore": "^5.0.0",
@@ -1761,9 +1751,9 @@
       },
       "dependencies": {
         "google-auth-library": {
-          "version": "7.0.2",
-          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.0.2.tgz",
-          "integrity": "sha512-vjyNZR3pDLC0u7GHLfj+Hw9tGprrJwoMwkYGqURCXYITjCrP9HprOyxVV+KekdLgATtWGuDkQG2MTh0qpUPUgg==",
+          "version": "7.0.4",
+          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.0.4.tgz",
+          "integrity": "sha512-o8irYyeijEiecTXeoEe8UKNEzV1X+uhR4b2oNdapDMZixypp0J+eHimGOyx5Joa3UAeokGngdtDLXtq9vDqG2Q==",
           "requires": {
             "arrify": "^2.0.0",
             "base64-js": "^1.3.0",
@@ -1843,9 +1833,9 @@
       }
     },
     "glob-parent": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
-      "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
       "dev": true,
       "requires": {
         "is-glob": "^4.0.1"
@@ -1994,9 +1984,9 @@
       }
     },
     "hosted-git-info": {
-      "version": "2.8.8",
-      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz",
-      "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
+      "version": "2.8.9",
+      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
+      "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==",
       "dev": true
     },
     "http-errors": {
@@ -2253,9 +2243,9 @@
       "dev": true
     },
     "js-yaml": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz",
-      "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
       "requires": {
         "argparse": "^2.0.1"
       }
@@ -2275,9 +2265,9 @@
       "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
     },
     "json": {
-      "version": "10.0.0",
-      "resolved": "https://registry.npmjs.org/json/-/json-10.0.0.tgz",
-      "integrity": "sha512-iK7tAZtpoghibjdB1ncCWykeBMmke3JThUe+rnkD4qkZaglOIQ70Pw7r5UJ4lyUT+7gnw7ehmmLUHDuhqzQD+g=="
+      "version": "11.0.0",
+      "resolved": "https://registry.npmjs.org/json/-/json-11.0.0.tgz",
+      "integrity": "sha512-N/ITv3Yw9Za8cGxuQqSqrq6RHnlaHWZkAFavcfpH/R52522c26EbihMxnY7A1chxfXJ4d+cEFIsyTgfi9GihrA=="
     },
     "json-bigint": {
       "version": "1.0.0",
@@ -2521,9 +2511,9 @@
           }
         },
         "chalk": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
-          "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
+          "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
           "dev": true,
           "requires": {
             "ansi-styles": "^4.1.0",
@@ -2719,11 +2709,18 @@
       }
     },
     "mkdirp": {
-      "version": "0.5.1",
-      "resolved": false,
-      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+      "version": "0.5.5",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+      "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
       "requires": {
-        "minimist": "0.0.8"
+        "minimist": "^1.2.5"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "1.2.5",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+          "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
+        }
       }
     },
     "mkdirp-classic": {
@@ -2732,9 +2729,9 @@
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
     },
     "mocha": {
-      "version": "8.3.0",
-      "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.3.0.tgz",
-      "integrity": "sha512-TQqyC89V1J/Vxx0DhJIXlq9gbbL9XFNdeLQ1+JsnZsVaSOV1z3tWfw0qZmQJGQRIfkvZcs7snQnZnOCKoldq1Q==",
+      "version": "8.4.0",
+      "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz",
+      "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==",
       "dev": true,
       "requires": {
         "@ungap/promise-all-settled": "1.1.2",
@@ -2833,6 +2830,15 @@
           "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
           "dev": true
         },
+        "js-yaml": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz",
+          "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==",
+          "dev": true,
+          "requires": {
+            "argparse": "^2.0.1"
+          }
+        },
         "locate-path": {
           "version": "6.0.0",
           "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@@ -2873,9 +2879,9 @@
           "dev": true
         },
         "string-width": {
-          "version": "4.2.0",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
-          "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==",
+          "version": "4.2.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+          "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
           "dev": true,
           "requires": {
             "emoji-regex": "^8.0.0",
@@ -2910,9 +2916,9 @@
           }
         },
         "y18n": {
-          "version": "5.0.5",
-          "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz",
-          "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==",
+          "version": "5.0.8",
+          "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
+          "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
           "dev": true
         },
         "yargs": {
@@ -3112,9 +3118,9 @@
       "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
     },
     "nock": {
-      "version": "13.0.7",
-      "resolved": "https://registry.npmjs.org/nock/-/nock-13.0.7.tgz",
-      "integrity": "sha512-WBz73VYIjdbO6BwmXODRQLtn7B5tldA9pNpWJe5QTtTEscQlY5KXU4srnGzBOK2fWakkXj69gfTnXGzmrsaRWw==",
+      "version": "13.0.11",
+      "resolved": "https://registry.npmjs.org/nock/-/nock-13.0.11.tgz",
+      "integrity": "sha512-sKZltNkkWblkqqPAsjYW0bm3s9DcHRPiMOyKO/PkfJ+ANHZ2+LA2PLe22r4lLrKgXaiSaDQwW3qGsJFtIpQIeQ==",
       "dev": true,
       "requires": {
         "debug": "^4.1.0",
@@ -3157,9 +3163,9 @@
       }
     },
     "node-sass": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-5.0.0.tgz",
-      "integrity": "sha512-opNgmlu83ZCF792U281Ry7tak9IbVC+AKnXGovcQ8LG8wFaJv6cLnRlc6DIHlmNxWEexB5bZxi9SZ9JyUuOYjw==",
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-6.0.0.tgz",
+      "integrity": "sha512-GDzDmNgWNc9GNzTcSLTi6DU6mzSPupVJoStIi7cF3GjwSE9q1cVakbvAAVSt59vzUjV9JJoSZFKoo9krbjKd2g==",
       "dev": true,
       "requires": {
         "async-foreach": "^0.1.3",
@@ -3223,9 +3229,9 @@
       }
     },
     "nodemailer": {
-      "version": "6.4.18",
-      "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.4.18.tgz",
-      "integrity": "sha512-ht9cXxQ+lTC+t00vkSIpKHIyM4aXIsQ1tcbQCn5IOnxYHi81W2XOaU66EQBFFpbtzLEBTC94gmkbD4mGZQzVpA=="
+      "version": "6.6.0",
+      "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.6.0.tgz",
+      "integrity": "sha512-ikSMDU1nZqpo2WUPE0wTTw/NGGImTkwpJKDIFPZT+YvvR9Sj+ze5wzu95JHkBMglQLoG2ITxU21WukCC/XsFkg=="
     },
     "nodemailer-fetch": {
       "version": "1.6.0",
@@ -3487,7 +3493,7 @@
     },
     "precond": {
       "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/precond/-/precond-0.2.3.tgz",
+      "resolved": false,
       "integrity": "sha1-qpWRvKokkj8eD0hJ0kD0fvwQdaw="
     },
     "pretty-bytes": {
@@ -3741,9 +3747,9 @@
       }
     },
     "readdirp": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz",
-      "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==",
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
+      "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==",
       "requires": {
         "picomatch": "^2.2.1"
       }
@@ -3885,9 +3891,9 @@
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
     "safetydance": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-1.1.1.tgz",
-      "integrity": "sha512-SwSS/lCMCGBZ33j9DV5cXvrJOKaeKr5UFI4/d5r/sISYvblPN58bwN0DbTsdYYH0a5f4BSuI7zr+6TLMXDxKoQ=="
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-2.0.1.tgz",
+      "integrity": "sha512-RuMpDOXn4bC+cIrzeZ6ZJR8/aaa+58KztATWO8KbEpfC4LRaYskn+Ll3H5KMikH1N1F47S+razKDqZklUkRkTg=="
     },
     "sass-graph": {
       "version": "2.2.5",
@@ -3947,9 +3953,9 @@
       }
     },
     "semver": {
-      "version": "7.3.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
-      "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
+      "version": "7.3.5",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+      "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
       "requires": {
         "lru-cache": "^6.0.0"
       },
@@ -4059,43 +4065,6 @@
       "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
       "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
     },
-    "showdown": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/showdown/-/showdown-1.9.1.tgz",
-      "integrity": "sha512-9cGuS382HcvExtf5AHk7Cb4pAeQQ+h0eTr33V1mu+crYWV4KvWAw6el92bDrqGEk5d46Ai/fhbEUwqJ/mTCNEA==",
-      "requires": {
-        "yargs": "^14.2"
-      },
-      "dependencies": {
-        "yargs": {
-          "version": "14.2.2",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.2.tgz",
-          "integrity": "sha512-/4ld+4VV5RnrynMhPZJ/ZpOCGSCeghMykZ3BhdFBDa9Wy/RH6uEGNWDJog+aUlq+9OM1CFTgtYRW5Is1Po9NOA==",
-          "requires": {
-            "cliui": "^5.0.0",
-            "decamelize": "^1.2.0",
-            "find-up": "^3.0.0",
-            "get-caller-file": "^2.0.1",
-            "require-directory": "^2.1.1",
-            "require-main-filename": "^2.0.0",
-            "set-blocking": "^2.0.0",
-            "string-width": "^3.0.0",
-            "which-module": "^2.0.0",
-            "y18n": "^4.0.0",
-            "yargs-parser": "^15.0.0"
-          }
-        },
-        "yargs-parser": {
-          "version": "15.0.0",
-          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.0.tgz",
-          "integrity": "sha512-xLTUnCMc4JhxrPEPUYD5IBR1mWCK/aT6+RJ/K29JY2y1vD+FhtgKK0AXRWvI262q3QSffAQuTouFIKUuHX89wQ==",
-          "requires": {
-            "camelcase": "^5.0.0",
-            "decamelize": "^1.2.0"
-          }
-        }
-      }
-    },
     "signal-exit": {
       "version": "3.0.2",
       "resolved": false,
@@ -4379,9 +4348,9 @@
           "integrity": "sha512-s8+wktIuDSLffCywiwSxQOMqtPxML11a/dtHE17tMn4B1MSWw/C22EKf7M2KGUBcDaVFEGT+S8N02geDXeuNKg=="
         },
         "minimist": {
-          "version": "1.2.0",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
-          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
+          "version": "1.2.5",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+          "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
         },
         "prettyjson": {
           "version": "1.2.1",
@@ -4651,6 +4620,11 @@
         "is-typedarray": "^1.0.0"
       }
     },
+    "ua-parser-js": {
+      "version": "0.7.28",
+      "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.28.tgz",
+      "integrity": "sha512-6Gurc1n//gjp9eQNXjD9O3M/sMwVtN5S8Lv9bvOYBfKfDNiIIhqiyi01vMBO45u4zkDE420w/e0se7Vs+sIg+g=="
+    },
     "uid-safe": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz",
@@ -4660,9 +4634,9 @@
       }
     },
     "underscore": {
-      "version": "1.12.0",
-      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.0.tgz",
-      "integrity": "sha512-21rQzss/XPMjolTiIezSu3JAjgagXKROtNrYFEOWK109qY1Uv2tVjPTZ1ci2HgvQDA16gHYSthQIJfB+XId/rQ=="
+      "version": "1.13.1",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz",
+      "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g=="
     },
     "unique-string": {
       "version": "2.0.0",
@@ -4772,9 +4746,9 @@
       }
     },
     "validator": {
-      "version": "13.5.2",
-      "resolved": "https://registry.npmjs.org/validator/-/validator-13.5.2.tgz",
-      "integrity": "sha512-mD45p0rvHVBlY2Zuy3F3ESIe1h5X58GPfAtslBjY7EtTqGquZTj+VX/J4RnHWN8FKq0C9WRVt1oWAcytWRuYLQ=="
+      "version": "13.6.0",
+      "resolved": "https://registry.npmjs.org/validator/-/validator-13.6.0.tgz",
+      "integrity": "sha512-gVgKbdbHgtxpRyR8K0O6oFZPhhB5tT1jeEHZR0Znr9Svg03U0+r9DXWMrnRAB+HtCStDQKlaIZm42tVsVjqtjg=="
     },
     "vary": {
       "version": "1.1.2",
@@ -4916,9 +4890,9 @@
       }
     },
     "ws": {
-      "version": "7.4.3",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.3.tgz",
-      "integrity": "sha512-hr6vCR76GsossIRsr8OLR9acVVm1jyfEWvhbNjtgPOrfvAlKzvyeg/P6r8RuDjRyrcQoPQT7K0DGEPc7Ae6jzA=="
+      "version": "7.4.5",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.5.tgz",
+      "integrity": "sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g=="
     },
     "xdg-basedir": {
       "version": "4.0.0",
@@ -4958,9 +4932,9 @@
       "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
     },
     "y18n": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
-      "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w=="
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz",
+      "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ=="
     },
     "yallist": {
       "version": "3.1.1",

package.json

@@ -12,15 +12,15 @@
   },
   "dependencies": {
     "@google-cloud/dns": "^2.1.0",
-    "@google-cloud/storage": "^5.8.0",
+    "@google-cloud/storage": "^5.8.5",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
     "async": "^3.2.0",
-    "aws-sdk": "^2.850.0",
+    "aws-sdk": "^2.906.0",
     "basic-auth": "^2.0.1",
     "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.10.1",
+    "cloudron-manifestformat": "^5.10.2",
     "connect": "^3.7.0",
-    "connect-lastmile": "^2.0.0",
+    "connect-lastmile": "^2.1.1",
     "connect-timeout": "^1.9.0",
     "cookie-parser": "^1.4.5",
     "cookie-session": "^1.4.0",
@@ -28,13 +28,14 @@
     "db-migrate": "^0.11.12",
     "db-migrate-mysql": "^2.1.2",
     "debug": "^4.3.1",
-    "dockerode": "^3.2.1",
+    "delay": "^5.0.0",
+    "dockerode": "^3.3.0",
     "ejs": "^3.1.6",
     "ejs-cli": "^2.2.1",
     "express": "^4.17.1",
     "ipaddr.js": "^2.0.0",
-    "js-yaml": "^4.0.0",
-    "json": "^10.0.0",
+    "js-yaml": "^4.1.0",
+    "json": "^11.0.0",
     "jsonwebtoken": "^8.5.1",
     "ldapjs": "^2.2.4",
     "lodash": "^4.17.21",
@@ -46,20 +47,19 @@
     "multiparty": "^4.2.2",
     "mustache-express": "^1.3.0",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.4.18",
+    "nodemailer": "^6.6.0",
     "nodemailer-smtp-transport": "^2.7.4",
     "once": "^1.4.0",
     "pretty-bytes": "^5.6.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
     "qrcode": "^1.4.4",
-    "readdirp": "^3.5.0",
+    "readdirp": "^3.6.0",
     "request": "^2.88.2",
     "rimraf": "^3.0.2",
     "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^1.1.1",
-    "semver": "^7.3.4",
-    "showdown": "^1.9.1",
+    "safetydance": "^2.0.1",
+    "semver": "^7.3.5",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
     "superagent": "^6.1.0",
@@ -67,20 +67,21 @@
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
-    "underscore": "^1.12.0",
+    "ua-parser-js": "^0.7.28",
+    "underscore": "^1.13.1",
     "uuid": "^8.3.2",
-    "validator": "^13.5.2",
-    "ws": "^7.4.3",
+    "validator": "^13.6.0",
+    "ws": "^7.4.5",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.4.1",
     "js2xmlparser": "^4.0.1",
-    "mocha": "^8.3.0",
+    "mocha": "^8.4.0",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^13.0.7",
-    "node-sass": "^5.0.0",
+    "nock": "^13.0.11",
+    "node-sass": "^6.0.0",
     "recursive-readdir": "^2.2.2"
   },
   "scripts": {

runTests

@@ -22,8 +22,8 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com boxdata/sftp/ssh
-mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
+mkdir -p boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update
 sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
 
 # translations
@@ -34,12 +34,14 @@ cp -r ${source_dir}/../dashboard/dist/translation/* box/dashboard/dist/translati
 echo "=> Generating a localhost selfsigned cert"
 openssl req -x509 -newkey rsa:2048 -keyout platformdata/nginx/cert/host.key -out platformdata/nginx/cert/host.cert -days 3650 -subj '/CN=localhost' -nodes -config <(cat /etc/ssl/openssl.cnf <(printf "\n[SAN]\nsubjectAltName=DNS:*.localhost"))
 
-# generate legacy key format for sftp
-ssh-keygen -m PEM -t rsa -f boxdata/sftp/ssh/ssh_host_rsa_key -q -N ""
-
-# clear out any containers
-echo "=> Delete all docker containers first"
-docker ps -qa | xargs --no-run-if-empty docker rm -f
+# clear out any containers if FAST is unset
+if [[ -z ${FAST+x} ]]; then
+    echo "=> Delete all docker containers first"
+    docker ps -qa | xargs --no-run-if-empty docker rm -f
+    echo "==> To skip this run with: FAST=1 ./runTests"
+else
+    echo "==> WARNING!! Skipping docker container cleanup, the database might not be pristine!"
+fi
 
 # create docker network (while the infra code does this, most tests skip infra setup)
 docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
@@ -60,6 +62,9 @@ while ! mysqladmin ping -h"${MYSQL_IP}" --silent; do
     sleep 1
 done
 
+echo "=> Create iptables blocklist"
+sudo ipset create cloudron_blocklist hash:net || true
+
 echo "=> Starting cloudron-syslog"
 cloudron-syslog --logdir "${DATA_DIR}/platformdata/logs/" &
 

scripts/cloudron-setup

@@ -41,7 +41,8 @@ if [[ "${disk_size_gb}" -lt "${MINIMUM_DISK_SIZE_GB}" ]]; then
     exit 1
 fi
 
-if systemctl -q is-active box; then
+# do not use is-active in case box service is down and user attempts to re-install
+if systemctl cat box.service >/dev/null 2>&1; then
     echo "Error: Cloudron is already installed. To reinstall, start afresh"
     exit 1
 fi

scripts/cloudron-support

@@ -37,12 +37,13 @@ while true; do
         # fall through
         ;&
     --owner-login)
-        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY createdAt LIMIT 1" 2>/dev/null)
+        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY creationTime LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
+        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/nul)
         ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
         printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > "${ghost_file}"
         chown yellowtent:yellowtent "${ghost_file}" && chmod o-r,g-r "${ghost_file}"
-        echo "Login as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
+        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
         exit 0
         ;;
     --) break;;

scripts/installer.sh

@@ -15,6 +15,48 @@ function log() {
   echo -e "$(date +'%Y-%m-%dT%H:%M:%S')" "==> installer: $1"
 }
 
+apt_ready="no"
+function prepare_apt_once() {
+    [[ "${apt_ready}" == "yes" ]] && return
+
+    log "Making sure apt is in a good state"
+
+    log "Waiting for all dpkg tasks to finish..."
+    while fuser /var/lib/dpkg/lock; do
+        sleep 1
+    done
+
+    # it's unclear what needs to be run first or whether both these command should be run. so keep trying both
+    for count in {1..3}; do
+        # alternative to apt-install -y --fix-missing ?
+        if ! dpkg --force-confold --configure -a; then
+            log "dpkg reconfigure failed (try $count)"
+            dpkg_configure="no"
+        else
+            dpkg_configure="yes"
+        fi
+
+        if ! apt update -y; then
+            log "apt update failed (try $count)"
+            apt_update="no"
+        else
+            apt_update="yes"
+        fi
+
+        [[ "${dpkg_configure}" == "yes" && "${apt_update}" == "yes" ]] && break
+
+        sleep 1
+    done
+
+    apt_ready="yes"
+
+    if [[ "${dpkg_configure}" == "yes" && "${apt_update}" == "yes" ]]; then
+        log "apt is ready"
+    else
+        log "apt is not ready but proceeding anyway"
+    fi
+}
+
 readonly user=yellowtent
 readonly box_src_dir=/home/${user}/box
 
@@ -38,21 +80,7 @@ if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]];
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
-    log "Waiting for all dpkg tasks to finish..."
-    while fuser /var/lib/dpkg/lock; do
-        sleep 1
-    done
-
-    while ! dpkg --force-confold --configure -a; do
-        log "Failed to fix packages. Retry"
-        sleep 1
-    done
-
-    # the latest docker might need newer packages
-    while ! apt update -y; do
-        log "Failed to update packages. Retry"
-        sleep 1
-    done
+    prepare_apt_once
 
     while ! apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb; do
         log "Failed to install docker. Retry"
@@ -66,11 +94,26 @@ readonly nginx_version=$(nginx -v 2>&1)
 if [[ "${nginx_version}" != *"1.18."* ]]; then
     log "installing nginx 1.18"
     $curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
+
+    prepare_apt_once
+
     # apt install with install deps (as opposed to dpkg -i)
     apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
     rm /tmp/nginx.deb
 fi
 
+if ! which mount.nfs; then
+    log "installing nfs-common"
+    prepare_apt_once
+    apt install -y nfs-common
+fi
+
+if ! which sshfs; then
+    log "installing sshfs"
+    prepare_apt_once
+    apt install -y sshfs
+fi
+
 log "updating node"
 readonly node_version=14.15.4
 if [[ "$(node --version)" != "v${node_version}" ]]; then

setup/start.sh

@@ -14,9 +14,10 @@ log "Cloudron Start"
 readonly USER="yellowtent"
 readonly HOME_DIR="/home/${USER}"
 readonly BOX_SRC_DIR="${HOME_DIR}/box"
-readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata" # platform data
-readonly APPS_DATA_DIR="${HOME_DIR}/appsdata" # app data
-readonly BOX_DATA_DIR="${HOME_DIR}/boxdata" # box data
+readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata"
+readonly APPS_DATA_DIR="${HOME_DIR}/appsdata"
+readonly BOX_DATA_DIR="${HOME_DIR}/boxdata"
+readonly MAIL_DATA_DIR="${HOME_DIR}/boxdata/mail"
 
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly json="$(realpath ${script_dir}/../node_modules/.bin/json)"
@@ -41,6 +42,7 @@ docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron |
 
 mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
+mkdir -p "${MAIL_DATA_DIR}/dkim"
 
 # keep these in sync with paths.js
 log "Ensuring directories"
@@ -61,20 +63,17 @@ mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
          "${PLATFORM_DATA_DIR}/logs/crash" \
          "${PLATFORM_DATA_DIR}/logs/collectd"
 mkdir -p "${PLATFORM_DATA_DIR}/update"
-
-mkdir -p "${BOX_DATA_DIR}/appicons"
-mkdir -p "${BOX_DATA_DIR}/firewall"
-mkdir -p "${BOX_DATA_DIR}/profileicons"
-mkdir -p "${BOX_DATA_DIR}/certs"
-mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
-mkdir -p "${BOX_DATA_DIR}/mail/dkim"
-mkdir -p "${BOX_DATA_DIR}/well-known" # .well-known documents
-mkdir -p "${BOX_DATA_DIR}/sftp/ssh" # sftp keys
+mkdir -p "${PLATFORM_DATA_DIR}/sftp/ssh" # sftp keys
+mkdir -p "${PLATFORM_DATA_DIR}/firewall"
 
 # ensure backups folder exists and is writeable
 mkdir -p /var/backups
 chmod 777 /var/backups
 
+# can be removed after 6.3
+[[ -f "${BOX_DATA_DIR}/updatechecker.json" ]] && mv "${BOX_DATA_DIR}/updatechecker.json" "${PLATFORM_DATA_DIR}/update/updatechecker.json"
+rm -rf "${BOX_DATA_DIR}/well-known"
+
 log "Configuring journald"
 sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
     -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
@@ -109,6 +108,7 @@ unbound-anchor -a /var/lib/unbound/root.key
 log "Adding systemd services"
 cp -r "${script_dir}/start/systemd/." /etc/systemd/system/
 [[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/MemoryMax/MemoryLimit/g' -i /etc/systemd/system/box.service
+[[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/Type=notify/Type=simple/g' -i /etc/systemd/system/unbound.service
 systemctl daemon-reload
 systemctl enable --now cloudron-syslog
 systemctl enable unbound
@@ -224,37 +224,20 @@ fi
 
 rm -f /etc/cloudron/cloudron.conf
 
-if [[ ! -f "${BOX_DATA_DIR}/dhparams.pem" ]]; then
-    log "Generating dhparams (takes forever)"
-    openssl dhparam -out "${BOX_DATA_DIR}/dhparams.pem" 2048
-    cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem"
-else
-    cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem"
-fi
-
-if [[ ! -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" ]]; then
-    # the key format in Ubuntu 20 changed, so we create keys in legacy format. for older ubuntu, just re-use the host keys
-    # see https://github.com/proftpd/proftpd/issues/793
-    if [[ "${ubuntu_version}" == "20.04" ]]; then
-        ssh-keygen -m PEM -t rsa -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" -q -N ""
-    else
-        cp /etc/ssh/ssh_host_rsa_key* ${BOX_DATA_DIR}/sftp/ssh
-    fi
-fi
-
 log "Changing ownership"
+# note, change ownership after db migrate. this allow db migrate to move files around as root and then we can fix it up here
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
 chown -R "${USER}" /etc/cloudron
-chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update"
+chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall"
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 chown "${USER}:${USER}" "${APPS_DATA_DIR}"
 
 # do not chown the boxdata/mail directory; dovecot gets upset
 chown "${USER}:${USER}" "${BOX_DATA_DIR}"
-find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${BOX_DATA_DIR}/mail" -exec chown -R "${USER}:${USER}" {} \;
-chown "${USER}:${USER}" "${BOX_DATA_DIR}/mail"
-chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys
+find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${MAIL_DATA_DIR}" -exec chown -R "${USER}:${USER}" {} \;
+chown "${USER}:${USER}" "${MAIL_DATA_DIR}"
+chown "${USER}:${USER}" -R "${MAIL_DATA_DIR}/dkim" # this is owned by box currently since it generates the keys
 
 log "Starting Cloudron"
 systemctl start box

setup/start/cloudron-firewall.sh

@@ -18,10 +18,10 @@ fi
 
 # allow related and establisted connections
 iptables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
-iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,25,80,202,443 -j ACCEPT # 202 is the alternate ssh port
+iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,443 -j ACCEPT # 202 is the alternate ssh port
 
 # whitelist any user ports. we used to use --dports but it has a 15 port limit (XT_MULTI_PORTS)
-ports_json="/home/yellowtent/boxdata/firewall/ports.json"
+ports_json="/home/yellowtent/platformdata/firewall/ports.json"
 if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(','))" 2>/dev/null); then
     IFS=',' arr=(${allowed_tcp_ports})
     for p in "${arr[@]}"; do
@@ -70,7 +70,7 @@ for port in 80 443; do
     iptables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
-# ssh smtp ssh msa imap sieve
+# ssh
 for port in 22 202; do
     iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
     iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG

setup/start/collectd/collectd.conf

@@ -164,7 +164,9 @@ LoadPlugin swap
 #LoadPlugin vmem
 #LoadPlugin vserver
 #LoadPlugin wireless
-LoadPlugin write_graphite
+<LoadPlugin write_graphite>
+   FlushInterval 20
+</LoadPlugin>
 #LoadPlugin write_http
 #LoadPlugin write_riemann
 

setup/start/collectd/df.py

@@ -34,4 +34,5 @@ def read():
         val.dispatch(values=[used], type_instance='used')
 
 collectd.register_init(init)
+# see Interval setting in collectd.conf for polling interval
 collectd.register_read(read)

setup/start/collectd/du.py

@@ -6,19 +6,26 @@ PATHS = [] # { name, dir, exclude }
 # there is a pattern in carbon/storage-schemas.conf which stores values every 12h for a year
 INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this
 
+# we used to pass the INTERVAL as a parameter to register_read. however, collectd write_graphite
+# takes a bit to load (tcp connection) and drops the du data. this then means that we have to wait
+# for INTERVAL secs for du data. instead, we just cache the value for INTERVAL instead
+CACHE = dict()
+CACHE_TIME = 0
+
 def du(pathinfo):
     # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
-    cmd = 'timeout 1800 du -DsB1 "{}"'.format(pathinfo['dir'])
+    dirname = pathinfo['dir']
+    cmd = 'timeout 1800 du -DsB1 "{}"'.format(dirname)
     if pathinfo['exclude'] != '':
         cmd += ' --exclude "{}"'.format(pathinfo['exclude'])
 
     collectd.info('computing size with command: %s' % cmd);
     try:
         size = subprocess.check_output(cmd, shell=True).split()[0].decode('utf-8')
-        collectd.info('\tsize of %s is %s (time: %i)' % (pathinfo['dir'], size, int(time.time())))
+        collectd.info('\tsize of %s is %s (time: %i)' % (dirname, size, int(time.time())))
         return size
     except Exception as e:
-        collectd.info('\terror getting the size of %s: %s' % (pathinfo['dir'], str(e)))
+        collectd.info('\terror getting the size of %s: %s' % (dirname, str(e)))
         return 0
 
 def parseSize(size):
@@ -64,19 +71,35 @@ def init():
     collectd.info('custom du plugin initialized with %s %s' % (PATHS, sys.version))
 
 def read():
+    global CACHE, CACHE_TIME
+
+    # read from cache if < 12 hours
+    read_cache = (time.time() - CACHE_TIME) < INTERVAL
+
+    if not read_cache:
+        CACHE_TIME = time.time()
+
     for pathinfo in PATHS:
-        size = du(pathinfo)
+        dirname = pathinfo['dir']
+        if read_cache and dirname in CACHE:
+            size = CACHE[dirname]
+        else:
+            size = du(pathinfo)
+            CACHE[dirname] = size
 
         # type comes from https://github.com/collectd/collectd/blob/master/src/types.db
         val = collectd.Values(type='capacity', plugin='du', plugin_instance=pathinfo['name'])
         val.dispatch(values=[size], type_instance='usage')
 
-    size = dockerSize()
+    if read_cache and 'docker' in CACHE:
+        size = CACHE['docker']
+    else:
+        size = dockerSize()
+        CACHE['docker'] = size
+
     val = collectd.Values(type='capacity', plugin='du', plugin_instance='docker')
     val.dispatch(values=[size], type_instance='usage')
 
-
-
 collectd.register_init(init)
 collectd.register_config(configure)
-collectd.register_read(read, INTERVAL)
+collectd.register_read(read)

setup/start/mysql.cnf

@@ -4,7 +4,7 @@
 # http://bugs.mysql.com/bug.php?id=68514
 [mysqld]
 performance_schema=OFF
-max_connections=50
+max_connections=200
 # on ec2, without this we get a sporadic connection drop when doing the initial migration
 max_allowed_packet=64M
 
@@ -18,6 +18,12 @@ default_time_zone='+00:00'
 # disable bin logs. they are only useful in replication mode
 skip-log-bin
 
+# this is used when creating an index using ALTER command
+innodb_sort_buffer_size=2097152
+
+# this is a per session sort (ORDER BY) variable for non-indexed fields
+sort_buffer_size = 4M
+
 [mysqldump]
 quick
 quote-names

setup/start/sudoers

@@ -13,9 +13,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/mkdirvolume.sh
 Defaults!/home/yellowtent/box/src/scripts/rmaddondir.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmaddondir.sh
 
-Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadnginx.sh
-
 Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 
@@ -41,11 +38,8 @@ yellowtent ALL=(root) NOPASSWD:SETENV: /home/yellowtent/box/src/scripts/backupup
 Defaults!/home/yellowtent/box/src/scripts/restart.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restart.sh
 
-Defaults!/home/yellowtent/box/src/scripts/restartdocker.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartdocker.sh
-
-Defaults!/home/yellowtent/box/src/scripts/restartunbound.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartunbound.sh
+Defaults!/home/yellowtent/box/src/scripts/restartservice.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartservice.sh
 
 Defaults!/home/yellowtent/box/src/scripts/rmmailbox.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmailbox.sh
@@ -59,3 +53,9 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/stoptask.sh
 Defaults!/home/yellowtent/box/src/scripts/setblocklist.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setblocklist.sh
 
+Defaults!/home/yellowtent/box/src/scripts/addmount.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/addmount.sh
+
+Defaults!/home/yellowtent/box/src/scripts/rmmount.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmount.sh
+

setup/start/systemd/box.service

@@ -13,7 +13,8 @@ Type=idle
 WorkingDirectory=/home/yellowtent/box
 Restart=always
 ExecStart=/home/yellowtent/box/box.js
-Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production"
+; we run commands like df which will parse properly only with correct locale
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C"
 ; kill apptask processes as well
 KillMode=control-group
 ; Do not kill this process on OOM. Children inherit this score. Do not set it to -1000 so that MemoryMax can keep working

setup/start/systemd/cloudron-firewall.service

@@ -5,6 +5,7 @@ PartOf=docker.service
 
 [Service]
 Type=oneshot
+Environment="BOX_ENV=cloudron"
 ExecStart="/home/yellowtent/box/setup/start/cloudron-firewall.sh"
 RemainAfterExit=yes
 

setup/start/systemd/unbound.service

@@ -2,13 +2,17 @@
 
 [Unit]
 Description=Unbound DNS Resolver
-After=network.target docker.service
+After=network-online.target docker.service
+Before=nss-lookup.target
+Wants=network-online.target nss-lookup.target
 
 [Service]
 PIDFile=/run/unbound.pid
 ExecStart=/usr/sbin/unbound -d
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
+# On ubuntu 16, this doesn't work for some reason
+Type=notify
 
 [Install]
 WantedBy=multi-user.target

src/accesscontrol.js

@@ -1,29 +1,31 @@
 'use strict';
 
 exports = module.exports = {
-    verifyToken: verifyToken
+    verifyToken
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    tokendb = require('./tokendb.js'),
-    users = require('./users.js');
+    safe = require('safetydance'),
+    tokens = require('./tokens.js'),
+    users = require('./users.js'),
+    util = require('util');
 
-function verifyToken(accessToken, callback) {
+const userGet = util.promisify(users.get);
+
+async function verifyToken(accessToken) {
     assert.strictEqual(typeof accessToken, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    tokendb.getByAccessToken(accessToken, function (error, token) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-        if (error) return callback(error);
+    const token = await tokens.getByAccessToken(accessToken);
+    if (!token) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'No such token');
 
-        users.get(token.identifier, function (error, user) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (error) return callback(error);
+    const [error, user] = await safe(userGet(token.identifier));
+    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not found');
+    if (error) throw error;
 
-            if (!user.active) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+    if (!user.active) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not active');
 
-            callback(null, user);
-        });
-    });
+    await safe(tokens.update(token.id, { lastUsedTime: new Date() })); // ignore any error
+
+    return user;
 }

src/acme2.js

@@ -0,0 +1,547 @@
+'use strict';
+
+exports = module.exports = {
+    getCertificate,
+
+    // testing
+    _name: 'acme',
+    _getChallengeSubdomain: getChallengeSubdomain
+};
+
+const assert = require('assert'),
+    async = require('async'),
+    BoxError = require('./boxerror.js'),
+    crypto = require('crypto'),
+    debug = require('debug')('box:cert/acme2'),
+    domains = require('./domains.js'),
+    fs = require('fs'),
+    os = require('os'),
+    path = require('path'),
+    promiseRetry = require('./promise-retry.js'),
+    superagent = require('superagent'),
+    safe = require('safetydance'),
+    _ = require('underscore');
+
+const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
+    CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory';
+
+// http://jose.readthedocs.org/en/latest/
+// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
+// https://community.letsencrypt.org/t/list-of-client-implementations/2103
+
+function Acme2(options) {
+    assert.strictEqual(typeof options, 'object');
+
+    this.accountKeyPem = options.accountKeyPem; // Buffer
+    this.email = options.email;
+    this.keyId = null;
+    this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
+    this.directory = {};
+    this.performHttpAuthorization = !!options.performHttpAuthorization;
+    this.wildcard = !!options.wildcard;
+}
+
+// urlsafe base64 encoding (jose)
+function urlBase64Encode(string) {
+    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
+}
+
+function b64(str) {
+    var buf = Buffer.isBuffer(str) ? str : Buffer.from(str);
+    return urlBase64Encode(buf.toString('base64'));
+}
+
+function getModulus(pem) {
+    assert(Buffer.isBuffer(pem));
+
+    var stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
+    if (!stdout) return null;
+    var match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
+    if (!match) return null;
+    return Buffer.from(match[1], 'hex');
+}
+
+Acme2.prototype.sendSignedRequest = async function (url, payload) {
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof payload, 'string');
+
+    assert(Buffer.isBuffer(this.accountKeyPem));
+
+    const that = this;
+    let header = {
+        url: url,
+        alg: 'RS256'
+    };
+
+    // keyId is null when registering account
+    if (this.keyId) {
+        header.kid = this.keyId;
+    } else {
+        header.jwk = {
+            e: b64(Buffer.from([0x01, 0x00, 0x01])), // exponent - 65537
+            kty: 'RSA',
+            n: b64(getModulus(this.accountKeyPem))
+        };
+    }
+
+    const payload64 = b64(payload);
+
+    let [error, response] = await safe(superagent.get(this.directory.newNonce).timeout(30000).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`);
+    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code when fetching nonce : ${response.status}`);
+
+    const nonce = response.headers['Replay-Nonce'.toLowerCase()];
+    if (!nonce) throw new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response');
+
+    debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
+
+    const protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
+
+    const signer = crypto.createSign('RSA-SHA256');
+    signer.update(protected64 + '.' + payload64, 'utf8');
+    const signature64 = urlBase64Encode(signer.sign(that.accountKeyPem, 'base64'));
+
+    const data = {
+        protected: protected64,
+        payload: payload64,
+        signature: signature64
+    };
+
+    [error, response] = await safe(superagent.post(url).send(data).set('Content-Type', 'application/jose+json').set('User-Agent', 'acme-cloudron').timeout(30000).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`);
+
+    return response;
+};
+
+// https://tools.ietf.org/html/rfc8555#section-6.3
+Acme2.prototype.postAsGet = async function (url) {
+    return await this.sendSignedRequest(url, '');
+};
+
+Acme2.prototype.updateContact = async function (registrationUri) {
+    assert.strictEqual(typeof registrationUri, 'string');
+
+    debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
+
+    // https://github.com/ietf-wg-acme/acme/issues/30
+    const payload = {
+        contact: [ 'mailto:' + this.email ]
+    };
+
+    const result = await this.sendSignedRequest(registrationUri, JSON.stringify(payload));
+    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
+
+    debug(`updateContact: contact of user updated to ${this.email}`);
+};
+
+Acme2.prototype.registerUser = async function () {
+    const payload = {
+        termsOfServiceAgreed: true
+    };
+
+    debug('registerUser: registering user');
+
+    const result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
+    // 200 if already exists. 201 for new accounts
+    if (result.status !== 200 && result.status !== 201) return new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${JSON.stringify(result.body)}`);
+
+    debug(`registerUser: user registered keyid: ${result.headers.location}`);
+
+    this.keyId = result.headers.location;
+
+    await this.updateContact(result.headers.location);
+};
+
+Acme2.prototype.newOrder = async function (domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    const payload = {
+        identifiers: [{
+            type: 'dns',
+            value: domain
+        }]
+    };
+
+    debug(`newOrder: ${domain}`);
+
+    const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload));
+    if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`);
+    if (result.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+
+    debug('newOrder: created order %s %j', domain, result.body);
+
+    const order = result.body, orderUrl = result.headers.location;
+
+    if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid authorizations in order');
+    if (typeof order.finalize !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid finalize in order');
+    if (typeof orderUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid order location in order header');
+
+    return { order, orderUrl };
+};
+
+Acme2.prototype.waitForOrder = async function (orderUrl) {
+    assert.strictEqual(typeof orderUrl, 'string');
+
+    debug(`waitForOrder: ${orderUrl}`);
+
+    return await promiseRetry({ times: 15, interval: 20000 }, async () => {
+        debug('waitForOrder: getting status');
+
+        const result = await this.postAsGet(orderUrl);
+        if (result.status !== 200) {
+            debug(`waitForOrder: invalid response code getting uri ${result.status}`);
+            throw new BoxError(BoxError.EXTERNAL_ERROR, `Bad response code: ${result.status}`);
+        }
+
+        debug('waitForOrder: status is "%s %j', result.body.status, result.body);
+
+        if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.TRY_AGAIN, `Request is in ${result.body.status} state`);
+        else if (result.body.status === 'valid' && result.body.certificate) return result.body.certificate;
+        else throw new BoxError(BoxError.EXTERNAL_ERROR, `Unexpected status or invalid response: ${JSON.stringify(result.body)}`);
+    });
+};
+
+Acme2.prototype.getKeyAuthorization = function (token) {
+    assert(Buffer.isBuffer(this.accountKeyPem));
+
+    let jwk = {
+        e: b64(Buffer.from([0x01, 0x00, 0x01])), // Exponent - 65537
+        kty: 'RSA',
+        n: b64(getModulus(this.accountKeyPem))
+    };
+
+    let shasum = crypto.createHash('sha256');
+    shasum.update(JSON.stringify(jwk));
+    let thumbprint = urlBase64Encode(shasum.digest('base64'));
+    return token + '.' + thumbprint;
+};
+
+Acme2.prototype.notifyChallengeReady = async function (challenge) {
+    assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token }
+
+    debug('notifyChallengeReady: %s was met', challenge.url);
+
+    const keyAuthorization = this.getKeyAuthorization(challenge.token);
+
+    const payload = {
+        resource: 'challenge',
+        keyAuthorization: keyAuthorization
+    };
+
+    const result = await this.sendSignedRequest(challenge.url, JSON.stringify(payload));
+    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+};
+
+Acme2.prototype.waitForChallenge = async function (challenge) {
+    assert.strictEqual(typeof challenge, 'object');
+
+    debug('waitingForChallenge: %j', challenge);
+
+    await promiseRetry({ times: 15, interval: 20000 }, async () => {
+        debug('waitingForChallenge: getting status');
+
+        const result = await this.postAsGet(challenge.url);
+        if (result.status !== 200) {
+            debug(`waitForChallenge: invalid response code getting uri ${result.status}`);
+            throw new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode);
+        }
+
+        debug(`waitForChallenge: status is "${result.body.status}" "${JSON.stringify(result.body)}"`);
+
+        if (result.body.status === 'pending') throw new BoxError(BoxError.TRY_AGAIN);
+        else if (result.body.status === 'valid') return;
+        else throw new BoxError(BoxError.EXTERNAL_ERROR, `Unexpected status: ${result.body.status}`);
+    });
+};
+
+// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
+Acme2.prototype.signCertificate = async function (domain, finalizationUrl, csrDer) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof finalizationUrl, 'string');
+    assert(Buffer.isBuffer(csrDer));
+
+    const payload = {
+        csr: b64(csrDer)
+    };
+
+    debug('signCertificate: sending sign request');
+
+    const result = await this.sendSignedRequest(finalizationUrl, JSON.stringify(payload));
+    // 429 means we reached the cert limit for this domain
+    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
+};
+
+Acme2.prototype.createKeyAndCsr = async function (hostname, keyFilePath, csrFilePath) {
+    assert.strictEqual(typeof hostname, 'string');
+
+    if (safe.fs.existsSync(keyFilePath)) {
+        debug('createKeyAndCsr: reuse the key for renewal at %s', keyFilePath);
+    } else {
+        let key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
+        if (!key) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+        if (!safe.fs.writeFileSync(keyFilePath, key)) throw new BoxError(BoxError.FS_ERROR, safe.error);
+
+        debug('createKeyAndCsr: key file saved at %s', keyFilePath);
+    }
+
+    const [error, tmpdir] = await safe(fs.promises.mkdtemp(path.join(os.tmpdir(), 'acme-')));
+    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating temporary directory for openssl config: ${error.message}`);
+
+    // OCSP must-staple is currently disabled because nginx does not provide staple on the first request (https://forum.cloudron.io/topic/4917/ocsp-stapling-for-tls-ssl/)
+    // ' -addext "tlsfeature = status_request"'; // this adds OCSP must-staple
+    // we used to use -addext to the CLI to add these but that arg doesn't work on Ubuntu 16.04
+    // empty distinguished_name section is required for Ubuntu 16 openssl
+    const conf = '[req]\nreq_extensions = v3_req\ndistinguished_name = req_distinguished_name\n'
+        + '[req_distinguished_name]\n\n'
+        + '[v3_req]\nbasicConstraints = CA:FALSE\nkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\nsubjectAltName = @alt_names\n'
+        + `[alt_names]\nDNS.1 = ${hostname}\n`;
+
+    const opensslConfigFile = path.join(tmpdir, 'openssl.conf');
+    if (!safe.fs.writeFileSync(opensslConfigFile, conf)) throw new BoxError(BoxError.FS_ERROR, `Failed to write openssl config: ${safe.error.message}`);
+
+    // while we pass the CN anyways, subjectAltName takes precedence
+    const csrDer = safe.child_process.execSync(`openssl req -new -key ${keyFilePath} -outform DER -subj /CN=${hostname} -config ${opensslConfigFile}`);
+    if (!csrDer) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+    if (!safe.fs.writeFileSync(csrFilePath, csrDer)) throw new BoxError(BoxError.FS_ERROR, safe.error); // bookkeeping. inspect with openssl req -text -noout -in hostname.csr -inform der
+
+    await safe(fs.promises.rmdir(tmpdir, { recursive: true }));
+
+    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFilePath);
+
+    return csrDer;
+};
+
+Acme2.prototype.downloadCertificate = async function (hostname, certUrl, certFilePath) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof certUrl, 'string');
+
+    await promiseRetry({ times: 5, interval: 20000 }, async () => {
+        debug('downloadCertificate: downloading certificate');
+
+        const result = await this.postAsGet(certUrl);
+        if (result.statusCode === 202) throw new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate');
+        if (result.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+
+        const fullChainPem = result.body; // buffer
+
+        if (!safe.fs.writeFileSync(certFilePath, fullChainPem)) throw new BoxError(BoxError.FS_ERROR, safe.error);
+
+        debug(`downloadCertificate: cert file for ${hostname} saved at ${certFilePath}`);
+    });
+};
+
+Acme2.prototype.prepareHttpChallenge = async function (hostname, domain, authorization, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof authorization, 'object');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
+
+    debug('prepareHttpChallenge: challenges: %j', authorization);
+    let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
+    if (httpChallenges.length === 0) throw new BoxError(BoxError.EXTERNAL_ERROR, 'no http challenges');
+    let challenge = httpChallenges[0];
+
+    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
+
+    let keyAuthorization = this.getKeyAuthorization(challenge.token);
+
+    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(acmeChallengesDir, challenge.token));
+
+    if (!safe.fs.writeFileSync(path.join(acmeChallengesDir, challenge.token), keyAuthorization)) throw new BoxError(BoxError.FS_ERROR, `Error writing challenge: ${safe.error.message}`);
+
+    return challenge;
+};
+
+Acme2.prototype.cleanupHttpChallenge = async function (hostname, domain, challenge, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
+
+    debug('cleanupHttpChallenge: unlinking %s', path.join(acmeChallengesDir, challenge.token));
+
+    if (!safe.fs.unlinkSync(path.join(acmeChallengesDir, challenge.token))) throw new BoxError(BoxError.FS_ERROR, `Error unlinking challenge: ${safe.error.message}`);
+};
+
+function getChallengeSubdomain(hostname, domain) {
+    let challengeSubdomain;
+
+    if (hostname === domain) {
+        challengeSubdomain = '_acme-challenge';
+    } else if (hostname.includes('*')) { // wildcard
+        let subdomain = hostname.slice(0, -domain.length - 1);
+        challengeSubdomain = subdomain ? subdomain.replace('*', '_acme-challenge') : '_acme-challenge';
+    } else {
+        challengeSubdomain = '_acme-challenge.' + hostname.slice(0, -domain.length - 1);
+    }
+
+    debug(`getChallengeSubdomain: challenge subdomain for hostname ${hostname} at domain ${domain} is ${challengeSubdomain}`);
+
+    return challengeSubdomain;
+}
+
+Acme2.prototype.prepareDnsChallenge = async function (hostname, domain, authorization) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof authorization, 'object');
+
+    debug('prepareDnsChallenge: challenges: %j', authorization);
+    const dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
+    if (dnsChallenges.length === 0) throw new BoxError(BoxError.EXTERNAL_ERROR, 'no dns challenges');
+    const challenge = dnsChallenges[0];
+
+    const keyAuthorization = this.getKeyAuthorization(challenge.token);
+    const shasum = crypto.createHash('sha256');
+    shasum.update(keyAuthorization);
+
+    const txtValue = urlBase64Encode(shasum.digest('base64'));
+    const challengeSubdomain = getChallengeSubdomain(hostname, domain);
+
+    debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
+
+    return new Promise((resolve, reject) => {
+        domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
+            if (error) return reject(error);
+
+            domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 }, function (error) {
+                if (error) return reject(error);
+
+                resolve(challenge);
+            });
+        });
+    });
+};
+
+Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challenge) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof challenge, 'object');
+
+    const keyAuthorization = this.getKeyAuthorization(challenge.token);
+    let shasum = crypto.createHash('sha256');
+    shasum.update(keyAuthorization);
+
+    const txtValue = urlBase64Encode(shasum.digest('base64'));
+    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
+
+    debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
+
+    return new Promise((resolve, reject) => {
+        domains.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
+            if (error) return reject(error);
+
+            resolve(null);
+        });
+    });
+};
+
+Acme2.prototype.prepareChallenge = async function (hostname, domain, authorizationUrl, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof authorizationUrl, 'string');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
+
+    debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
+
+    const response = await this.postAsGet(authorizationUrl);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code getting authorization : ${response.status}`);
+
+    const authorization = response.body;
+
+    if (this.performHttpAuthorization) {
+        return await this.prepareHttpChallenge(hostname, domain, authorization, acmeChallengesDir);
+    } else {
+        return await this.prepareDnsChallenge(hostname, domain, authorization);
+    }
+};
+
+Acme2.prototype.cleanupChallenge = async function (hostname, domain, challenge, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
+
+    debug(`cleanupChallenge: http: ${this.performHttpAuthorization}`);
+
+    if (this.performHttpAuthorization) {
+        await this.cleanupHttpChallenge(hostname, domain, challenge, acmeChallengesDir);
+    } else {
+        await this.cleanupDnsChallenge(hostname, domain, challenge);
+    }
+};
+
+Acme2.prototype.acmeFlow = async function (hostname, domain, paths) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof paths, 'object');
+
+    const { certFilePath, keyFilePath, csrFilePath, acmeChallengesDir } = paths;
+
+    await this.registerUser();
+    const { order, orderUrl } = await this.newOrder(hostname);
+
+    for (let i = 0; i < order.authorizations.length; i++) {
+        const authorizationUrl = order.authorizations[i];
+        debug(`acmeFlow: authorizing ${authorizationUrl}`);
+
+        const challenge = await this.prepareChallenge(hostname, domain, authorizationUrl, acmeChallengesDir);
+        await this.notifyChallengeReady(challenge);
+        await this.waitForChallenge(challenge);
+        const csrDer = await this.createKeyAndCsr(hostname, keyFilePath, csrFilePath);
+        await this.signCertificate(hostname, order.finalize, csrDer);
+        const certUrl = await this.waitForOrder(orderUrl);
+        await this.downloadCertificate(hostname, certUrl, certFilePath);
+
+        try {
+            await this.cleanupChallenge(hostname, domain, challenge, acmeChallengesDir);
+        } catch (cleanupError) {
+            debug('acmeFlow: ignoring error when cleaning up challenge:', cleanupError);
+        }
+    }
+};
+
+Acme2.prototype.loadDirectory = async function () {
+    await promiseRetry({ times: 3, interval: 20000 }, async () => {
+        const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true);
+
+        if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code when fetching directory : ${response.status}`);
+
+        if (typeof response.body.newNonce !== 'string' ||
+            typeof response.body.newOrder !== 'string' ||
+            typeof response.body.newAccount !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`);
+
+        this.directory = response.body;
+    });
+};
+
+Acme2.prototype.getCertificate = async function (vhost, domain, paths) {
+    assert.strictEqual(typeof vhost, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof paths, 'object');
+
+    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
+
+    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
+        vhost = domains.makeWildcard(vhost);
+        debug(`getCertificate: will get wildcard cert for ${vhost}`);
+    }
+
+    await this.loadDirectory();
+    await this.acmeFlow(vhost, domain, paths);
+};
+
+function getCertificate(vhost, domain, paths, options, callback) {
+    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof paths, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    let attempt = 1;
+    async.retry({ times: 3, interval: 0 }, function (retryCallback) {
+        debug(`getCertificate: attempt ${attempt++}`);
+
+        let acme = new Acme2(options || { });
+        acme.getCertificate(vhost, domain, paths).then(callback).catch(retryCallback);
+    }, callback);
+}

src/appdb.js

@@ -19,6 +19,8 @@ exports = module.exports = {
     getAppIdByAddonConfigValue,
     getByIpAddress,
 
+    getIcons,
+
     setHealth,
     setTask,
     getAppStoreIds,
@@ -31,7 +33,7 @@ exports = module.exports = {
     _clear: clear
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
     database = require('./database.js'),
@@ -42,8 +44,8 @@ const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationS
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp',
-    'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
-    'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(',');
+    'apps.creationTime', 'apps.updateTime', 'apps.enableMailbox', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
+    'apps.dataDir', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
 
 const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
 
@@ -83,10 +85,13 @@ function postProcess(result) {
     if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
     delete result.accessRestrictionJson;
 
-    result.sso = !!result.sso; // make it bool
-    result.enableBackup = !!result.enableBackup; // make it bool
-    result.enableAutomaticUpdate = !!result.enableAutomaticUpdate; // make it bool
+    result.sso = !!result.sso;
+    result.enableBackup = !!result.enableBackup;
+    result.enableAutomaticUpdate = !!result.enableAutomaticUpdate;
+    result.enableMailbox = !!result.enableMailbox;
     result.proxyAuth = !!result.proxyAuth;
+    result.hasIcon = !!result.hasIcon;
+    result.hasAppStoreIcon = !!result.hasAppStoreIcon;
 
     assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
     result.debugMode = safe.JSON.parse(result.debugModeJson);
@@ -215,15 +220,18 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
     const mailboxName = data.mailboxName || null;
     const mailboxDomain = data.mailboxDomain || null;
     const reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null;
+    const servicesConfigJson = data.servicesConfig ? JSON.stringify(data.servicesConfig) : null;
+    const enableMailbox = data.enableMailbox || false;
+    const icon = data.icon || null;
 
-    var queries = [];
+    let queries = [];
 
     queries.push({
         query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
-            + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+            + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox) '
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
         args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
-            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson ]
+            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox ]
     });
 
     queries.push({
@@ -291,7 +299,7 @@ function getPortBindings(id, callback) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
         var portBindings = { };
-        for (var i = 0; i < results.length; i++) {
+        for (let i = 0; i < results.length; i++) {
             portBindings[results[i].environmentVariable] = { hostPort: results[i].hostPort, type: results[i].type };
         }
 
@@ -299,6 +307,17 @@ function getPortBindings(id, callback) {
     });
 }
 
+function getIcons(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT icon, appStoreIcon FROM apps WHERE id = ?', [ id ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, { icon: results[0].icon, appStoreIcon: results[0].appStoreIcon });
+    });
+}
+
 function delPortBinding(hostPort, type, callback) {
     assert.strictEqual(typeof hostPort, 'number');
     assert.strictEqual(typeof type, 'string');
@@ -349,6 +368,9 @@ function clear(callback) {
 }
 
 function update(id, app, callback) {
+    // ts is useful as a versioning mechanism (for example, icon changed). update the timestamp explicity in code instead of db.
+    // this way health and healthTime can be updated without changing ts
+    app.ts = new Date();
     updateWithConstraints(id, app, '', callback);
 }
 
@@ -412,7 +434,7 @@ function updateWithConstraints(id, app, constraints, callback) {
     }
 
     var fields = [ ], values = [ ];
-    for (var p in app) {
+    for (let p in app) {
         if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig') {
             fields.push(`${p}Json = ?`);
             values.push(JSON.stringify(app[p]));
@@ -439,10 +461,10 @@ function updateWithConstraints(id, app, constraints, callback) {
 function setHealth(appId, health, healthTime, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof health, 'string');
-    assert(util.isDate(healthTime));
+    assert(util.types.isDate(healthTime));
     assert.strictEqual(typeof callback, 'function');
 
-    var values = { health, healthTime };
+    const values = { health, healthTime };
 
     updateWithConstraints(appId, values, '', callback);
 }
@@ -453,7 +475,9 @@ function setTask(appId, values, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!options.requireNullTaskId)  return updateWithConstraints(appId, values, '', callback);
+    values.ts = new Date();
+
+    if (!options.requireNullTaskId) return updateWithConstraints(appId, values, '', callback);
 
     if (options.requiredState === null) {
         updateWithConstraints(appId, values, 'AND taskId IS NULL', callback);
@@ -475,7 +499,7 @@ function getAppStoreIds(callback) {
 function setAddonConfig(appId, addonId, env, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof addonId, 'string');
-    assert(util.isArray(env));
+    assert(Array.isArray(env));
     assert.strictEqual(typeof callback, 'function');
 
     unsetAddonConfig(appId, addonId, function (error) {

src/apphealthmonitor.js

@@ -1,11 +1,12 @@
 'use strict';
 
-var appdb = require('./appdb.js'),
+const appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:apphealthmonitor'),
     docker = require('./docker.js'),
     eventlog = require('./eventlog.js'),
@@ -19,7 +20,7 @@ exports = module.exports = {
 const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
 const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes
 
-const OOM_EVENT_LIMIT = 60 * 60 * 1000; // 60 minutes
+const OOM_EVENT_LIMIT = 60 * 60 * 1000; // will only raise 1 oom event every hour
 let gStartTime = null; // time when apphealthmonitor was started
 let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago
 
@@ -95,7 +96,7 @@ function checkAppHealth(app, callback) {
             .end(function (error, res) {
                 if (error && !error.response) {
                     setHealth(app, apps.HEALTH_UNHEALTHY, callback);
-                } else if (res.statusCode >= 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
+                } else if (res.statusCode > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
                     setHealth(app, apps.HEALTH_UNHEALTHY, callback);
                 } else {
                     setHealth(app, apps.HEALTH_HEALTHY, callback);
@@ -110,7 +111,7 @@ function getContainerInfo(containerId, callback) {
 
         const appId = safe.query(result, 'Config.Labels.appId', null);
 
-        if (!appId) return callback(null, null /* app */, { name: result.Name }); // addon
+        if (!appId) return callback(null, null /* app */, { name: result.Name.slice(1) }); // addon . Name has a '/' in the beginning for some reason
 
         apps.get(appId, callback); // don't get by container id as this can be an exec container
     });
@@ -118,8 +119,7 @@ function getContainerInfo(containerId, callback) {
 
 /*
     OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:2.0.0 /bin/bash
-        apt-get update && apt-get install stress
+        docker run -ti -m 100M cloudron/base:3.0.0 /bin/bash
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
 function processDockerEvents(intervalSecs, callback) {
@@ -142,12 +142,12 @@ function processDockerEvents(intervalSecs, callback) {
                 const now = Date.now();
                 const notifyUser = !(app && app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
 
-                debug('OOM %s notifyUser: %s. lastOomTime: %s (now: %s)', program, notifyUser, gLastOomMailTime, now);
+                debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
 
                 // do not send mails for dev apps
                 if (notifyUser) {
                     // app can be null for addon containers
-                    eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event: event, containerId: containerId, addon: addon || null, app: app || null });
+                    eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event, containerId, addon: addon || null, app: app || null });
 
                     gLastOomMailTime = now;
                 }
@@ -187,6 +187,8 @@ function run(intervalSecs, callback) {
     assert.strictEqual(typeof intervalSecs, 'number');
     assert.strictEqual(typeof callback, 'function');
 
+    if (constants.TEST) return;
+
     if (!gStartTime) gStartTime = new Date();
 
     async.series([

src/apps.js

@@ -44,6 +44,8 @@ exports = module.exports = {
     getLocalLogfilePaths,
     getLogs,
 
+    getCertificate,
+
     start,
     stop,
     restart,
@@ -62,12 +64,15 @@ exports = module.exports = {
     restartAppsUsingAddons,
 
     getDataDir,
-    getIconPath,
+    getIcon,
     getMemoryLimit,
 
     downloadFile,
     uploadFile,
 
+    backupConfig,
+    restoreConfig,
+
     PORT_TYPE_TCP: 'tcp',
     PORT_TYPE_UDP: 'udp',
 
@@ -82,6 +87,7 @@ exports = module.exports = {
     ISTATE_PENDING_DEBUG: 'pending_debug',
     ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
     ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
+    ISTATE_PENDING_IMPORT: 'pending_import', // import from external backup
     ISTATE_PENDING_UPDATE: 'pending_update', // update from installed state preserving data
     ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app. this is state because it blocks other operations
     ISTATE_PENDING_START: 'pending_start',
@@ -107,7 +113,7 @@ exports = module.exports = {
     _MOCK_GET_BY_IP_APP_ID: ''
 };
 
-var appdb = require('./appdb.js'),
+const appdb = require('./appdb.js'),
     appstore = require('./appstore.js'),
     appTaskManager = require('./apptaskmanager.js'),
     assert = require('assert'),
@@ -115,6 +121,7 @@ var appdb = require('./appdb.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
+    database = require('./database.js'),
     debug = require('debug')('box:apps'),
     docker = require('./docker.js'),
     domaindb = require('./domaindb.js'),
@@ -130,6 +137,7 @@ var appdb = require('./appdb.js'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
     semver = require('semver'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     spawn = require('child_process').spawn,
     split = require('split'),
@@ -411,7 +419,7 @@ function removeInternalFields(app) {
         'location', 'domain', 'fqdn', 'mailboxName', 'mailboxDomain',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'alternateDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts');
+        'label', 'alternateDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts', 'enableMailbox');
 }
 
 // non-admins can only see these
@@ -421,34 +429,20 @@ function removeRestrictedFields(app) {
         'location', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup');
 }
 
-function getIconUrlSync(app) {
-    const iconUrl = '/api/v1/apps/' + app.id + '/icon';
-
-    const userIconPath = `${paths.APP_ICONS_DIR}/${app.id}.user.png`;
-    if (safe.fs.existsSync(userIconPath)) return iconUrl;
-
-    const appstoreIconPath = `${paths.APP_ICONS_DIR}/${app.id}.png`;
-    if (safe.fs.existsSync(appstoreIconPath)) return iconUrl;
-
-    return null;
-}
-
-function getIconPath(app, options, callback) {
+function getIcon(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    const appId = app.id;
+    appdb.getIcons(app.id, function (error, icons) {
+        if (error) return callback(error);
 
-    if (!options.original) {
-        const userIconPath = `${paths.APP_ICONS_DIR}/${appId}.user.png`;
-        if (safe.fs.existsSync(userIconPath)) return callback(null, userIconPath);
-    }
+        if (!options.original && icons.icon) return callback(null, icons.icon);
 
-    const appstoreIconPath = `${paths.APP_ICONS_DIR}/${appId}.png`;
-    if (safe.fs.existsSync(appstoreIconPath)) return callback(null, appstoreIconPath);
+        if (icons.appStoreIcon) return callback(null, icons.appStoreIcon);
 
-    callback(new BoxError(BoxError.NOT_FOUND, 'No icon'));
+        callback(new BoxError(BoxError.NOT_FOUND, 'No icon'));
+    });
 }
 
 function getMemoryLimit(app) {
@@ -471,8 +465,7 @@ function postProcess(app, domainObjectMap) {
         result[portName] = app.portBindings[portName].hostPort;
     }
     app.portBindings = result;
-
-    app.iconUrl = getIconUrlSync(app);
+    app.iconUrl = app.hasIcon || app.hasAppStoreIcon  ? `/api/v1/apps/${app.id}/icon` : null;
     app.fqdn = domains.fqdn(app.location, domainObjectMap[app.domain]);
     app.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
     app.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
@@ -629,7 +622,8 @@ function scheduleTask(appId, installationState, taskId, callback) {
         if (error) return callback(error);
 
         let memoryLimit = 400;
-        if (installationState === exports.ISTATE_PENDING_BACKUP || installationState === exports.ISTATE_PENDING_CLONE || installationState === exports.ISTATE_PENDING_RESTORE) {
+        if (installationState === exports.ISTATE_PENDING_BACKUP || installationState === exports.ISTATE_PENDING_CLONE || installationState === exports.ISTATE_PENDING_RESTORE
+            || installationState === exports.ISTATE_PENDING_IMPORT || installationState === exports.ISTATE_PENDING_UPDATE) {
             memoryLimit = 'memoryLimit' in backupConfig ? Math.max(backupConfig.memoryLimit/1024/1024, 400) : 400;
         }
 
@@ -648,6 +642,8 @@ function scheduleTask(appId, installationState, taskId, callback) {
                 appdb.update(appId, { installationState: exports.ISTATE_ERROR, error: boxError.toPlainObject(), taskId: null }, callback);
             } else if (!(installationState === exports.ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall
                 appdb.update(appId, { taskId: null }, callback);
+            } else {
+                callback(null);
             }
         });
     });
@@ -672,7 +668,7 @@ function addTask(appId, installationState, task, callback) {
             if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_STATE, 'Another task is scheduled for this app')); // could be because app went away OR a taskId exists
             if (error) return callback(error);
 
-            if (scheduleNow) scheduleTask(appId, installationState, taskId, NOOP_CALLBACK);
+            if (scheduleNow) scheduleTask(appId, installationState, taskId, task.onFinished || NOOP_CALLBACK);
 
             callback(null, { taskId });
         });
@@ -690,12 +686,12 @@ function checkAppState(app, state) {
         if (app.error.installationState === state) return null;
 
         // allow uninstall from any state
-        if (state !== exports.ISTATE_PENDING_UNINSTALL && state !== exports.ISTATE_PENDING_RESTORE) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
+        if (state !== exports.ISTATE_PENDING_UNINSTALL && state !== exports.ISTATE_PENDING_RESTORE && state !== exports.ISTATE_PENDING_IMPORT) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
     }
 
     if (app.runState === exports.RSTATE_STOPPED) {
         // can't backup or restore since app addons are down. can't update because migration scripts won't run
-        if (state === exports.ISTATE_PENDING_UPDATE || state === exports.ISTATE_PENDING_BACKUP || state === exports.ISTATE_PENDING_RESTORE)  return new BoxError(BoxError.BAD_STATE, 'Not allowed in stopped state');
+        if (state === exports.ISTATE_PENDING_UPDATE || state === exports.ISTATE_PENDING_BACKUP || state === exports.ISTATE_PENDING_RESTORE || state === exports.ISTATE_PENDING_IMPORT)  return new BoxError(BoxError.BAD_STATE, 'Not allowed in stopped state');
     }
 
     return null;
@@ -736,13 +732,11 @@ function install(data, auditSource, callback) {
 
     assert.strictEqual(typeof data.manifest, 'object'); // manifest is already downloaded
 
-    var location = data.location.toLowerCase(),
+    let location = data.location.toLowerCase(),
         domain = data.domain.toLowerCase(),
         portBindings = data.portBindings || null,
         accessRestriction = data.accessRestriction || null,
         icon = data.icon || null,
-        cert = data.cert || null,
-        key = data.key || null,
         memoryLimit = data.memoryLimit || 0,
         sso = 'sso' in data ? data.sso : null,
         debugMode = data.debugMode || null,
@@ -756,6 +750,7 @@ function install(data, auditSource, callback) {
         overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false,
         skipDnsSetup = 'skipDnsSetup' in data ? data.skipDnsSetup : false,
         appStoreId = data.appStoreId,
+        enableMailbox = 'enabledMailbox' in data ? data.enableMailbox : true,
         manifest = data.manifest;
 
     let error = manifestFormat.parse(manifest);
@@ -797,10 +792,7 @@ function install(data, auditSource, callback) {
 
     if (icon) {
         if (!validator.isBase64(icon)) return callback(new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' }));
-
-        if (!safe.fs.writeFileSync(path.join(paths.APP_ICONS_DIR, appId + '.png'), Buffer.from(icon, 'base64'))) {
-            return callback(new BoxError(BoxError.FS_ERROR, 'Error saving icon:' + safe.error.message));
-        }
+        icon = Buffer.from(icon, 'base64');
     }
 
     const locations = [{ subdomain: location, domain, type: 'primary' }]
@@ -810,14 +802,9 @@ function install(data, auditSource, callback) {
     validateLocations(locations, function (error, domainObjectMap) {
         if (error) return callback(error);
 
-        if (cert && key) {
-            error = reverseProxy.validateCertificate(location, domainObjectMap[domain], { cert, key });
-            if (error) return callback(new BoxError(BoxError.BAD_FIELD, error.message, { field: 'cert' }));
-        }
-
         debug('Will install app with id : ' + appId);
 
-        var data = {
+        const data = {
             accessRestriction,
             memoryLimit,
             sso,
@@ -831,6 +818,8 @@ function install(data, auditSource, callback) {
             env,
             label,
             tags,
+            icon,
+            enableMailbox,
             runState: exports.RSTATE_RUNNING,
             installationState: exports.ISTATE_PENDING_INSTALL
         };
@@ -842,12 +831,6 @@ function install(data, auditSource, callback) {
             purchaseApp({ appId: appId, appstoreId: appStoreId, manifestId: manifest.id || 'customapp' }, function (error) {
                 if (error) return callback(error);
 
-                // save cert to boxdata/certs
-                if (cert && key) {
-                    let error = reverseProxy.setAppCertificateSync(location, domainObjectMap[domain], { cert, key });
-                    if (error) return callback(error);
-                }
-
                 const task = {
                     args: { restoreConfig: null, skipDnsSetup, overwriteDns },
                     values: { },
@@ -857,7 +840,7 @@ function install(data, auditSource, callback) {
                 addTask(appId, data.installationState, task, function (error, result) {
                     if (error) return callback(error);
 
-                    const newApp = _.extend({}, data, { appStoreId, manifest, location, domain, portBindings });
+                    const newApp = _.extend({}, _.omit(data, 'icon'), { appStoreId, manifest, location, domain, portBindings });
                     newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
                     newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
                     newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
@@ -881,7 +864,7 @@ function setAccessRestriction(app, accessRestriction, auditSource, callback) {
     let error = validateAccessRestriction(accessRestriction);
     if (error) return callback(error);
 
-    appdb.update(appId, { accessRestriction: accessRestriction }, function (error) {
+    appdb.update(appId, { accessRestriction }, function (error) {
         if (error) return callback(error);
 
         eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, accessRestriction });
@@ -900,7 +883,7 @@ function setLabel(app, label, auditSource, callback) {
     let error = validateLabel(label);
     if (error) return callback(error);
 
-    appdb.update(appId, { label: label }, function (error) {
+    appdb.update(appId, { label }, function (error) {
         if (error) return callback(error);
 
         eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, label });
@@ -919,7 +902,7 @@ function setTags(app, tags, auditSource, callback) {
     let error = validateTags(tags);
     if (error) return callback(error);
 
-    appdb.update(appId, { tags: tags }, function (error) {
+    appdb.update(appId, { tags }, function (error) {
         if (error) return callback(error);
 
         eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, tags });
@@ -938,17 +921,15 @@ function setIcon(app, icon, auditSource, callback) {
 
     if (icon) {
         if (!validator.isBase64(icon)) return callback(new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' }));
-
-        if (!safe.fs.writeFileSync(path.join(paths.APP_ICONS_DIR, appId + '.user.png'), Buffer.from(icon, 'base64'))) {
-            return callback(new BoxError(BoxError.FS_ERROR, 'Error saving icon:' + safe.error.message));
-        }
-    } else {
-        safe.fs.unlinkSync(path.join(paths.APP_ICONS_DIR, appId + '.user.png'));
+        icon = Buffer.from(icon, 'base64');
     }
 
-    eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, icon });
+    appdb.update(appId, { icon }, function (error) {
+        if (error) return callback(error);
 
-    callback();
+        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, iconChanged: true });
+        callback();
+    });
 }
 
 function setMemoryLimit(app, memoryLimit, auditSource, callback) {
@@ -1079,13 +1060,18 @@ function setDebugMode(app, debugMode, auditSource, callback) {
     });
 }
 
-function setMailbox(app, mailboxName, mailboxDomain, auditSource, callback) {
+function setMailbox(app, data, auditSource, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert(mailboxName === null || typeof mailboxName === 'string');
-    assert.strictEqual(typeof mailboxDomain, 'string');
+    assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    const { enable, mailboxDomain } = data;
+    let mailboxName = data.mailboxName;
+    assert.strictEqual(typeof enable, 'boolean');
+    assert(mailboxName === null || typeof mailboxName === 'string');
+    assert.strictEqual(typeof mailboxDomain, 'string');
+
     const appId = app.id;
     let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER);
     if (error) return callback(error);
@@ -1104,7 +1090,7 @@ function setMailbox(app, mailboxName, mailboxDomain, auditSource, callback) {
 
         const task = {
             args: {},
-            values: { mailboxName, mailboxDomain }
+            values: { enableMailbox: enable, mailboxName, mailboxDomain }
         };
         addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task, function (error, result) {
             if (error) return callback(error);
@@ -1176,28 +1162,30 @@ function setReverseProxyConfig(app, reverseProxyConfig, auditSource, callback) {
     });
 }
 
-function setCertificate(app, bundle, auditSource, callback) {
+function setCertificate(app, data, auditSource, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert(bundle && typeof bundle === 'object');
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     const appId = app.id;
+    const { location, domain, cert, key } = data;
 
-    domains.get(app.domain, function (error, domainObject) {
+    domains.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
-        if (bundle.cert && bundle.key) {
-            error = reverseProxy.validateCertificate(app.location, domainObject, { cert: bundle.cert, key: bundle.key });
-            if (error) return callback(new BoxError(BoxError.BAD_FIELD, error.message, { field: 'cert' }));
+        if (cert && key) {
+            error = reverseProxy.validateCertificate(location, domainObject, { cert, key });
+            if (error) return callback(error);
         }
 
-        error = reverseProxy.setAppCertificateSync(app.location, domainObject, { cert: bundle.cert, key: bundle.key });
-        if (error) return callback(error);
+        reverseProxy.setAppCertificateSync(location, domainObject, { cert, key }, function (error) {
+            if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, cert: bundle.cert, key: bundle.key });
+            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, cert, key });
 
-        callback();
+            callback();
+        });
     });
 }
 
@@ -1286,7 +1274,10 @@ function setDataDir(app, dataDir, auditSource, callback) {
 
     const task = {
         args: { newDataDir: dataDir },
-        values: { }
+        values: { },
+        onFinished: (error) => {
+            if (!error) services.rebuildService('sftp', NOOP_CALLBACK);
+        }
     };
     addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, function (error, result) {
         if (error) return callback(error);
@@ -1340,13 +1331,9 @@ function update(app, data, auditSource, callback) {
     if ('icon' in data) {
         if (data.icon) {
             if (!validator.isBase64(data.icon)) return callback(new BoxError(BoxError.BAD_FIELD, 'icon is not base64', { field: 'icon' }));
-
-            if (!safe.fs.writeFileSync(path.join(paths.APP_ICONS_DIR, appId + '.user.png'), Buffer.from(data.icon, 'base64'))) {
-                return callback(new BoxError(BoxError.FS_ERROR, 'Error saving icon:' + safe.error.message));
-            }
-        } else {
-            safe.fs.unlinkSync(path.join(paths.APP_ICONS_DIR, appId + '.user.png'));
+            data.icon = Buffer.from(data.icon, 'base64');
         }
+        values.icon = data.icon;
     }
 
     // do not update apps in debug mode
@@ -1367,7 +1354,10 @@ function update(app, data, auditSource, callback) {
 
     const task = {
         args: { updateConfig },
-        values
+        values,
+        onFinished: (error) => {
+            eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditSource, { app, success: !error, errorMessage: error ? error.message : null }, () => {}); // ignore error
+        }
     };
     addTask(appId, exports.ISTATE_PENDING_UPDATE, task, function (error, result) {
         if (error) return callback(error);
@@ -1438,6 +1428,15 @@ function getLogs(app, options, callback) {
     return callback(null, transformStream);
 }
 
+async function getCertificate(subdomain, domain) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domain, 'string');
+
+    const result = await database.query('SELECT certificateJson FROM subdomains WHERE subdomain=? AND domain=?', [ subdomain, domain ]);
+    if (result.length === 0) return null;
+    return JSON.parse(result[0].certificateJson);
+}
+
 // does a re-configure when called from most states. for install/clone errors, it re-installs with an optional manifest
 // re-configure can take a dockerImage but not a manifest because re-configure does not clean up addons
 function repair(app, data, auditSource, callback) {
@@ -1562,7 +1561,7 @@ function importApp(app, data, auditSource, callback) {
     let error = backupFormat ? validateBackupFormat(backupFormat) : null;
     if (error) return callback(error);
 
-    error = checkAppState(app, exports.ISTATE_PENDING_RESTORE);
+    error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
     if (error) return callback(error);
 
     // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
@@ -1591,10 +1590,10 @@ function importApp(app, data, auditSource, callback) {
             },
             values: {}
         };
-        addTask(appId, exports.ISTATE_PENDING_RESTORE, task, function (error, result) {
+        addTask(appId, exports.ISTATE_PENDING_IMPORT, task, function (error, result) {
             if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId: result.taskId });
+            eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app: app, backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId: result.taskId });
 
             callback(null, { taskId: result.taskId });
         });
@@ -1644,7 +1643,7 @@ function clone(app, data, user, auditSource, callback) {
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var location = data.location.toLowerCase(),
+    const location = data.location.toLowerCase(),
         domain = data.domain.toLowerCase(),
         portBindings = data.portBindings || null,
         backupId = data.backupId,
@@ -1680,47 +1679,58 @@ function clone(app, data, user, auditSource, callback) {
         validateLocations(locations, function (error, domainObjectMap) {
             if (error) return callback(error);
 
-            var newAppId = uuid.v4();
+            const newAppId = uuid.v4();
 
-            var data = {
-                installationState: exports.ISTATE_PENDING_CLONE,
-                runState: exports.RSTATE_RUNNING,
-                memoryLimit: app.memoryLimit,
-                accessRestriction: app.accessRestriction,
-                sso: !!app.sso,
-                mailboxName: mailboxName,
-                mailboxDomain: mailboxDomain,
-                enableBackup: app.enableBackup,
-                reverseProxyConfig: app.reverseProxyConfig,
-                env: app.env,
-                alternateDomains: [],
-                aliasDomains: []
-            };
-
-            appdb.add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) {
-                if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, locations, domainObjectMap, portBindings));
+            appdb.getIcons(app.id, function (error, icons) {
                 if (error) return callback(error);
 
-                purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' }, function (error) {
+                const data = {
+                    installationState: exports.ISTATE_PENDING_CLONE,
+                    runState: exports.RSTATE_RUNNING,
+                    memoryLimit: app.memoryLimit,
+                    cpuShares: app.cpuShares,
+                    accessRestriction: app.accessRestriction,
+                    sso: !!app.sso,
+                    mailboxName: mailboxName,
+                    mailboxDomain: mailboxDomain,
+                    enableBackup: app.enableBackup,
+                    reverseProxyConfig: app.reverseProxyConfig,
+                    env: app.env,
+                    alternateDomains: [],
+                    aliasDomains: [],
+                    servicesConfig: app.servicesConfig,
+                    label: app.label ? `${app.label}-clone` : '',
+                    tags: app.tags,
+                    enableAutomaticUpdate: app.enableAutomaticUpdate,
+                    icon: icons.icon,
+                    enableMailbox: app.enableMailbox
+                };
+
+                appdb.add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) {
+                    if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(error.message, locations, domainObjectMap, portBindings));
                     if (error) return callback(error);
 
-                    const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format };
-                    const task = {
-                        args: { restoreConfig, overwriteDns, skipDnsSetup, oldManifest: null },
-                        values: {},
-                        requiredState: exports.ISTATE_PENDING_CLONE
-                    };
-                    addTask(newAppId, exports.ISTATE_PENDING_CLONE, task, function (error, result) {
+                    purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' }, function (error) {
                         if (error) return callback(error);
 
-                        const newApp = _.extend({}, data, { appStoreId, manifest, location, domain, portBindings });
-                        newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
-                        newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
-                        newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+                        const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format };
+                        const task = {
+                            args: { restoreConfig, overwriteDns, skipDnsSetup, oldManifest: null },
+                            values: {},
+                            requiredState: exports.ISTATE_PENDING_CLONE
+                        };
+                        addTask(newAppId, exports.ISTATE_PENDING_CLONE, task, function (error, result) {
+                            if (error) return callback(error);
 
-                        eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, oldApp: app, newApp: newApp, taskId: result.taskId });
+                            const newApp = _.extend({}, _.omit(data, 'icon'), { appStoreId, manifest, location, domain, portBindings });
+                            newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
+                            newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+                            newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
-                        callback(null, { id: newAppId, taskId: result.taskId });
+                            eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, oldApp: app, newApp: newApp, taskId: result.taskId });
+
+                            callback(null, { id: newAppId, taskId: result.taskId });
+                        });
                     });
                 });
             });
@@ -1844,8 +1854,8 @@ function exec(app, options, callback) {
     assert(options && typeof options === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var cmd = options.cmd || [ '/bin/bash' ];
-    assert(util.isArray(cmd) && cmd.length > 0);
+    let cmd = options.cmd || [ '/bin/bash' ];
+    assert(Array.isArray(cmd) && cmd.length > 0);
 
     if (app.installationState !== exports.ISTATE_INSTALLED || app.runState !== exports.RSTATE_RUNNING) {
         return callback(new BoxError(BoxError.BAD_STATE, 'App not installed or running'));
@@ -2200,3 +2210,34 @@ function uploadFile(app, sourceFilePath, destFilePath, callback) {
         readFile.pipe(stream);
     });
 }
+
+function backupConfig(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json'), JSON.stringify(app))) {
+        return callback(new BoxError(BoxError.FS_ERROR, 'Error creating config.json: ' + safe.error.message));
+    }
+
+    appdb.getIcons(app.id, function (error, icons) {
+        if (!error && icons.icon) safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/icon.png'), icons.icon);
+
+        callback(null);
+    });
+}
+
+function restoreConfig(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const appConfig = safe.JSON.parse(safe.fs.readFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json')));
+    let data = {};
+    if (appConfig) {
+        data = _.pick(appConfig, 'memoryLimit', 'cpuShares', 'enableBackup', 'reverseProxyConfig', 'env', 'servicesConfig', 'label', 'tags', 'enableAutomaticUpdate');
+    }
+
+    const icon = safe.fs.readFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/icon.png'));
+    if (icon) data.icon = icon;
+
+    appdb.update(app.id, data, callback);
+}

src/appstore.js

@@ -7,10 +7,8 @@ exports = module.exports = {
     getApp,
     getAppVersion,
 
-    trackBeginSetup,
-    trackFinishedSetup,
-
     registerWithLoginCredentials,
+    updateCloudron,
 
     purchaseApp,
     unpurchaseApp,
@@ -343,30 +341,29 @@ function registerCloudron(data, callback) {
     });
 }
 
-// This works without a Cloudron token as this Cloudron was not yet registered
-let gBeginSetupAlreadyTracked = false;
-function trackBeginSetup() {
-    // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
-    if (gBeginSetupAlreadyTracked) return;
-    gBeginSetupAlreadyTracked = true;
+function updateCloudron(data, callback) {
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
+    getCloudronToken(function (error, token) {
+        if (error && error.reason === BoxError.LICENSE_ERROR) return callback(null); // missing token. not registered yet
+        if (error) return callback(error);
 
-    superagent.post(url).send({}).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return debug(`trackBeginSetup: ${error.message}`);
-        if (result.statusCode !== 200) return debug(`trackBeginSetup: ${result.statusCode} ${error.message}`);
-    });
-}
+        const url = `${settings.apiServerOrigin()}/api/v1/update_cloudron`;
+        const query = {
+            accessToken: token
+        };
 
-// This works without a Cloudron token as this Cloudron was not yet registered
-function trackFinishedSetup(domain) {
-    assert.strictEqual(typeof domain, 'string');
+        superagent.post(url).query(query).send(data).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;
+            debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
 
-    superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return debug(`trackFinishedSetup: ${error.message}`);
-        if (result.statusCode !== 200) return debug(`trackFinishedSetup: ${result.statusCode} ${error.message}`);
+            callback();
+        });
     });
 }
 
@@ -389,7 +386,7 @@ function registerWithLoginCredentials(options, callback) {
             login(options.email, options.password, options.totpToken || '', function (error, result) {
                 if (error) return callback(error);
 
-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
+                registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
             });
         });
     });

src/apptask.js

@@ -6,8 +6,6 @@ exports = module.exports = {
     run,
 
     // exported for testing
-    _configureReverseProxy: configureReverseProxy,
-    _unconfigureReverseProxy: unconfigureReverseProxy,
     _createAppDir: createAppDir,
     _deleteAppDir: deleteAppDir,
     _verifyManifest: verifyManifest,
@@ -18,7 +16,6 @@ const appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
-    auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     collectd = require('./collectd.js'),
@@ -28,7 +25,6 @@ const appdb = require('./appdb.js'),
     docker = require('./docker.js'),
     domains = require('./domains.js'),
     ejs = require('ejs'),
-    eventlog = require('./eventlog.js'),
     fs = require('fs'),
     iputils = require('./iputils.js'),
     manifestFormat = require('cloudron-manifestformat'),
@@ -73,8 +69,6 @@ function updateApp(app, values, callback) {
     assert.strictEqual(typeof values, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debugApp(app, 'updating app with values: %j', values);
-
     appdb.update(app.id, values, function (error) {
         if (error) return callback(error);
 
@@ -269,7 +263,7 @@ function cleanupLogs(app, callback) {
 
     // note that redis container logs are cleaned up by the addon
     rimraf(path.join(paths.LOG_DIR, app.id), function (error) {
-        if (error) debugApp(app, 'cannot cleanup logs: %s', error);
+        if (error) debugApp(app, 'cannot cleanup logs:', error);
 
         callback(null);
     });
@@ -295,9 +289,9 @@ function downloadIcon(app, callback) {
     // nothing to download if we dont have an appStoreId
     if (!app.appStoreId) return callback(null);
 
-    debugApp(app, 'Downloading icon of %s@%s', app.appStoreId, app.manifest.version);
+    debugApp(app, `Downloading icon of ${app.appStoreId}@${app.manifest.version}`);
 
-    var iconUrl = settings.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';
+    const iconUrl = settings.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';
 
     async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
         superagent
@@ -308,29 +302,11 @@ function downloadIcon(app, callback) {
                 if (error && !error.response) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error downloading icon : ${error.message}`));
                 if (res.statusCode !== 200) return retryCallback(null); // ignore error. this can also happen for apps installed with cloudron-cli
 
-                const iconPath = path.join(paths.APP_ICONS_DIR, app.id + '.png');
-                if (!safe.fs.writeFileSync(iconPath, res.body)) return retryCallback(new BoxError(BoxError.FS_ERROR, `Error saving icon to ${iconPath}: ${safe.error.message}`));
-
-                retryCallback(null);
+                updateApp(app, { appStoreIcon: res.body }, retryCallback);
             });
     }, callback);
 }
 
-function removeIcon(app, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!safe.fs.unlinkSync(path.join(paths.APP_ICONS_DIR, app.id + '.png'))) {
-        if (safe.error.code !== 'ENOENT') debugApp(app, 'cannot remove icon : %s', safe.error);
-    }
-
-    if (!safe.fs.unlinkSync(path.join(paths.APP_ICONS_DIR, app.id + '.user.png'))) {
-        if (safe.error.code !== 'ENOENT') debugApp(app, 'cannot remove user icon : %s', safe.error);
-    }
-
-    callback(null);
-}
-
 function waitForDnsPropagation(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -483,6 +459,7 @@ function install(app, args, progressCallback, callback) {
                     progressCallback.bind(null, { percent: 60, message: 'Importing addons in-place' }),
                     services.setupAddons.bind(null, app, app.manifest.addons),
                     services.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
+                    apps.restoreConfig.bind(null, app),
                     services.restoreAddons.bind(null, app, app.manifest.addons),
                 ], next);
             } else {
@@ -493,12 +470,14 @@ function install(app, args, progressCallback, callback) {
                     backups.downloadApp.bind(null, app, restoreConfig, (progress) => {
                         progressCallback({ percent: 65, message: progress.message });
                     }),
+                    (done) => { if (app.installationState === apps.ISTATE_PENDING_IMPORT) apps.restoreConfig(app, done); else done(); },
+                    progressCallback.bind(null, { percent: 70, message: 'Restoring addons' }),
                     services.restoreAddons.bind(null, app, app.manifest.addons)
                 ], next);
             }
         },
 
-        progressCallback.bind(null, { percent: 70, message: 'Creating container' }),
+        progressCallback.bind(null, { percent: 80, message: 'Creating container' }),
         createContainer.bind(null, app),
 
         startApp.bind(null, app),
@@ -519,7 +498,7 @@ function install(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error installing app: %s', error);
+            debugApp(app, 'error installing app:', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -542,7 +521,7 @@ function backup(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error backing up app: %s', error);
+            debugApp(app, 'error backing up app:', error);
             // return to installed state intentionally. the error is stashed only in the task and not the app (the UI shows error state otherwise)
             return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }, callback.bind(null, makeTaskError(error, app)));
         }
@@ -573,7 +552,7 @@ function create(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error creating : %s', error);
+            debugApp(app, 'error creating :', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -647,7 +626,7 @@ function changeLocation(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error changing location : %s', error);
+            debugApp(app, 'error changing location:', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -686,7 +665,7 @@ function migrateDataDir(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error migrating data dir : %s', error);
+            debugApp(app, 'error migrating data dir:', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
 
@@ -731,7 +710,7 @@ function configure(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error reconfiguring : %s', error);
+            debugApp(app, 'error reconfiguring:', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
 
@@ -842,10 +821,10 @@ function update(app, args, progressCallback, callback) {
             debugApp(app, 'update aborted because backup failed', error);
             updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, callback.bind(null, error));
         } else if (error) {
-            debugApp(app, 'Error updating app: %s', error);
+            debugApp(app, 'Error updating app:', error);
             updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         } else {
-            eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditSource.APP_TASK, { app: app, success: true }, () => callback()); // ignore error
+            callback(null);
         }
     });
 }
@@ -874,7 +853,7 @@ function start(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error starting app: %s', error);
+            debugApp(app, 'error starting app:', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -901,7 +880,7 @@ function stop(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error starting app: %s', error);
+            debugApp(app, 'error starting app:', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -922,7 +901,7 @@ function restart(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error starting app: %s', error);
+            debugApp(app, 'error starting app:', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -954,9 +933,6 @@ function uninstall(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 70, message: 'Unregistering domains' }),
         domains.unregisterLocations.bind(null, [ { subdomain: app.location, domain: app.domain } ].concat(app.alternateDomains).concat(app.aliasDomains), progressCallback),
 
-        progressCallback.bind(null, { percent: 80, message: 'Cleanup icon' }),
-        removeIcon.bind(null, app),
-
         progressCallback.bind(null, { percent: 90, message: 'Cleanup logs' }),
         cleanupLogs.bind(null, app),
 
@@ -964,7 +940,7 @@ function uninstall(app, args, progressCallback, callback) {
         appdb.del.bind(null, app.id)
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error uninstalling app: %s', error);
+            debugApp(app, 'error uninstalling app:', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -981,12 +957,13 @@ function run(appId, args, progressCallback, callback) {
     apps.get(appId, function (error, app) {
         if (error) return callback(error);
 
-        debugApp(app, 'startTask installationState: %s runState: %s', app.installationState, app.runState);
+        debugApp(app, `startTask installationState: ${app.installationState} runState: ${app.runState}`);
 
         switch (app.installationState) {
         case apps.ISTATE_PENDING_INSTALL:
         case apps.ISTATE_PENDING_CLONE:
         case apps.ISTATE_PENDING_RESTORE:
+        case apps.ISTATE_PENDING_IMPORT:
             return install(app, args, progressCallback, callback);
         case apps.ISTATE_PENDING_CONFIGURE:
             return configure(app, args, progressCallback, callback);

src/apptaskmanager.js

@@ -4,7 +4,7 @@ exports = module.exports = {
     scheduleTask
 };
 
-let assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:apptaskmanager'),
     fs = require('fs'),
@@ -13,7 +13,6 @@ let assert = require('assert'),
     path = require('path'),
     paths = require('./paths.js'),
     scheduler = require('./scheduler.js'),
-    services = require('./services.js'),
     tasks = require('./tasks.js');
 
 let gActiveTasks = { }; // indexed by app id
@@ -79,8 +78,6 @@ function scheduleTask(appId, taskId, options, callback) {
         delete gActiveTasks[appId];
         locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
 
-        // post app task hooks
-        services.rebuildService('sftp', error => { if (error) debug('Unable to rebuild sftp:', error); });
         scheduler.resumeJobs(appId);
     });
 }
@@ -93,4 +90,3 @@ function startNextTask() {
     const t = gPendingTasks.shift();
     scheduleTask(t.appId, t.taskId, t.options, t.callback);
 }
-

src/auditsource.js

@@ -3,14 +3,13 @@
 exports = module.exports = {
     CRON:  { userId: null, username: 'cron' },
     HEALTH_MONITOR: { userId: null, username: 'healthmonitor' },
-    APP_TASK: { userId: null, username: 'apptask' },
     EXTERNAL_LDAP_TASK: { userId: null, username: 'externalldap' },
     EXTERNAL_LDAP_AUTO_CREATE: { userId: null, username: 'externalldap' },
 
-    fromRequest: fromRequest
+    fromRequest
 };
 
 function fromRequest(req) {
-    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+    const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
 }

src/backupdb.js

@@ -1,12 +1,11 @@
 'use strict';
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     database = require('./database.js'),
-    safe = require('safetydance'),
-    util = require('util');
+    safe = require('safetydance');
 
-var BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
+const BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
 
 exports = module.exports = {
     add,
@@ -119,7 +118,7 @@ function add(id, data, callback) {
     assert.strictEqual(typeof data.type, 'string');
     assert.strictEqual(typeof data.identifier, 'string');
     assert.strictEqual(typeof data.state, 'string');
-    assert(util.isArray(data.dependsOn));
+    assert(Array.isArray(data.dependsOn));
     assert.strictEqual(typeof data.manifest, 'object');
     assert.strictEqual(typeof data.format, 'string');
     assert.strictEqual(typeof callback, 'function');

src/backups.js

@@ -26,11 +26,10 @@ exports = module.exports = {
     injectPrivateFields,
     removePrivateFields,
 
-    checkConfiguration,
-
     configureCollectd,
 
     generateEncryptionKeysSync,
+    isMountProvider,
 
     BACKUP_IDENTIFIER_BOX: 'box',
 
@@ -96,6 +95,8 @@ function api(provider) {
     case 'nfs': return require('./storage/filesystem.js');
     case 'cifs': return require('./storage/filesystem.js');
     case 'sshfs': return require('./storage/filesystem.js');
+    case 'mountpoint': return require('./storage/filesystem.js');
+    case 'ext4': return require('./storage/filesystem.js');
     case 's3': return require('./storage/s3.js');
     case 'gcs': return require('./storage/gcs.js');
     case 'filesystem': return require('./storage/filesystem.js');
@@ -109,11 +110,16 @@ function api(provider) {
     case 'linode-objectstorage': return require('./storage/s3.js');
     case 'ovh-objectstorage': return require('./storage/s3.js');
     case 'ionos-objectstorage': return require('./storage/s3.js');
+    case 'vultr-objectstorage': return require('./storage/s3.js');
     case 'noop': return require('./storage/noop.js');
     default: return null;
     }
 }
 
+function isMountProvider(provider) {
+    return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4';
+}
+
 function injectPrivateFields(newConfig, currentConfig) {
     if ('password' in newConfig) {
         if (newConfig.password === constants.SECRET_PLACEHOLDER) {
@@ -139,7 +145,7 @@ function testConfig(backupConfig, callback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var func = api(backupConfig.provider);
+    const func = api(backupConfig.provider);
     if (!func) return callback(new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' }));
 
     if (backupConfig.format !== 'tgz' && backupConfig.format !== 'rsync') return callback(new BoxError(BoxError.BAD_FIELD, 'unknown format', { field: 'format' }));
@@ -986,10 +992,7 @@ function rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallb
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    var snapshotInfo = getSnapshotInfo('box');
-
-    const snapshotTime = snapshotInfo.timestamp.replace(/[T.]/g, '-').replace(/[:Z]/g,''); // add this to filename to make it unique, so it's easy to download them
-    const backupId = util.format('%s/box_%s_v%s', tag, snapshotTime, constants.VERSION);
+    const backupId = `${tag}/box_v${constants.VERSION}`;
     const format = backupConfig.format;
 
     debug(`Rotating box backup to id ${backupId}`);
@@ -1064,16 +1067,16 @@ function snapshotApp(app, progressCallback, callback) {
     const startTime = new Date();
     progressCallback({ message: `Snapshotting app ${app.fqdn}` });
 
-    if (!safe.fs.writeFileSync(path.join(paths.APPS_DATA_DIR, app.id + '/config.json'), JSON.stringify(app))) {
-        return callback(new BoxError(BoxError.FS_ERROR, 'Error creating config.json: ' + safe.error.message));
-    }
+    apps.backupConfig(app, function (error) {
+        if (error) return callback(error);
 
-    services.backupAddons(app, app.manifest.addons, function (error) {
-        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+        services.backupAddons(app, app.manifest.addons, function (error) {
+            if (error) return callback(error);
 
-        debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`);
+            debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`);
 
-        return callback(null);
+            return callback(null);
+        });
     });
 }
 
@@ -1087,11 +1090,10 @@ function rotateAppBackup(backupConfig, app, tag, options, progressCallback, call
 
     const startTime = new Date();
 
-    var snapshotInfo = getSnapshotInfo(app.id);
+    const snapshotInfo = getSnapshotInfo(app.id);
 
-    var manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat
-    const snapshotTime = snapshotInfo.timestamp.replace(/[T.]/g, '-').replace(/[:Z]/g,''); // add this for unique filename which helps when downloading them
-    const backupId = util.format('%s/app_%s_%s_v%s', tag, app.id, snapshotTime, manifest.version);
+    const manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat
+    const backupId = `${tag}/app_${app.fqdn}_v${manifest.version}`;
     const format = backupConfig.format;
 
     debug(`Rotating app backup of ${app.id} to id ${backupId}`);
@@ -1110,7 +1112,7 @@ function rotateAppBackup(backupConfig, app, tag, options, progressCallback, call
     backupdb.add(backupId, data, function (error) {
         if (error) return callback(error);
 
-        var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format));
+        const copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format));
         copy.on('progress', (message) => progressCallback({ message: `${message} (${app.fqdn})` }));
         copy.on('done', function (copyBackupError) {
             const state = copyBackupError ? exports.BACKUP_STATE_ERROR : exports.BACKUP_STATE_NORMAL;
@@ -1268,13 +1270,13 @@ function startBackupTask(auditSource, callback) {
 
             eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { taskId });
 
-            tasks.startTask(taskId, { timeout: 24 * 60 * 60 * 1000 /* 24 hours */, nice: 15, memoryLimit }, function (error, backupId) {
+            tasks.startTask(taskId, { timeout: 24 * 60 * 60 * 1000 /* 24 hours */, nice: 15, memoryLimit }, async function (error, backupId) {
                 locker.unlock(locker.OP_FULL_BACKUP);
 
                 const errorMessage = error ? error.message : '';
                 const timedOut = error ? error.code === tasks.ETIMEOUT : false;
 
-                eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId, errorMessage, timedOut, backupId });
+                await safe(eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId, errorMessage, timedOut, backupId }));
             });
 
             callback(null, taskId);
@@ -1457,6 +1459,8 @@ function cleanupMissingBackups(backupConfig, progressCallback, callback) {
 
     let page = 1, perPage = 1000, more = false, missingBackupIds = [];
 
+    if (constants.TEST) return callback(null, missingBackupIds);
+
     async.doWhilst(function (whilstCallback) {
         backupdb.list(page, perPage, function (error, result) {
             if (error) return whilstCallback(error);
@@ -1598,23 +1602,6 @@ function startCleanupTask(auditSource, callback) {
     });
 }
 
-function checkConfiguration(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    settings.getBackupConfig(function (error, backupConfig) {
-        if (error) return callback(error);
-
-        let message = '';
-        if (backupConfig.provider === 'noop') {
-            message = 'Cloudron backups are disabled. Please ensure this server is backed up using alternate means. See https://docs.cloudron.io/backups/#storage-providers for more information.';
-        } else if (backupConfig.provider === 'filesystem' && !backupConfig.externalDisk) {
-            message = 'Cloudron backups are currently on the same disk as the Cloudron server instance. This is dangerous and can lead to complete data loss if the disk fails. See https://docs.cloudron.io/backups/#storage-providers for storing backups in an external location.';
-        }
-
-        callback(null, message);
-    });
-}
-
 function configureCollectd(backupConfig, callback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof callback, 'function');

src/blobs.js

@@ -0,0 +1,102 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    get,
+    set,
+    del,
+
+    initSecrets,
+
+    ACME_ACCOUNT_KEY: 'acme_account_key',
+    ADDON_TURN_SECRET: 'addon_turn_secret',
+    DHPARAMS: 'dhparams',
+    SFTP_PUBLIC_KEY: 'sftp_public_key',
+    SFTP_PRIVATE_KEY: 'sftp_private_key',
+
+    CERT_PREFIX: 'cert',
+
+    _clear: clear
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    constants = require('./constants.js'),
+    crypto = require('crypto'),
+    database = require('./database.js'),
+    debug = require('debug')('box:blobs'),
+    paths = require('./paths.js'),
+    safe = require('safetydance');
+
+const BLOBS_FIELDS = [ 'id', 'value' ].join(',');
+
+async function get(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query(`SELECT ${BLOBS_FIELDS} FROM blobs WHERE id = ?`, [ id ]);
+    if (result.length === 0) return null;
+    return result[0].value;
+}
+
+async function set(id, value) {
+    assert.strictEqual(typeof id, 'string');
+    assert(value === null || Buffer.isBuffer(value));
+
+    await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, value ]);
+}
+
+async function del(id) {
+    await database.query('DELETE FROM blobs WHERE id=?', [ id ]);
+}
+
+async function clear() {
+    await database.query('DELETE FROM blobs');
+}
+
+async function initSecrets() {
+    let acmeAccountKey = await get(exports.ACME_ACCOUNT_KEY);
+    if (!acmeAccountKey) {
+        acmeAccountKey = safe.child_process.execSync('openssl genrsa 4096');
+        if (!acmeAccountKey) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate acme account key: ${safe.error.message}`);
+        await set(exports.ACME_ACCOUNT_KEY, acmeAccountKey);
+    }
+
+    let turnSecret = await get(exports.ADDON_TURN_SECRET);
+    if (!turnSecret) {
+        turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter
+        await set(exports.ADDON_TURN_SECRET, Buffer.from(turnSecret));
+    }
+
+    if (!constants.TEST) {
+        let dhparams = await get(exports.DHPARAMS);
+        if (!dhparams) {
+            debug('initSecrets: generating dhparams.pem. this takes forever');
+            dhparams = safe.child_process.execSync('openssl dhparam 2048');
+            if (!dhparams) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+            if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
+            await set(exports.DHPARAMS, dhparams);
+        } else if (!safe.fs.existsSync(paths.DHPARAMS_FILE)) {
+            if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
+        }
+    }
+
+    let sftpPrivateKey = await get(exports.SFTP_PRIVATE_KEY);
+    let sftpPublicKey = await get(exports.SFTP_PUBLIC_KEY);
+
+    if (!sftpPrivateKey || !sftpPublicKey) {
+        debug('initSecrets: generate sftp keys');
+        if (constants.TEST) {
+            safe.fs.unlinkSync(paths.SFTP_PUBLIC_KEY_FILE);
+            safe.fs.unlinkSync(paths.SFTP_PRIVATE_KEY_FILE);
+        }
+        if (!safe.child_process.execSync(`ssh-keygen -m PEM -t rsa -f "${paths.SFTP_KEYS_DIR}/ssh_host_rsa_key" -q -N ""`)) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate sftp ssh keys: ${safe.error.message}`);
+        sftpPublicKey = safe.fs.readFileSync(paths.SFTP_PUBLIC_KEY_FILE);
+        await set(exports.SFTP_PUBLIC_KEY, sftpPublicKey);
+        sftpPrivateKey = safe.fs.readFileSync(paths.SFTP_PRIVATE_KEY_FILE);
+        await set(exports.SFTP_PRIVATE_KEY, sftpPrivateKey);
+    } else if (!safe.fs.existsSync(paths.SFTP_PUBLIC_KEY_FILE) || !safe.fs.existsSync(paths.SFTP_PRIVATE_KEY_FILE)) {
+        if (!safe.fs.writeFileSync(paths.SFTP_PUBLIC_KEY_FILE, sftpPublicKey)) throw new BoxError(BoxError.FS_ERROR, `Could not save sftp public key: ${safe.error.message}`);
+        if (!safe.fs.writeFileSync(paths.SFTP_PRIVATE_KEY_FILE, sftpPrivateKey)) throw new BoxError(BoxError.FS_ERROR, `Could not save sftp private key: ${safe.error.message}`);
+    }
+}

src/boxerror.js

@@ -9,17 +9,17 @@ const assert = require('assert'),
 
 exports = module.exports = BoxError;
 
-function BoxError(reason, errorOrMessage, details) {
+function BoxError(reason, errorOrMessage, override) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
-    assert(typeof details === 'object' || typeof details === 'undefined');
+    assert(typeof override === 'object' || typeof override === 'undefined');
 
     Error.call(this);
     Error.captureStackTrace(this, this.constructor);
 
     this.name = this.constructor.name;
     this.reason = reason;
-    this.details = details || {};
+    this.details = {};
 
     if (typeof errorOrMessage === 'undefined') {
         this.message = reason;
@@ -28,7 +28,7 @@ function BoxError(reason, errorOrMessage, details) {
     } else { // error object
         this.message = errorOrMessage.message;
         this.nestedError = errorOrMessage;
-        _.extend(this.details, errorOrMessage); // copy enumerable properies
+        _.extend(this, override); // copy enumerable properies
     }
 }
 util.inherits(BoxError, Error);
@@ -47,13 +47,14 @@ BoxError.DOCKER_ERROR = 'Docker Error';
 BoxError.EXTERNAL_ERROR = 'External Error'; // use this for external API errors
 BoxError.FEATURE_DISABLED = 'Feature Disabled';
 BoxError.FS_ERROR = 'FileSystem Error';
-BoxError.INACTIVE = 'Inactive';
+BoxError.INACTIVE = 'Inactive'; // service/volume/mount
 BoxError.INTERNAL_ERROR = 'Internal Error';
 BoxError.INVALID_CREDENTIALS = 'Invalid Credentials';
 BoxError.IPTABLES_ERROR = 'IPTables Error';
 BoxError.LICENSE_ERROR = 'License Error';
 BoxError.LOGROTATE_ERROR = 'Logrotate Error';
 BoxError.MAIL_ERROR = 'Mail Error';
+BoxError.MOUNT_ERROR = 'Mount Error';
 BoxError.NETWORK_ERROR = 'Network Error';
 BoxError.NGINX_ERROR = 'Nginx Error';
 BoxError.NOT_FOUND = 'Not found';
@@ -90,6 +91,7 @@ BoxError.toHttpError = function (error) {
     case BoxError.EXTERNAL_ERROR:
     case BoxError.NETWORK_ERROR:
     case BoxError.FS_ERROR:
+    case BoxError.MOUNT_ERROR:
     case BoxError.MAIL_ERROR:
     case BoxError.DOCKER_ERROR:
     case BoxError.ADDONS_ERROR:

src/cert/acme2.js

@@ -1,628 +0,0 @@
-'use strict';
-
-var assert = require('assert'),
-    async = require('async'),
-    BoxError = require('../boxerror.js'),
-    crypto = require('crypto'),
-    debug = require('debug')('box:cert/acme2'),
-    domains = require('../domains.js'),
-    fs = require('fs'),
-    path = require('path'),
-    paths = require('../paths.js'),
-    request = require('request'),
-    safe = require('safetydance'),
-    util = require('util'),
-    _ = require('underscore');
-
-const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
-    CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory';
-
-exports = module.exports = {
-    getCertificate: getCertificate,
-
-    // testing
-    _name: 'acme',
-    _getChallengeSubdomain: getChallengeSubdomain
-};
-
-// http://jose.readthedocs.org/en/latest/
-// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
-// https://community.letsencrypt.org/t/list-of-client-implementations/2103
-
-function Acme2(options) {
-    assert.strictEqual(typeof options, 'object');
-
-    this.accountKeyPem = null; // Buffer
-    this.email = options.email;
-    this.keyId = null;
-    this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
-    this.directory = {};
-    this.performHttpAuthorization = !!options.performHttpAuthorization;
-    this.wildcard = !!options.wildcard;
-}
-
-// urlsafe base64 encoding (jose)
-function urlBase64Encode(string) {
-    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
-}
-
-function b64(str) {
-    var buf = util.isBuffer(str) ? str : Buffer.from(str);
-    return urlBase64Encode(buf.toString('base64'));
-}
-
-function getModulus(pem) {
-    assert(util.isBuffer(pem));
-
-    var stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
-    if (!stdout) return null;
-    var match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
-    if (!match) return null;
-    return Buffer.from(match[1], 'hex');
-}
-
-Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
-    assert.strictEqual(typeof url, 'string');
-    assert.strictEqual(typeof payload, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    assert(util.isBuffer(this.accountKeyPem));
-
-    const that = this;
-    let header = {
-        url: url,
-        alg: 'RS256'
-    };
-
-    // keyId is null when registering account
-    if (this.keyId) {
-        header.kid = this.keyId;
-    } else {
-        header.jwk = {
-            e: b64(Buffer.from([0x01, 0x00, 0x01])), // exponent - 65537
-            kty: 'RSA',
-            n: b64(getModulus(this.accountKeyPem))
-        };
-    }
-
-    var payload64 = b64(payload);
-
-    request.get(this.directory.newNonce, { json: true, timeout: 30000 }, function (error, response) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`));
-        if (response.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching nonce : ' + response.statusCode));
-
-        const nonce = response.headers['Replay-Nonce'.toLowerCase()];
-        if (!nonce) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response'));
-
-        debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
-
-        var protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
-
-        var signer = crypto.createSign('RSA-SHA256');
-        signer.update(protected64 + '.' + payload64, 'utf8');
-        var signature64 = urlBase64Encode(signer.sign(that.accountKeyPem, 'base64'));
-
-        var data = {
-            protected: protected64,
-            payload: payload64,
-            signature: signature64
-        };
-
-        request.post(url, { headers: { 'Content-Type': 'application/jose+json', 'User-Agent': 'acme-cloudron' }, body: JSON.stringify(data), timeout: 30000 }, function (error, response) {
-            if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`)); // network error
-
-            // we don't set json: true in request because it ends up mangling the content-type
-            // we don't set json: true in request because it ends up mangling the content-type
-            if (response.headers['content-type'] === 'application/json') response.body = safe.JSON.parse(response.body);
-
-            callback(null, response);
-        });
-    });
-};
-
-// https://tools.ietf.org/html/rfc8555#section-6.3
-Acme2.prototype.postAsGet = function (url, callback) {
-    this.sendSignedRequest(url, '', callback);
-};
-
-Acme2.prototype.updateContact = function (registrationUri, callback) {
-    assert.strictEqual(typeof registrationUri, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
-
-    // https://github.com/ietf-wg-acme/acme/issues/30
-    const payload = {
-        contact: [ 'mailto:' + this.email ]
-    };
-
-    const that = this;
-    this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        debug(`updateContact: contact of user updated to ${that.email}`);
-
-        callback();
-    });
-};
-
-Acme2.prototype.registerUser = function (callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var payload = {
-        termsOfServiceAgreed: true
-    };
-
-    debug('registerUser: registering user');
-
-    var that = this;
-    this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        // 200 if already exists. 201 for new accounts
-        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        debug(`registerUser: user registered keyid: ${result.headers.location}`);
-
-        that.keyId = result.headers.location;
-
-        that.updateContact(result.headers.location, callback);
-    });
-};
-
-Acme2.prototype.newOrder = function (domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var payload = {
-        identifiers: [{
-            type: 'dns',
-            value: domain
-        }]
-    };
-
-    debug('newOrder: %s', domain);
-
-    this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        debug('newOrder: created order %s %j', domain, result.body);
-
-        const order = result.body, orderUrl = result.headers.location;
-
-        if (!Array.isArray(order.authorizations)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid authorizations in order'));
-        if (typeof order.finalize !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid finalize in order'));
-        if (typeof orderUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid order location in order header'));
-
-        callback(null, order, orderUrl);
-    });
-};
-
-Acme2.prototype.waitForOrder = function (orderUrl, callback) {
-    assert.strictEqual(typeof orderUrl, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`waitForOrder: ${orderUrl}`);
-    const that = this;
-
-    async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
-        debug('waitForOrder: getting status');
-
-        that.postAsGet(orderUrl, function (error, result) {
-            if (error) {
-                debug('waitForOrder: network error getting uri %s', orderUrl);
-                return retryCallback(error);
-            }
-            if (result.statusCode !== 200) {
-                debug('waitForOrder: invalid response code getting uri %s', result.statusCode);
-                return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
-            }
-
-            debug('waitForOrder: status is "%s %j', result.body.status, result.body);
-
-            if (result.body.status === 'pending' || result.body.status === 'processing') return retryCallback(new BoxError(BoxError.TRY_AGAIN, `Request is in ${result.body.status} state`));
-            else if (result.body.status === 'valid' && result.body.certificate) return retryCallback(null, result.body.certificate);
-            else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status or invalid response: ' + result.body));
-        });
-    }, callback);
-};
-
-Acme2.prototype.getKeyAuthorization = function (token) {
-    assert(util.isBuffer(this.accountKeyPem));
-
-    let jwk = {
-        e: b64(Buffer.from([0x01, 0x00, 0x01])), // Exponent - 65537
-        kty: 'RSA',
-        n: b64(getModulus(this.accountKeyPem))
-    };
-
-    let shasum = crypto.createHash('sha256');
-    shasum.update(JSON.stringify(jwk));
-    let thumbprint = urlBase64Encode(shasum.digest('base64'));
-    return token + '.' + thumbprint;
-};
-
-Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
-    assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token }
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('notifyChallengeReady: %s was met', challenge.url);
-
-    const keyAuthorization = this.getKeyAuthorization(challenge.token);
-
-    var payload = {
-        resource: 'challenge',
-        keyAuthorization: keyAuthorization
-    };
-
-    this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        callback();
-    });
-};
-
-Acme2.prototype.waitForChallenge = function (challenge, callback) {
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('waitingForChallenge: %j', challenge);
-    const that = this;
-
-    async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
-        debug('waitingForChallenge: getting status');
-
-        that.postAsGet(challenge.url, function (error, result) {
-            if (error) {
-                debug('waitForChallenge: network error getting uri %s', challenge.url);
-                return retryCallback(error);
-            }
-            if (result.statusCode !== 200) {
-                debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
-                return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
-            }
-
-            debug('waitForChallenge: status is "%s" %j', result.body.status, result.body);
-
-            if (result.body.status === 'pending') return retryCallback(new BoxError(BoxError.TRY_AGAIN));
-            else if (result.body.status === 'valid') return retryCallback();
-            else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
-        });
-    }, function retryFinished(error) {
-        // async.retry will pass 'undefined' as second arg making it unusable with async.waterfall()
-        callback(error);
-    });
-};
-
-// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
-Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof finalizationUrl, 'string');
-    assert(util.isBuffer(csrDer));
-    assert.strictEqual(typeof callback, 'function');
-
-    const payload = {
-        csr: b64(csrDer)
-    };
-
-    debug('signCertificate: sending sign request');
-
-    this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        // 429 means we reached the cert limit for this domain
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        return callback(null);
-    });
-};
-
-Acme2.prototype.createKeyAndCsr = function (hostname, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var outdir = paths.APP_CERTS_DIR;
-    const certName = hostname.replace('*.', '_.');
-    var csrFile = path.join(outdir, `${certName}.csr`);
-    var privateKeyFile = path.join(outdir, `${certName}.key`);
-
-    if (safe.fs.existsSync(privateKeyFile)) {
-        // in some old releases, csr file was corrupt. so always regenerate it
-        debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
-    } else {
-        var key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
-        if (!key) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
-        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error));
-
-        debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
-    }
-
-    var csrDer = safe.child_process.execSync(`openssl req -new -key ${privateKeyFile} -outform DER -subj /CN=${hostname}`);
-    if (!csrDer) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
-    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new BoxError(BoxError.FS_ERROR, safe.error)); // bookkeeping
-
-    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
-
-    callback(null, csrDer);
-};
-
-Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof certUrl, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var outdir = paths.APP_CERTS_DIR;
-    const that = this;
-
-    async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-        debug('downloadCertificate: downloading certificate');
-
-        that.postAsGet(certUrl, function (error, result) {
-            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
-            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
-            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-            const fullChainPem = result.body; // buffer
-
-            const certName = hostname.replace('*.', '_.');
-            var certificateFile = path.join(outdir, `${certName}.cert`);
-            if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return retryCallback(new BoxError(BoxError.FS_ERROR, safe.error));
-
-            debug('downloadCertificate: cert file for %s saved at %s', hostname, certificateFile);
-
-            retryCallback(null);
-        });
-    }, callback);
-};
-
-Acme2.prototype.prepareHttpChallenge = function (hostname, domain, authorization, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof authorization, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('acmeFlow: challenges: %j', authorization);
-    let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
-    if (httpChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no http challenges'));
-    let challenge = httpChallenges[0];
-
-    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
-
-    let keyAuthorization = this.getKeyAuthorization(challenge.token);
-
-    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
-
-    fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), keyAuthorization, function (error) {
-        if (error) return callback(new BoxError(BoxError.FS_ERROR, error));
-
-        callback(null, challenge);
-    });
-};
-
-Acme2.prototype.cleanupHttpChallenge = function (hostname, domain, challenge, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('cleanupHttpChallenge: unlinking %s', path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
-
-    fs.unlink(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), callback);
-};
-
-function getChallengeSubdomain(hostname, domain) {
-    let challengeSubdomain;
-
-    if (hostname === domain) {
-        challengeSubdomain = '_acme-challenge';
-    } else if (hostname.includes('*')) { // wildcard
-        let subdomain = hostname.slice(0, -domain.length - 1);
-        challengeSubdomain = subdomain ? subdomain.replace('*', '_acme-challenge') : '_acme-challenge';
-    } else {
-        challengeSubdomain = '_acme-challenge.' + hostname.slice(0, -domain.length - 1);
-    }
-
-    debug(`getChallengeSubdomain: challenge subdomain for hostname ${hostname} at domain ${domain} is ${challengeSubdomain}`);
-
-    return challengeSubdomain;
-}
-
-Acme2.prototype.prepareDnsChallenge = function (hostname, domain, authorization, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof authorization, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('acmeFlow: challenges: %j', authorization);
-    let dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
-    if (dnsChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no dns challenges'));
-    let challenge = dnsChallenges[0];
-
-    const keyAuthorization = this.getKeyAuthorization(challenge.token);
-    let shasum = crypto.createHash('sha256');
-    shasum.update(keyAuthorization);
-
-    const txtValue = urlBase64Encode(shasum.digest('base64'));
-    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
-
-    debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
-
-    domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-        if (error) return callback(error);
-
-        domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 }, function (error) {
-            if (error) return callback(error);
-
-            callback(null, challenge);
-        });
-    });
-};
-
-Acme2.prototype.cleanupDnsChallenge = function (hostname, domain, challenge, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const keyAuthorization = this.getKeyAuthorization(challenge.token);
-    let shasum = crypto.createHash('sha256');
-    shasum.update(keyAuthorization);
-
-    const txtValue = urlBase64Encode(shasum.digest('base64'));
-    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
-
-    debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
-
-    domains.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-        if (error) return callback(error);
-
-        callback(null);
-    });
-};
-
-Acme2.prototype.prepareChallenge = function (hostname, domain, authorizationUrl, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof authorizationUrl, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
-
-    const that = this;
-    this.postAsGet(authorizationUrl, function (error, response) {
-        if (error) return callback(error);
-        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code getting authorization : ' + response.statusCode));
-
-        const authorization = response.body;
-
-        if (that.performHttpAuthorization) {
-            that.prepareHttpChallenge(hostname, domain, authorization, callback);
-        } else {
-            that.prepareDnsChallenge(hostname, domain, authorization, callback);
-        }
-    });
-};
-
-Acme2.prototype.cleanupChallenge = function (hostname, domain, challenge, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`cleanupChallenge: http: ${this.performHttpAuthorization}`);
-
-    if (this.performHttpAuthorization) {
-        this.cleanupHttpChallenge(hostname, domain, challenge, callback);
-    } else {
-        this.cleanupDnsChallenge(hostname, domain, challenge, callback);
-    }
-};
-
-Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
-        debug('getCertificate: generating acme account key on first run');
-        this.accountKeyPem = safe.child_process.execSync('openssl genrsa 4096');
-        if (!this.accountKeyPem) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
-
-        safe.fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, this.accountKeyPem);
-    } else {
-        debug('getCertificate: using existing acme account key');
-        this.accountKeyPem = fs.readFileSync(paths.ACME_ACCOUNT_KEY_FILE);
-    }
-
-    var that = this;
-    this.registerUser(function (error) {
-        if (error) return callback(error);
-
-        that.newOrder(hostname, function (error, order, orderUrl) {
-            if (error) return callback(error);
-
-            async.eachSeries(order.authorizations, function (authorizationUrl, iteratorCallback) {
-                debug(`acmeFlow: authorizing ${authorizationUrl}`);
-
-                that.prepareChallenge(hostname, domain, authorizationUrl, function (error, challenge) {
-                    if (error) return iteratorCallback(error);
-
-                    async.waterfall([
-                        that.notifyChallengeReady.bind(that, challenge),
-                        that.waitForChallenge.bind(that, challenge),
-                        that.createKeyAndCsr.bind(that, hostname),
-                        that.signCertificate.bind(that, hostname, order.finalize),
-                        that.waitForOrder.bind(that, orderUrl),
-                        that.downloadCertificate.bind(that, hostname)
-                    ], function (error) {
-                        that.cleanupChallenge(hostname, domain, challenge, function (cleanupError) {
-                            if (cleanupError) debug('acmeFlow: ignoring error when cleaning up challenge:', cleanupError);
-
-                            iteratorCallback(error);
-                        });
-                    });
-                });
-            }, callback);
-        });
-    });
-};
-
-Acme2.prototype.getDirectory = function (callback) {
-    const that = this;
-
-    request.get(this.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
-        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
-
-        if (typeof response.body.newNonce !== 'string' ||
-            typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));
-
-        that.directory = response.body;
-
-        callback(null);
-    });
-};
-
-Acme2.prototype.getCertificate = function (vhost, domain, callback) {
-    assert.strictEqual(typeof vhost, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
-
-    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
-        vhost = domains.makeWildcard(vhost);
-        debug(`getCertificate: will get wildcard cert for ${vhost}`);
-    }
-
-    const that = this;
-    this.getDirectory(function (error) {
-        if (error) return callback(error);
-
-        that.acmeFlow(vhost, domain, function (error) {
-            if (error) return callback(error);
-
-            var outdir = paths.APP_CERTS_DIR;
-            const certName = vhost.replace('*.', '_.');
-            callback(null, path.join(outdir, `${certName}.cert`), path.join(outdir, `${certName}.key`));
-        });
-    });
-};
-
-function getCertificate(vhost, domain, options, callback) {
-    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    let attempt = 1;
-    async.retry({ times: 3, interval: 0 }, function (retryCallback) {
-        debug(`getCertificate: attempt ${attempt++}`);
-
-        let acme = new Acme2(options || { });
-        acme.getCertificate(vhost, domain, retryCallback);
-    }, callback);
-}

src/changelog.js

@@ -5,7 +5,7 @@ let assert = require('assert'),
     path = require('path');
 
 exports = module.exports = {
-    getChanges: getChanges
+    getChanges
 };
 
 function getChanges(version) {

src/cloudron.js

@@ -52,16 +52,14 @@ const apps = require('./apps.js'),
     tasks = require('./tasks.js'),
     users = require('./users.js');
 
-var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');
+const REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');
 
 const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function initialize() {
     runStartupTasks();
 
-    notifyUpdate(callback);
+    await notifyUpdate();
 }
 
 function uninitialize(callback) {
@@ -77,39 +75,33 @@ function onActivated(options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    debug('onActivated: running post activation tasks');
+
     // Starting the platform after a user is available means:
     // 1. mail bounces can now be sent to the cloudron owner
     // 2. the restore code path can run without sudo (since mail/ is non-root)
     async.series([
         platform.start.bind(null, options),
         cron.startJobs,
-        function checkBackupConfiguration(done) {
-            backups.checkConfiguration(function (error, message) {
-                if (error) return done(error);
-                notifications.alert(notifications.ALERT_BACKUP_CONFIG, 'Backup configuration is unsafe', message, done);
-            });
-        },
         // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
         // the UI some time to query the dashboard domain in the restore code path
         (done) => setTimeout(() => reverseProxy.writeDefaultConfig({ activated :true }, done), 30000)
     ], callback);
 }
 
-function notifyUpdate(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function notifyUpdate() {
     const version = safe.fs.readFileSync(paths.VERSION_FILE, 'utf8');
-    if (version === constants.VERSION) return callback();
+    if (version === constants.VERSION) return;
 
-    eventlog.add(eventlog.ACTION_UPDATE_FINISH, auditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION }, function (error) {
-        if (error) return callback(error);
+    await eventlog.add(eventlog.ACTION_UPDATE_FINISH, auditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
 
+    return new Promise((resolve, reject) => {
         tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }, function (error) {
-            if (error && error.reason !== BoxError.NOT_FOUND) return callback(error); // when hotfixing, task may not exist
+            if (error && error.reason !== BoxError.NOT_FOUND) return reject(error); // when hotfixing, task may not exist
 
             safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
 
-            callback();
+            resolve();
         });
     });
 }
@@ -131,9 +123,9 @@ function runStartupTasks() {
 
         // always generate webadmin config since we have no versioning mechanism for the ejs
         function (callback) {
-            if (!settings.adminDomain()) return callback();
+            if (!settings.dashboardDomain()) return callback();
 
-            reverseProxy.writeDashboardConfig(settings.adminDomain(), callback);
+            reverseProxy.writeDashboardConfig(settings.dashboardDomain(), callback);
         },
 
         // check activation state and start the platform
@@ -165,6 +157,10 @@ function runStartupTasks() {
 function getConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    const release = safe.fs.readFileSync('/etc/lsb-release', 'utf-8');
+    if (release === null) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
+    const ubuntuVersion = release.match(/DISTRIB_DESCRIPTION="(.*)"/)[1];
+
     settings.getAll(function (error, allSettings) {
         if (error) return callback(error);
 
@@ -172,10 +168,11 @@ function getConfig(callback) {
         callback(null, {
             apiServerOrigin: settings.apiServerOrigin(),
             webServerOrigin: settings.webServerOrigin(),
-            adminDomain: settings.adminDomain(),
-            adminFqdn: settings.adminFqdn(),
+            adminDomain: settings.dashboardDomain(),
+            adminFqdn: settings.dashboardFqdn(),
             mailFqdn: settings.mailFqdn(),
             version: constants.VERSION,
+            ubuntuVersion,
             isDemo: settings.isDemo(),
             cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
             footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
@@ -205,17 +202,18 @@ function isRebootRequired(callback) {
 function runSystemChecks(callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    debug('runSystemChecks: checking status');
+
     async.parallel([
         checkMailStatus,
-        checkRebootRequired
+        checkRebootRequired,
+        checkUbuntuVersion
     ], callback);
 }
 
 function checkMailStatus(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    debug('checking mail status');
-
     mail.checkConfiguration(function (error, message) {
         if (error) return callback(error);
 
@@ -226,8 +224,6 @@ function checkMailStatus(callback) {
 function checkRebootRequired(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    debug('checking if reboot required');
-
     isRebootRequired(function (error, rebootRequired) {
         if (error) return callback(error);
 
@@ -235,6 +231,15 @@ function checkRebootRequired(callback) {
     });
 }
 
+function checkUbuntuVersion(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    const isXenial = fs.readFileSync('/etc/lsb-release', 'utf-8').includes('16.04');
+    if (!isXenial) return callback();
+
+    notifications.alert(notifications.ALERT_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 16.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-18/ to upgrade to Ubuntu 18 at the earliest.', callback);
+}
+
 function getLogs(unit, options, callback) {
     assert.strictEqual(typeof unit, 'string');
     assert(options && typeof options === 'object');
@@ -293,7 +298,7 @@ function prepareDashboardDomain(domain, auditSource, callback) {
     domains.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
-        const fqdn = domains.fqdn(constants.ADMIN_LOCATION, domainObject);
+        const fqdn = domains.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
         apps.getAll(function (error, result) {
             if (error) return callback(error);
@@ -301,7 +306,7 @@ function prepareDashboardDomain(domain, auditSource, callback) {
             const conflict = result.filter(app => app.fqdn === fqdn);
             if (conflict.length) return callback(new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app'));
 
-            tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.ADMIN_LOCATION, domain, auditSource ], function (error, taskId) {
+            tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_LOCATION, domain, auditSource ], function (error, taskId) {
                 if (error) return callback(error);
 
                 tasks.startTask(taskId, {}, NOOP_CALLBACK);
@@ -326,12 +331,14 @@ function setDashboardDomain(domain, auditSource, callback) {
         reverseProxy.writeDashboardConfig(domain, function (error) {
             if (error) return callback(error);
 
-            const fqdn = domains.fqdn(constants.ADMIN_LOCATION, domainObject);
+            const fqdn = domains.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
-            settings.setAdminLocation(domain, fqdn, function (error) {
+            settings.setDashboardLocation(domain, fqdn, function (error) {
                 if (error) return callback(error);
 
-                eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain: domain, fqdn: fqdn });
+                appstore.updateCloudron({ domain }, NOOP_CALLBACK);
+
+                eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain, fqdn });
 
                 callback(null);
             });
@@ -363,7 +370,7 @@ function renewCerts(options, auditSource, callback) {
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    tasks.add(tasks.TASK_RENEW_CERTS, [ options, auditSource ], function (error, taskId) {
+    tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ], function (error, taskId) {
         if (error) return callback(error);
 
         tasks.startTask(taskId, {}, NOOP_CALLBACK);
@@ -382,17 +389,17 @@ function setupDnsAndCert(subdomain, domain, auditSource, progressCallback, callb
     domains.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
-        const adminFqdn = domains.fqdn(subdomain, domainObject);
+        const dashboardFqdn = domains.fqdn(subdomain, domainObject);
 
         sysinfo.getServerIp(function (error, ip) {
             if (error) return callback(error);
 
             async.series([
-                (done) => { progressCallback({ message: `Updating DNS of ${adminFqdn}` }); done(); },
+                (done) => { progressCallback({ message: `Updating DNS of ${dashboardFqdn}` }); done(); },
                 domains.upsertDnsRecords.bind(null, subdomain, domain, 'A', [ ip ]),
-                (done) => { progressCallback({ message: `Waiting for DNS of ${adminFqdn}` }); done(); },
+                (done) => { progressCallback({ message: `Waiting for DNS of ${dashboardFqdn}` }); done(); },
                 domains.waitForDnsRecord.bind(null, subdomain, domain, 'A', ip, { interval: 30000, times: 50000 }),
-                (done) => { progressCallback({ message: `Getting certificate of ${adminFqdn}` }); done(); },
+                (done) => { progressCallback({ message: `Getting certificate of ${dashboardFqdn}` }); done(); },
                 reverseProxy.ensureCertificate.bind(null, domains.fqdn(subdomain, domainObject), domain, auditSource)
             ], function (error) {
                 if (error) return callback(error);

src/constants.js

@@ -22,7 +22,7 @@ exports = module.exports = {
         'admins', 'users'         // ldap code uses 'users' pseudo group
     ],
 
-    ADMIN_LOCATION: 'my',
+    DASHBOARD_LOCATION: 'my',
 
     PORT: CLOUDRON ? 3000 : 5454,
     INTERNAL_SMTP_PORT: 2525, // this value comes from the mail container
@@ -32,7 +32,8 @@ exports = module.exports = {
 
     NGINX_DEFAULT_CONFIG_FILE_NAME: 'default.conf',
 
-    DEFAULT_TOKEN_EXPIRATION: 365 * 24 * 60 * 60 * 1000, // 1 year
+    DEFAULT_TOKEN_EXPIRATION_MSECS: 365 * 24 * 60 * 60 * 1000, // 1 year
+    DEFAULT_TOKEN_EXPIRATION_DAYS: 365,
 
     DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024), // see also client.js
 

src/crashnotifier.js

@@ -1,10 +1,10 @@
 'use strict';
 
 exports = module.exports = {
-    sendFailureLogs: sendFailureLogs
+    sendFailureLogs
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     auditSource = require('./auditsource.js'),
     eventlog = require('./eventlog.js'),
     safe = require('safetydance'),
@@ -38,7 +38,7 @@ function sendFailureLogs(unitName, callback) {
         return callback();
     }
 
-    collectLogs(unitName, function (error, logs) {
+    collectLogs(unitName, async function (error, logs) {
         if (error) {
             console.error('Failed to collect logs.', error);
             logs = util.format('Failed to collect logs.', error);
@@ -49,12 +49,11 @@ function sendFailureLogs(unitName, callback) {
 
         if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, `${crashId}.log`), logs)) console.log(`Failed to stash logs to ${crashId}.log:`, safe.error);
 
-        eventlog.add(eventlog.ACTION_PROCESS_CRASH, auditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }, function (error) {
-            if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);
+        [error] = await safe(eventlog.add(eventlog.ACTION_PROCESS_CRASH, auditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }));
+        if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);
 
-            safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
+        safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
 
-            callback();
-        });
+        callback();
     });
 }

src/cron.js

@@ -16,7 +16,7 @@ exports = module.exports = {
     DEFAULT_AUTOUPDATE_PATTERN,
 };
 
-var appHealthMonitor = require('./apphealthmonitor.js'),
+const appHealthMonitor = require('./apphealthmonitor.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
@@ -101,7 +101,7 @@ function startJobs(callback) {
 
     gJobs.cleanupEventlog = new CronJob({
         cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: eventlog.cleanup,
+        onTick: eventlog.cleanup.bind(null, new Date(Date.now() - 60 * 60 * 24 * 10 * 1000)), // 10 days ago
         start: true
     });
 

src/database.js

@@ -1,18 +1,18 @@
 'use strict';
 
 exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize,
-    query: query,
-    transaction: transaction,
+    initialize,
+    uninitialize,
+    query,
+    transaction,
 
-    importFromFile: importFromFile,
-    exportToFile: exportToFile,
+    importFromFile,
+    exportToFile,
 
     _clear: clear
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
@@ -45,6 +45,8 @@ function initialize(callback) {
     // https://github.com/mysqljs/mysql#pool-options
     gConnectionPool  = mysql.createPool({
         connectionLimit: 5,
+        acquireTimeout: 60000,
+        connectTimeout: 60000,
         host: gDatabase.hostname,
         user: gDatabase.username,
         password: gDatabase.password,
@@ -86,40 +88,52 @@ function clear(callback) {
 }
 
 function query() {
-    const args = Array.prototype.slice.call(arguments);
-    const callback = args[args.length - 1];
-    assert.strictEqual(typeof callback, 'function');
+    assert.notStrictEqual(gConnectionPool, null);
 
-    if (constants.TEST && !gConnectionPool) return callback(new BoxError(BoxError.DATABASE_ERROR, 'database.js not initialized'));
+    return new Promise((resolve, reject) => {
+        let args = Array.prototype.slice.call(arguments);
+        const callback = typeof args[args.length - 1] === 'function' ? args.pop() : null;
 
-    gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
+        args.push(function queryCallback(error, result) {
+            if (error) return callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+
+            callback ? callback(null, result) : resolve(result);
+        });
+
+        gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
+    });
 }
 
-function transaction(queries, callback) {
-    assert(util.isArray(queries));
-    assert.strictEqual(typeof callback, 'function');
+function transaction(queries) {
+    assert(Array.isArray(queries));
 
-    callback = once(callback);
+    const args = Array.prototype.slice.call(arguments);
+    const callback = typeof args[args.length - 1] === 'function' ? once(args.pop()) : null;
 
-    gConnectionPool.getConnection(function (error, connection) {
-        if (error) return callback(error);
+    return new Promise((resolve, reject) => {
+        gConnectionPool.getConnection(function (error, connection) {
+            if (error) return callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
 
-        const releaseConnection = (error) => { connection.release(); callback(error); };
+            const releaseConnection = (error) => {
+                connection.release();
+                callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+            };
 
-        connection.beginTransaction(function (error) {
-            if (error) return releaseConnection(error);
+            connection.beginTransaction(function (error) {
+                if (error) return releaseConnection(error);
 
-            async.mapSeries(queries, function iterator(query, done) {
-                connection.query(query.query, query.args, done);
-            }, function seriesDone(error, results) {
-                if (error) return connection.rollback(() => releaseConnection(error));
-
-                connection.commit(function (error) {
+                async.mapSeries(queries, function iterator(query, done) {
+                    connection.query(query.query, query.args, done);
+                }, function seriesDone(error, results) {
                     if (error) return connection.rollback(() => releaseConnection(error));
 
-                    connection.release();
+                    connection.commit(function (error) {
+                        if (error) return connection.rollback(() => releaseConnection(error));
 
-                    callback(null, results);
+                        connection.release();
+
+                        callback ? callback(null, results) : resolve(results);
+                    });
                 });
             });
         });

src/dns/cloudflare.js

@@ -110,7 +110,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
@@ -206,7 +206,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,

src/dns/digitalocean.js

@@ -10,7 +10,7 @@ exports = module.exports = {
     verifyDnsConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     async = require('async'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
@@ -22,7 +22,7 @@ var assert = require('assert'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
 
-var DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com';
+const DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com';
 
 function formatError(response) {
     return util.format('DigitalOcean DNS error [%s] %j', response.statusCode, response.body);
@@ -82,7 +82,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
@@ -185,7 +185,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,

src/dns/gandi.js

@@ -39,7 +39,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
@@ -98,7 +98,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,

src/dns/gcdns.js

@@ -73,7 +73,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
@@ -144,7 +144,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,

src/dns/godaddy.js

@@ -45,7 +45,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
@@ -118,7 +118,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,

src/dns/interface.js

@@ -34,7 +34,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     // Result: none
@@ -57,7 +57,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     // Result: none

src/dns/linode.js

@@ -135,7 +135,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
@@ -217,7 +217,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,

src/dns/manual.js

@@ -31,7 +31,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
@@ -52,7 +52,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     return callback();

src/dns/namecheap.js

@@ -16,10 +16,10 @@ var assert = require('assert'),
     debug = require('debug')('box:dns/namecheap'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
+    querystring = require('querystring'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     sysinfo = require('../sysinfo.js'),
-    util = require('util'),
     waitForDns = require('./waitfordns.js'),
     xml2js = require('xml2js');
 
@@ -50,11 +50,9 @@ function getQuery(dnsConfig, callback) {
     });
 }
 
-function getInternal(dnsConfig, zoneName, subdomain, type, callback) {
+function getZone(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     getQuery(dnsConfig, function (error, query) {
@@ -89,7 +87,7 @@ function getInternal(dnsConfig, zoneName, subdomain, type, callback) {
     });
 }
 
-function setInternal(dnsConfig, zoneName, hosts, callback) {
+function setZone(dnsConfig, zoneName, hosts, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert(Array.isArray(hosts));
@@ -116,7 +114,10 @@ function setInternal(dnsConfig, zoneName, hosts, callback) {
             }
         });
 
-        superagent.post(ENDPOINT).query(query).end(function (error, result) {
+        // namecheap recommends sending as POSTDATA with > 10 records
+        const qs = querystring.stringify(query);
+
+        superagent.post(ENDPOINT).set('Content-Type', 'application/x-www-form-urlencoded').send(qs).end(function (error, result) {
             if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
             var parser = new xml2js.Parser();
@@ -144,7 +145,7 @@ function upsert(domainObject, subdomain, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config;
@@ -154,17 +155,17 @@ function upsert(domainObject, subdomain, type, values, callback) {
 
     debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+    getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
 
         // Array to keep track of records that need to be inserted
         let toInsert = [];
 
-        for (var i = 0; i < values.length; i++) {
+        for (let i = 0; i < values.length; i++) {
             let curValue = values[i];
             let wasUpdate = false;
 
-            for (var j = 0; j < result.length; j++) {
+            for (let j = 0; j < result.length; j++) {
                 let curHost = result[j];
 
                 if (curHost.Type === type && curHost.Name === subdomain) {
@@ -198,9 +199,9 @@ function upsert(domainObject, subdomain, type, values, callback) {
             }
         }
 
-        let toUpsert = result.concat(toInsert);
+        const hosts = result.concat(toInsert);
 
-        setInternal(dnsConfig, zoneName, toUpsert, callback);
+        setZone(dnsConfig, zoneName, hosts, callback);
     });
 }
 
@@ -215,16 +216,16 @@ function get(domainObject, subdomain, type, callback) {
 
     subdomain = domains.getName(domainObject, subdomain, type) || '@';
 
-    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+    getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
 
         // We need to filter hosts to ones with this subdomain and type
-        let actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
+        const actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
 
         // We only return the value string
-        var tmp = actualHosts.map(function (record) { return record.Address; });
+        const tmp = actualHosts.map(function (record) { return record.Address; });
 
-        debug('get: %j', tmp);
+        debug(`get: subdomain: ${subdomain} type:${type} value:${JSON.stringify(tmp)}`);
 
         return callback(null, tmp);
     });
@@ -234,7 +235,7 @@ function del(domainObject, subdomain, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config;
@@ -244,29 +245,19 @@ function del(domainObject, subdomain, type, values, callback) {
 
     debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+    getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
 
         if (result.length === 0) return callback();
+        const originalLength = result.length;
 
-        let removed = false;
-
-        for (var i = 0; i < values.length; i++) {
+        for (let i = 0; i < values.length; i++) {
             let curValue = values[i];
 
-            for (var j = 0; j < result.length; j++) {
-                let curHost = result[i];
-
-                if (curHost.Type === type && curHost.Name === subdomain && curHost.Address === curValue) {
-                    removed = true;
-
-                    result.splice(i, 1); // Remove element from result array
-                }
-            }
+            result = result.filter(curHost => curHost.Type !== type || curHost.Name !== subdomain || curHost.Address !== curValue);
         }
 
-        // Only set hosts if we actually removed a host
-        if (removed) return setInternal(dnsConfig, zoneName, result, callback);
+        if (result.length !== originalLength) return setZone(dnsConfig, zoneName, result, callback);
 
         callback();
     });

src/dns/namecom.js

@@ -157,7 +157,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
@@ -200,7 +200,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,

src/dns/netcup.js

@@ -90,7 +90,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
@@ -183,7 +183,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,

src/dns/noop.js

@@ -26,7 +26,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
@@ -47,7 +47,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     return callback();

src/dns/route53.js

@@ -97,7 +97,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
@@ -178,7 +178,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,

src/dns/vultr.js

@@ -0,0 +1,280 @@
+'use strict';
+
+exports = module.exports = {
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
+};
+
+const async = require('async'),
+    assert = require('assert'),
+    constants = require('../constants.js'),
+    BoxError = require('../boxerror.js'),
+    debug = require('debug')('box:dns/vultr'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
+    safe = require('safetydance'),
+    superagent = require('superagent'),
+    util = require('util'),
+    waitForDns = require('./waitfordns.js');
+
+const VULTR_ENDPOINT = 'https://api.vultr.com/v2';
+
+function formatError(response) {
+    return util.format('Vultr DNS error [%s] %j', response.statusCode, response.body);
+}
+
+function removePrivateFields(domainObject) {
+    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    return domainObject;
+}
+
+function injectPrivateFields(newConfig, currentConfig) {
+    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+}
+
+function getZoneRecords(dnsConfig, zoneName, name, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
+
+    let per_page = 100, cursor= null;
+    let records = [];
+
+    async.doWhilst(function (iteratorDone) {
+        const url = `${VULTR_ENDPOINT}/domains/${zoneName}/records?per_page=${per_page}` + (cursor ? `&cursor=${cursor}` : '');
+
+        superagent.get(url)
+            .set('Authorization', 'Bearer ' + dnsConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .end(function (error, result) {
+                if (error && !error.response) return iteratorDone(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 404) return iteratorDone(new BoxError(BoxError.NOT_FOUND, formatError(result)));
+                if (result.statusCode === 403 || result.statusCode === 401) return iteratorDone(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
+                if (result.statusCode !== 200) return iteratorDone(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                records = records.concat(result.body.records.filter(function (record) {
+                    return (record.type === type && record.name === name);
+                }));
+
+                cursor = safe.query(result.body, 'meta.links.next');
+
+                iteratorDone();
+            });
+    }, function (testDone) { return testDone(null, !!cursor); }, function (error) {
+        debug('getZoneRecords:', error, JSON.stringify(records));
+
+        if (error) return callback(error);
+
+        callback(null, records);
+    });
+}
+
+function get(domainObject, location, type, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '';
+
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
+        if (error) return callback(error);
+
+        var tmp = records.map(function (record) { return record.data; });
+
+        debug('get: %j', tmp);
+
+        return callback(null, tmp);
+    });
+}
+
+function upsert(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '';
+
+    debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
+
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
+        if (error) return callback(error);
+
+        let i = 0, recordIds = []; // used to track available records to update instead of create
+
+        async.eachSeries(values, function (value, iteratorCallback) {
+            let data = {
+                type,
+                ttl: 300 // lowest
+            };
+
+            if (type === 'MX') {
+                data.priority = parseInt(value.split(' ')[0], 10);
+                data.data = value.split(' ')[1];
+            } else if (type === 'TXT') {
+                data.data = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
+            } else {
+                data.data = value;
+            }
+
+            if (i >= records.length) {
+                data.name = name; // only set for new records
+
+                superagent.post(`${VULTR_ENDPOINT}/domains/${zoneName}/records`)
+                    .set('Authorization', 'Bearer ' + dnsConfig.token)
+                    .send(data)
+                    .timeout(30 * 1000)
+                    .retry(5)
+                    .end(function (error, result) {
+                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
+                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
+                        if (result.statusCode !== 201) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                        recordIds.push(result.body.record.id);
+
+                        return iteratorCallback(null);
+                    });
+            } else {
+                superagent.patch(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${records[i].id}`)
+                    .set('Authorization', 'Bearer ' + dnsConfig.token)
+                    .send(data)
+                    .timeout(30 * 1000)
+                    .retry(5)
+                    .end(function (error, result) {
+                        // increment, as we have consumed the record
+                        ++i;
+
+                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
+                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
+                        if (result.statusCode !== 204) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                        recordIds.push(records[i-1].id);
+
+                        return iteratorCallback(null);
+                    });
+            }
+        }, function (error) {
+            if (error) return callback(error);
+
+            debug('upsert: completed with recordIds:%j', recordIds);
+
+            callback();
+        });
+    });
+}
+
+function del(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '';
+
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, records) {
+        if (error) return callback(error);
+
+        if (records.length === 0) return callback(null);
+
+        const tmp = records.filter(function (record) { return values.some(function (value) { return value === record.data; }); });
+
+        debug('del: %j', tmp);
+
+        if (tmp.length === 0) return callback(null);
+
+        // FIXME we only handle the first one currently
+
+        superagent.del(`${VULTR_ENDPOINT}/domains/${zoneName}/records/${tmp[0].id}`)
+            .set('Authorization', 'Bearer ' + dnsConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 404) return callback(null);
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
+                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                debug('del: done');
+
+                return callback(null);
+            });
+    });
+}
+
+function wait(domainObject, location, type, value, options, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+    assert.strictEqual(typeof callback, 'function');
+
+    const fqdn = domains.fqdn(location, domainObject);
+
+    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+}
+
+function verifyDnsConfig(domainObject, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+
+    const ip = '127.0.0.1';
+
+    var credentials = {
+        token: dnsConfig.token
+    };
+
+    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+
+    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
+        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
+        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+
+        if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.vultr.com') === -1) {
+            debug('verifyDnsConfig: %j does not contains vultr NS', nameservers);
+            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Vultr', { field: 'nameservers' }));
+        }
+
+        const location = 'cloudrontestdns';
+
+        upsert(domainObject, location, 'A', [ ip ], function (error) {
+            if (error) return callback(error);
+
+            debug('verifyDnsConfig: Test A record added');
+
+            del(domainObject, location, 'A', [ ip ], function (error) {
+                if (error) return callback(error);
+
+                debug('verifyDnsConfig: Test A record removed again');
+
+                callback(null, credentials);
+            });
+        });
+    });
+}

src/dns/wildcard.js

@@ -31,7 +31,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
@@ -52,7 +52,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     return callback();

src/docker.js

@@ -49,7 +49,6 @@ const apps = require('./apps.js'),
     shell = require('./shell.js'),
     safe = require('safetydance'),
     system = require('./system.js'),
-    util = require('util'),
     volumes = require('./volumes.js'),
     _ = require('underscore');
 
@@ -194,33 +193,30 @@ function downloadImage(manifest, callback) {
     });
 }
 
-function getVolumeMounts(app, callback) {
+async function getVolumeMounts(app) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     let mounts = [];
 
-    if (app.mounts.length === 0) return callback(null, []);
+    if (app.mounts.length === 0) return [];
 
-    volumes.list(function (error, result) {
-        if (error) return callback(error);
+    const result = await volumes.list();
 
-        let volumesById = {};
-        result.forEach(r => volumesById[r.id] = r);
+    let volumesById = {};
+    result.forEach(r => volumesById[r.id] = r);
 
-        for (const mount of app.mounts) {
-            const volume = volumesById[mount.volumeId];
+    for (const mount of app.mounts) {
+        const volume = volumesById[mount.volumeId];
 
-            mounts.push({
-                Source: volume.hostPath,
-                Target: `/media/${volume.name}`,
-                Type: 'bind',
-                ReadOnly: mount.readOnly
-            });
-        }
+        mounts.push({
+            Source: volume.hostPath,
+            Target: `/media/${volume.name}`,
+            Type: 'bind',
+            ReadOnly: mount.readOnly
+        });
+    }
 
-        callback(null, mounts);
-    });
+    return mounts;
 }
 
 function getAddonMounts(app, callback) {
@@ -244,7 +240,7 @@ function getAddonMounts(app, callback) {
 
             return iteratorDone();
         case 'tls':
-            reverseProxy.getCertificate(app.fqdn, app.domain, function (error, bundle) {
+            reverseProxy.getCertificatePath(app.fqdn, app.domain, function (error, bundle) {
                 if (error) return iteratorDone(error);
 
                 mounts.push({
@@ -273,18 +269,17 @@ function getAddonMounts(app, callback) {
     });
 }
 
-function getMounts(app, callback) {
+async function getMounts(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    getVolumeMounts(app, function (error, volumeMounts) {
+    const [error, volumeMounts] = await safe(getVolumeMounts(app));
+    if (error) return callback(error);
+
+    getAddonMounts(app, function (error, addonMounts) {
         if (error) return callback(error);
 
-        getAddonMounts(app, function (error, addonMounts) {
-            if (error) return callback(error);
-
-            callback(null, volumeMounts.concat(addonMounts));
-        });
+        callback(null, volumeMounts.concat(addonMounts));
     });
 }
 
@@ -303,7 +298,7 @@ function getLowerUpIp() { // see getifaddrs and IFF_LOWER_UP and netdevice
 function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof name, 'string');
-    assert(!cmd || util.isArray(cmd));
+    assert(!cmd || Array.isArray(cmd));
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -319,8 +314,8 @@ function createSubcontainer(app, name, cmd, options, callback) {
         'CLOUDRON=1',
         'CLOUDRON_PROXY_IP=172.18.0.1',
         `CLOUDRON_APP_HOSTNAME=${app.id}`,
-        `${envPrefix}WEBADMIN_ORIGIN=${settings.adminOrigin()}`,
-        `${envPrefix}API_ORIGIN=${settings.adminOrigin()}`,
+        `${envPrefix}WEBADMIN_ORIGIN=${settings.dashboardOrigin()}`,
+        `${envPrefix}API_ORIGIN=${settings.dashboardOrigin()}`,
         `${envPrefix}APP_ORIGIN=https://${domain}`,
         `${envPrefix}APP_DOMAIN=${domain}`
     ];
@@ -570,10 +565,11 @@ function deleteImage(manifest, callback) {
     assert(!manifest || typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var dockerImage = manifest ? manifest.dockerImage : null;
+    const dockerImage = manifest ? manifest.dockerImage : null;
     if (!dockerImage) return callback(null);
+    if (dockerImage.includes('//')) return callback(null); // a common mistake is to paste a https:// as docker image. this results in a crash at runtime in dockerode module
 
-    var removeOptions = {
+    const removeOptions = {
         force: false, // might be shared with another instance of this app
         noprune: false // delete untagged parents
     };

src/dockerproxy.js

@@ -1,8 +1,8 @@
 'use strict';
 
 exports = module.exports = {
-    start: start,
-    stop: stop
+    start,
+    stop
 };
 
 var apps = require('./apps.js'),

src/domaindb.js

@@ -3,20 +3,20 @@
 'use strict';
 
 exports = module.exports = {
-    add: add,
-    get: get,
-    getAll: getAll,
-    update: update,
-    del: del,
-    clear: clear
+    add,
+    get,
+    getAll,
+    update,
+    del,
+    clear
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     database = require('./database.js'),
     safe = require('safetydance');
 
-var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson' ].join(',');
+const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(',');
 
 function postProcess(data) {
     data.config = safe.JSON.parse(data.configJson);
@@ -28,6 +28,9 @@ function postProcess(data) {
     data.wellKnown = safe.JSON.parse(data.wellKnownJson);
     delete data.wellKnownJson;
 
+    data.fallbackCertificate = safe.JSON.parse(data.fallbackCertificateJson);
+    delete data.fallbackCertificateJson;
+
     return data;
 }
 
@@ -62,10 +65,12 @@ function add(name, data, callback) {
     assert.strictEqual(typeof data.provider, 'string');
     assert.strictEqual(typeof data.config, 'object');
     assert.strictEqual(typeof data.tlsConfig, 'object');
+    assert.strictEqual(typeof data.fallbackCertificate, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     let queries = [
-        { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson) VALUES (?, ?, ?, ?, ?)', args: [ name, data.zoneName, data.provider, JSON.stringify(data.config), JSON.stringify(data.tlsConfig) ] },
+        { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson, fallbackCertificateJson) VALUES (?, ?, ?, ?, ?, ?)',
+            args: [ name, data.zoneName, data.provider, JSON.stringify(data.config), JSON.stringify(data.tlsConfig), JSON.stringify(data.fallbackCertificate) ] },
         { query: 'INSERT INTO mail (domain, dkimSelector) VALUES (?, ?)', args: [ name, data.dkimSelector || 'cloudron' ] },
     ];
 
@@ -84,14 +89,8 @@ function update(name, domain, callback) {
 
     var args = [ ], fields = [ ];
     for (var k in domain) {
-        if (k === 'config') {
-            fields.push('configJson = ?');
-            args.push(JSON.stringify(domain[k]));
-        } else if (k === 'tlsConfig') {
-            fields.push('tlsConfigJson = ?');
-            args.push(JSON.stringify(domain[k]));
-        } else if (k === 'wellKnown') {
-            fields.push('wellKnownJson = ?');
+        if (k === 'config' || k === 'tlsConfig' || k === 'wellKnown' || k === 'fallbackCertificate') { // json fields
+            fields.push(`${k}Json = ?`);
             args.push(JSON.stringify(domain[k]));
         } else {
             fields.push(k + ' = ?');

src/domains.js

@@ -44,11 +44,9 @@ const apps = require('./apps.js'),
     eventlog = require('./eventlog.js'),
     mail = require('./mail.js'),
     reverseProxy = require('./reverseproxy.js'),
-    safe = require('safetydance'),
     settings = require('./settings.js'),
     sysinfo = require('./sysinfo.js'),
     tld = require('tldjs'),
-    util = require('util'),
     _ = require('underscore');
 
 const NOOP_CALLBACK = function (error) { if (error) debug(error); };
@@ -65,6 +63,7 @@ function api(provider) {
     case 'gandi': return require('./dns/gandi.js');
     case 'godaddy': return require('./dns/godaddy.js');
     case 'linode': return require('./dns/linode.js');
+    case 'vultr': return require('./dns/vultr.js');
     case 'namecom': return require('./dns/namecom.js');
     case 'namecheap': return require('./dns/namecheap.js');
     case 'netcup': return require('./dns/netcup.js');
@@ -121,7 +120,7 @@ function validateHostname(location, domainObject) {
     ];
     if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
 
-    if (hostname === settings.adminFqdn()) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
+    if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
 
     // workaround https://github.com/oncletom/tld.js/issues/73
     var tmp = hostname.replace('_', '-');
@@ -191,7 +190,7 @@ function add(domain, data, auditSource, callback) {
         let error = reverseProxy.validateCertificate('test', { domain, config }, fallbackCertificate);
         if (error) return callback(error);
     } else {
-        fallbackCertificate = reverseProxy.generateFallbackCertificateSync({ domain, config });
+        fallbackCertificate = reverseProxy.generateFallbackCertificateSync(domain);
         if (fallbackCertificate.error) return callback(fallbackCertificate.error);
     }
 
@@ -200,14 +199,14 @@ function add(domain, data, auditSource, callback) {
 
     if (!dkimSelector) {
         // create a unique suffix. this lets one add this domain can be added in another cloudron instance and not have their dkim selector conflict
-        const suffix = crypto.createHash('sha256').update(settings.adminDomain()).digest('hex').substr(0, 6);
+        const suffix = crypto.createHash('sha256').update(settings.dashboardDomain()).digest('hex').substr(0, 6);
         dkimSelector = `cloudron-${suffix}`;
     }
 
     verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
         if (error) return callback(error);
 
-        domaindb.add(domain, { zoneName, provider, config: sanitizedConfig, tlsConfig, dkimSelector }, function (error) {
+        domaindb.add(domain, { zoneName, provider, config: sanitizedConfig, tlsConfig, dkimSelector, fallbackCertificate }, function (error) {
             if (error) return callback(error);
 
             reverseProxy.setFallbackCertificate(domain, fallbackCertificate, function (error) {
@@ -230,17 +229,7 @@ function get(domain, callback) {
     domaindb.get(domain, function (error, result) {
         if (error) return callback(error);
 
-        reverseProxy.getFallbackCertificate(domain, function (_, bundle) { // never returns an error
-            var cert = safe.fs.readFileSync(bundle.certFilePath, 'utf-8');
-            var key = safe.fs.readFileSync(bundle.keyFilePath, 'utf-8');
-
-            // do not error here. otherwise, there is no way to fix things up from the UI
-            if (!cert || !key) debug(`Unable to read fallback certificates of ${domain} from disk`);
-
-            result.fallbackCertificate = { cert: cert, key: key };
-
-            return callback(null, result);
-        });
+        return callback(null, result);
     });
 }
 
@@ -266,7 +255,7 @@ function update(domain, data, auditSource, callback) {
 
     let { zoneName, provider, config, fallbackCertificate, tlsConfig, wellKnown } = data;
 
-    if (settings.isDemo() && (domain === settings.adminDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+    if (settings.isDemo() && (domain === settings.dashboardDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
 
     domaindb.get(domain, function (error, domainObject) {
         if (error) return callback(error);
@@ -298,9 +287,11 @@ function update(domain, data, auditSource, callback) {
                 zoneName,
                 provider,
                 tlsConfig,
-                wellKnown
+                wellKnown,
             };
 
+            if (fallbackCertificate) newData.fallbackCertificate = fallbackCertificate;
+
             domaindb.update(domain, newData, function (error) {
                 if (error) return callback(error);
 
@@ -323,7 +314,7 @@ function del(domain, auditSource, callback) {
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (domain === settings.adminDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain'));
+    if (domain === settings.dashboardDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain'));
     if (domain === settings.mailDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove mail domain. Change the mail server location first'));
 
     domaindb.del(domain, function (error) {
@@ -398,7 +389,7 @@ function upsertDnsRecords(location, domain, type, values, callback) {
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     debug('upsertDNSRecord: %s on %s type %s values', location, domain, type, values);
@@ -418,7 +409,7 @@ function removeDnsRecords(location, domain, type, values, callback) {
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     debug('removeDNSRecord: %s on %s type %s values', location, domain, type, values);
@@ -578,8 +569,8 @@ function syncDnsRecords(options, progressCallback, callback) {
                 progress += Math.round(100/(1+domains.length));
 
                 let locations = [];
-                if (domain.domain === settings.adminDomain()) locations.push({ subdomain: constants.ADMIN_LOCATION, domain: settings.adminDomain() });
-                if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.adminFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
+                if (domain.domain === settings.dashboardDomain()) locations.push({ subdomain: constants.DASHBOARD_LOCATION, domain: settings.dashboardDomain() });
+                if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.dashboardFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
 
                 allApps.forEach(function (app) {
                     const appLocations = [{ subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains);

src/dyndns.js

@@ -1,7 +1,7 @@
 'use strict';
 
 exports = module.exports = {
-    sync: sync
+    sync
 };
 
 let apps = require('./apps.js'),
@@ -32,7 +32,7 @@ function sync(auditSource, callback) {
 
         debug(`refreshDNS: updating ip from ${info.ip} to ${ip}`);
 
-        domains.upsertDnsRecords(constants.ADMIN_LOCATION, settings.adminDomain(), 'A', [ ip ], function (error) {
+        domains.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'A', [ ip ], function (error) {
             if (error) return callback(error);
 
             debug('refreshDNS: updated admin location');

src/eventlog.js

@@ -2,11 +2,11 @@
 
 exports = module.exports = {
     add,
-    upsert,
+    upsertLoginEvent,
     get,
     getAllPaged,
-    getByCreationTime,
     cleanup,
+    _clear: clear,
 
     // keep in sync with webadmin index.js filter
     ACTION_ACTIVATE: 'cloudron.activate',
@@ -15,6 +15,7 @@ exports = module.exports = {
     ACTION_APP_REPAIR: 'app.repair',
     ACTION_APP_INSTALL: 'app.install',
     ACTION_APP_RESTORE: 'app.restore',
+    ACTION_APP_IMPORT: 'app.import',
     ACTION_APP_UNINSTALL: 'app.uninstall',
     ACTION_APP_UPDATE: 'app.update',
     ACTION_APP_UPDATE_FINISH: 'app.update.finish',
@@ -75,94 +76,119 @@ exports = module.exports = {
     ACTION_PROCESS_CRASH: 'system.crash'
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
+    database = require('./database.js'),
     debug = require('debug')('box:eventlog'),
-    eventlogdb = require('./eventlogdb.js'),
+    mysql = require('mysql'),
     notifications = require('./notifications.js'),
-    util = require('util'),
+    safe = require('safetydance'),
     uuid = require('uuid');
 
-var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+const EVENTLOG_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
 
-function add(action, source, data, callback) {
+function postProcess(record) {
+    // usually we have sourceJson and dataJson, however since this used to be the JSON data type, we don't
+    record.source = safe.JSON.parse(record.source);
+    record.data = safe.JSON.parse(record.data);
+
+    return record;
+}
+
+// never throws, only logs because previously code did not take a callback
+async function add(action, source, data) {
     assert.strictEqual(typeof action, 'string');
     assert.strictEqual(typeof source, 'object');
     assert.strictEqual(typeof data, 'object');
-    assert(!callback || typeof callback === 'function');
 
-    callback = callback || NOOP_CALLBACK;
-
-    eventlogdb.add(uuid.v4(), action, source, data, function (error, id) {
-        if (error) return callback(error);
-
-        callback(null, { id: id });
-
-        notifications.onEvent(id, action, source, data, NOOP_CALLBACK);
-    });
+    const id = uuid.v4();
+    try {
+        await database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ]);
+        await notifications.onEvent(id, action, source, data);
+        return id;
+    } catch (error) {
+        debug('add: error adding event', error);
+        return null;
+    }
 }
 
-function upsert(action, source, data, callback) {
+// never throws, only logs because previously code did not take a callback
+async function upsertLoginEvent(action, source, data) {
     assert.strictEqual(typeof action, 'string');
     assert.strictEqual(typeof source, 'object');
     assert.strictEqual(typeof data, 'object');
-    assert(!callback || typeof callback === 'function');
 
-    callback = callback || NOOP_CALLBACK;
+    // can't do a real sql upsert, for frequent eventlog entries we only have to do 2 queries once a day
+    const queries = [{
+        query: 'UPDATE eventlog SET creationTime=NOW(), data=? WHERE action = ? AND source LIKE ? AND DATE(creationTime)=CURDATE()',
+        args: [ JSON.stringify(data), action, JSON.stringify(source) ]
+    }, {
+        query: 'SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog WHERE action = ? AND source LIKE ? AND DATE(creationTime)=CURDATE()',
+        args: [ action, JSON.stringify(source) ]
+    }];
 
-    eventlogdb.upsert(uuid.v4(), action, source, data, function (error, id) {
-        if (error) return callback(error);
+    try {
+        const result = await database.transaction(queries);
+        if (result[0].affectedRows >= 1) return result[1][0].id;
 
-        callback(null, { id: id });
-
-        notifications.onEvent(id, action, source, data, NOOP_CALLBACK);
-    });
+        // no existing eventlog found, create one
+        return await add(action, source, data);
+    } catch (error) {
+        debug('add: error adding event', error);
+        return null;
+    }
 }
 
-function get(id, callback) {
+async function get(id) {
     assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    eventlogdb.get(id, function (error, result) {
-        if (error) return callback(error);
+    const result = await database.query('SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog WHERE id = ?', [ id ]);
+    if (result.length === 0) return null;
 
-        callback(null, result);
-    });
+    return postProcess(result[0]);
 }
 
-function getAllPaged(actions, search, page, perPage, callback) {
+async function getAllPaged(actions, search, page, perPage) {
     assert(Array.isArray(actions));
     assert(typeof search === 'string' || search === null);
     assert.strictEqual(typeof page, 'number');
     assert.strictEqual(typeof perPage, 'number');
-    assert.strictEqual(typeof callback, 'function');
 
-    eventlogdb.getAllPaged(actions, search, page, perPage, function (error, events) {
-        if (error) return callback(error);
+    let data = [];
+    let query = `SELECT ${EVENTLOG_FIELDS} FROM eventlog`;
 
-        callback(null, events);
+    if (actions.length || search) query += ' WHERE';
+    if (search) query += ' (source LIKE ' + mysql.escape('%' + search + '%') + ' OR data LIKE ' + mysql.escape('%' + search + '%') + ')';
+
+    if (actions.length && search) query += ' AND ( ';
+    actions.forEach(function (action, i) {
+        query += ' (action LIKE ' + mysql.escape(`%${action}%`) + ') ';
+        if (i < actions.length-1) query += ' OR ';
     });
+    if (actions.length && search) query += ' ) ';
+
+    query += ' ORDER BY creationTime DESC LIMIT ?,?';
+
+    data.push((page-1)*perPage);
+    data.push(perPage);
+
+    const results = await database.query(query, data);
+    results.forEach(postProcess);
+    return results;
 }
 
-function getByCreationTime(creationTime, callback) {
-    assert(util.isDate(creationTime));
-    assert.strictEqual(typeof callback, 'function');
+async function cleanup(options) {
+    assert.strictEqual(typeof options, 'object');
 
-    eventlogdb.getByCreationTime(creationTime, function (error, events) {
-        if (error) return callback(error);
+    const creationTime = options.creationTime;
 
-        callback(null, events);
-    });
+    const results = await database.query('SELECT * FROM eventlog WHERE creationTime <= ?', [ creationTime ]);
+
+    for (const result of results) {
+        await database.query('DELETE FROM notifications WHERE eventId=?', [ result.id ]); // remove notifications that reference the events as well
+        await database.query('DELETE FROM eventlog WHERE id=?', [ result.id ]);
+    }
 }
 
-function cleanup(callback) {
-    callback = callback || NOOP_CALLBACK;
-
-    var d = new Date();
-    d.setDate(d.getDate() - 10); // 10 days ago
-
-    eventlogdb.delByCreationTime(d, function (error) {
-        if (error) return callback(error);
-
-        callback(null);
-    });
+async function clear() {
+    await database.query('DELETE FROM eventlog');
 }

src/eventlogdb.js

@@ -1,171 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get: get,
-    getAllPaged: getAllPaged,
-    getByCreationTime: getByCreationTime,
-    add: add,
-    upsert: upsert,
-    count: count,
-    delByCreationTime: delByCreationTime,
-
-    _clear: clear
-};
-
-var assert = require('assert'),
-    async = require('async'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js'),
-    mysql = require('mysql'),
-    safe = require('safetydance'),
-    util = require('util');
-
-var EVENTLOG_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
-
-function postProcess(eventLog) {
-    // usually we have sourceJson and dataJson, however since this used to be the JSON data type, we don't
-    eventLog.source = safe.JSON.parse(eventLog.source);
-    eventLog.data = safe.JSON.parse(eventLog.data);
-
-    return eventLog;
-}
-
-function get(eventId, callback) {
-    assert.strictEqual(typeof eventId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog WHERE id = ?', [ eventId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Eventlog not found'));
-
-        callback(null, postProcess(result[0]));
-    });
-}
-
-function getAllPaged(actions, search, page, perPage, callback) {
-    assert(Array.isArray(actions));
-    assert(typeof search === 'string' || search === null);
-    assert.strictEqual(typeof page, 'number');
-    assert.strictEqual(typeof perPage, 'number');
-    assert.strictEqual(typeof callback, 'function');
-
-    var data = [];
-    var query = 'SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog';
-
-    if (actions.length || search) query += ' WHERE';
-    if (search) query += ' (source LIKE ' + mysql.escape('%' + search + '%') + ' OR data LIKE ' + mysql.escape('%' + search + '%') + ')';
-
-    if (actions.length && search) query += ' AND ( ';
-    actions.forEach(function (action, i) {
-        query += ' (action LIKE ' + mysql.escape(`%${action}%`) + ') ';
-        if (i < actions.length-1) query += ' OR ';
-    });
-    if (actions.length && search) query += ' ) ';
-
-    query += ' ORDER BY creationTime DESC LIMIT ?,?';
-
-    data.push((page-1)*perPage);
-    data.push(perPage);
-
-    database.query(query, data, function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        results.forEach(postProcess);
-
-        callback(null, results);
-    });
-}
-
-function getByCreationTime(creationTime, callback) {
-    assert(util.isDate(creationTime));
-    assert.strictEqual(typeof callback, 'function');
-
-    var query = 'SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog WHERE creationTime >= ? ORDER BY creationTime DESC';
-    database.query(query, [ creationTime ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        results.forEach(postProcess);
-
-        callback(null, results);
-    });
-}
-
-function add(id, action, source, data, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof action, 'string');
-    assert.strictEqual(typeof source, 'object');
-    assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ], function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
-        if (error || result.affectedRows !== 1) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, id);
-    });
-}
-
-// id is only used if we didn't do an update but insert instead
-function upsert(id, action, source, data, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof action, 'string');
-    assert.strictEqual(typeof source, 'object');
-    assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    // can't do a real sql upsert, for frequent eventlog entries we only have to do 2 queries once a day
-    var queries = [{
-        query: 'UPDATE eventlog SET creationTime=NOW(), data=? WHERE action = ? AND source LIKE ? AND DATE(creationTime)=CURDATE()',
-        args: [ JSON.stringify(data), action, JSON.stringify(source) ]
-    }, {
-        query: 'SELECT ' + EVENTLOG_FIELDS + ' FROM eventlog WHERE action = ? AND source LIKE ? AND DATE(creationTime)=CURDATE()',
-        args: [ action, JSON.stringify(source) ]
-    }];
-
-    database.transaction(queries, function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result[0].affectedRows >= 1) return callback(null, result[1][0].id);
-
-        // no existing eventlog found, create one
-        add(id, action, source, data, callback);
-    });
-}
-
-function count(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT COUNT(*) AS total FROM eventlog', function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        return callback(null, result[0].total);
-    });
-}
-
-function clear(callback) {
-    database.query('DELETE FROM eventlog', function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function delByCreationTime(creationTime, callback) {
-    assert(util.isDate(creationTime));
-    assert.strictEqual(typeof callback, 'function');
-
-    // remove notifications that reference the events as well
-    database.query('SELECT * FROM eventlog WHERE creationTime <= ?', [ creationTime ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        async.eachSeries(result, function (item, iteratorCallback) {
-            async.series([
-                database.query.bind(null, 'DELETE FROM notifications WHERE eventId=?', [ item.id ]),
-                database.query.bind(null, 'DELETE FROM eventlog WHERE id=?', [ item.id ])
-            ], iteratorCallback);
-        }, function (error) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            callback(null);
-        });
-    });
-}

src/externalldap.js

@@ -1,17 +1,17 @@
 'use strict';
 
 exports = module.exports = {
-    search: search,
-    verifyPassword: verifyPassword,
-    createAndVerifyUserIfNotExist: createAndVerifyUserIfNotExist,
+    search,
+    verifyPassword,
+    createAndVerifyUserIfNotExist,
 
-    testConfig: testConfig,
-    startSyncer: startSyncer,
+    testConfig,
+    startSyncer,
 
-    injectPrivateFields: injectPrivateFields,
-    removePrivateFields: removePrivateFields,
+    injectPrivateFields,
+    removePrivateFields,
 
-    sync: sync
+    sync
 };
 
 var assert = require('assert'),

src/graphite.js

@@ -1,50 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    start,
-
-    DEFAULT_MEMORY_LIMIT: 256 * 1024 * 1024
-};
-
-var assert = require('assert'),
-    async = require('async'),
-    infra = require('./infra_version.js'),
-    paths = require('./paths.js'),
-    shell = require('./shell.js'),
-    system = require('./system.js');
-
-function start(existingInfra, serviceConfig, callback) {
-    assert.strictEqual(typeof existingInfra, 'object');
-    assert.strictEqual(typeof serviceConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const tag = infra.images.graphite.tag;
-    const dataDir = paths.PLATFORM_DATA_DIR;
-    const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT;
-    const memory = system.getMemoryAllocation(memoryLimit);
-
-    const cmd = `docker run --restart=always -d --name="graphite" \
-                --hostname graphite \
-                --net cloudron \
-                --net-alias graphite \
-                --log-driver syslog \
-                --log-opt syslog-address=udp://127.0.0.1:2514 \
-                --log-opt syslog-format=rfc5424 \
-                --log-opt tag=graphite \
-                -m ${memory} \
-                --memory-swap ${memoryLimit} \
-                --dns 172.18.0.1 \
-                --dns-search=. \
-                -p 127.0.0.1:2003:2003 \
-                -p 127.0.0.1:2004:2004 \
-                -p 127.0.0.1:8417:8000 \
-                -v "${dataDir}/graphite:/var/lib/graphite" \
-                --label isCloudronManaged=true \
-                --read-only -v /tmp -v /run "${tag}"`;
-
-    async.series([
-        shell.exec.bind(null, 'stopGraphite', 'docker stop graphite || true'),
-        shell.exec.bind(null, 'removeGraphite', 'docker rm -f graphite || true'),
-        shell.exec.bind(null, 'startGraphite', cmd)
-    ], callback);
-}

src/groupdb.js

@@ -1,24 +1,24 @@
 'use strict';
 
 exports = module.exports = {
-    get: get,
-    getByName: getByName,
-    getWithMembers: getWithMembers,
-    getAll: getAll,
-    getAllWithMembers: getAllWithMembers,
-    add: add,
-    update: update,
-    del: del,
-    count: count,
+    get,
+    getByName,
+    getWithMembers,
+    getAll,
+    getAllWithMembers,
+    add,
+    update,
+    del,
+    count,
 
-    getMembers: getMembers,
-    addMember: addMember,
-    removeMember: removeMember,
-    setMembers: setMembers,
-    isMember: isMember,
+    getMembers,
+    addMember,
+    removeMember,
+    setMembers,
+    isMember,
 
-    getMembership: getMembership,
-    setMembership: setMembership,
+    getMembership,
+    setMembership,
 
     _clear: clear
 };

src/infra_version.js

@@ -6,7 +6,7 @@
 
 exports = module.exports = {
     // a version change recreates all containers with latest docker config
-    'version': '48.18.0',
+    'version': '48.20.0',
 
     'baseImages': [
         { repo: 'cloudron/base', tag: 'cloudron/base:3.0.0@sha256:455c70428723e3a823198c57472785437eb6eab082e79b3ff04ea584faf46e92' }
@@ -15,13 +15,13 @@ exports = module.exports = {
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.3.0@sha256:386fb755fc41edd7086f7bcb230f7f28078936f9ae4ead6d97c741df1cc194ae' },
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.0.4@sha256:4d688c746f27b195d98f35a7d24ec01f3f754e0ca61e9de0b0bc9793553880f1' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.0.2@sha256:424081fd38ebd35f3606c64f8f99138570e5f4d5066f12cfb4142447d249d3e7' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.0.1@sha256:ad20a9a5dcb2ab132374a7c8d44b89af0ec37651cf889e570f7625b02ee85fdf' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.0.2@sha256:caaa1f7f4055ae8990d8ec65bd100567496df7e4ed5eb427867f3717a8dcbf92' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.2.3@sha256:fdc4aa6d2c85aeafe65eaa4243aada0cc2e57b94f6eaee02c9b1a8fb89b01dd7' },
-        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.4.0@sha256:953bbd8b72a9108a8526d2c0bdbba67e1e1563ff59d0a117f0884dba1576f3dd' },
-        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.2.0@sha256:61e8247ded1e07cf882ca478dab180960357c614472e80b938f1f690a46788c2' }
+        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.3.1@sha256:759cafab7625ff538418a1f2ed5558b1d5bff08c576bba577d865d6d02b49091' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.0.7@sha256:6679c2fb96f8d6d62349b607748570640a90fc46b50aad80ca2c0161655d07f4' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.0.6@sha256:e583082e15e8e41b0e3b80c3efc917ec429f19fa08a19e14fc27144a8bfe446a' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.0.2@sha256:9df297ccc3370f38c54f8d614e214e082b363777cd1c6c9522e29663cc8f5362' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.0.3@sha256:37e5222e01ae89bc5a742ce12030631de25a127b5deec8a0e992c68df0fdec10' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.3.3@sha256:b1093e6f38bebf4a9ae903ca385aea3a32e7cccae5ede7f2e01a34681e361a5f' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.0.1@sha256:bed9f6b5d06fe2c5289e895e806cfa5b74ad62993d705be55d4554a67d128029' },
+        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.3.0@sha256:183c11150d5a681cb02f7d2bd542ddb8a8f097422feafb7fac8fdbca0ca55d47' }
     }
 };

src/janitor.js

@@ -1,35 +1,29 @@
 'use strict';
 
-var assert = require('assert'),
+const assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:janitor'),
     Docker = require('dockerode'),
-    tokendb = require('./tokendb.js');
+    safe = require('safetydance'),
+    tokens = require('./tokens.js');
 
 exports = module.exports = {
-    cleanupTokens: cleanupTokens,
-    cleanupDockerVolumes: cleanupDockerVolumes
+    cleanupTokens,
+    cleanupDockerVolumes
 };
 
 const NOOP_CALLBACK = function () { };
 
 const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });
 
-function cleanupTokens(callback) {
-    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
-
-    callback = callback || NOOP_CALLBACK;
-
+async function cleanupTokens() {
     debug('Cleaning up expired tokens');
 
-    tokendb.delExpired(function (error, result) {
-        if (error) return debug('cleanupTokens: error removing expired tokens', error);
+    const [error, result] = await safe(tokens.delExpired());
+    if (error) return debug('cleanupTokens: error removing expired tokens', error);
 
-        debug('Cleaned up %s expired tokens.', result);
-
-        callback(null);
-    });
+    debug(`Cleaned up ${result} expired tokens`,);
 }
 
 function cleanupTmpVolume(containerInfo, callback) {

src/ldap.js

@@ -17,7 +17,6 @@ const assert = require('assert'),
     ldap = require('ldapjs'),
     mail = require('./mail.js'),
     mailboxdb = require('./mailboxdb.js'),
-    path = require('path'),
     safe = require('safetydance'),
     services = require('./services.js'),
     users = require('./users.js');
@@ -264,7 +263,9 @@ function mailboxSearch(req, res, next) {
             if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
             if (error) return next(new ldap.OperationsError(error.toString()));
 
-            var obj = {
+            if (!mailbox.active) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+            let obj = {
                 dn: req.dn.toString(),
                 attributes: {
                     objectclass: ['mailbox'],
@@ -289,10 +290,11 @@ function mailboxSearch(req, res, next) {
         var domain = req.dn.rdns[0].attrs.domain.value.toLowerCase();
 
         mailboxdb.listMailboxes(domain, 1, 1000, function (error, mailboxes) {
-
             if (error) return next(new ldap.OperationsError(error.toString()));
 
-            var results = [];
+            mailboxes = mailboxes.filter(m => m.active);
+
+            let results = [];
 
             // send mailbox objects
             mailboxes.forEach(function (mailbox) {
@@ -323,7 +325,9 @@ function mailboxSearch(req, res, next) {
         mailboxdb.listAllMailboxes(1, 1000, function (error, mailboxes) {
             if (error) return next(new ldap.OperationsError(error.toString()));
 
-            var results = [];
+            mailboxes = mailboxes.filter(m => m.active);
+
+            let results = [];
 
             // send mailbox objects
             async.eachSeries(mailboxes, function (mailbox, callback) {
@@ -382,10 +386,12 @@ function mailAliasSearch(req, res, next) {
         if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.toString()));
 
+        if (!alias.active) return next(new ldap.NoSuchObjectError(req.dn.toString())); // there is no way to disable an alias. this is just here for completeness
+
         // https://wiki.debian.org/LDAP/MigrationTools/Examples
         // https://docs.oracle.com/cd/E19455-01/806-5580/6jej518pp/index.html
         // member is fully qualified - https://docs.oracle.com/cd/E19957-01/816-6082-10/chap4.doc.html#43314
-        var obj = {
+        let obj = {
             dn: req.dn.toString(),
             attributes: {
                 objectclass: ['nisMailAlias'],
@@ -421,6 +427,8 @@ function mailingListSearch(req, res, next) {
         if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.toString()));
 
+        if (!list.active) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
         // http://ldapwiki.willeke.com/wiki/Original%20Mailgroup%20Schema%20From%20Netscape
         // members are fully qualified (https://docs.oracle.com/cd/E19444-01/816-6018-10/groups.htm#13356)
         var obj = {
@@ -482,13 +490,13 @@ function authorizeUserForApp(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
     assert.strictEqual(typeof req.app, 'object');
 
-    apps.hasAccessTo(req.app, req.user, function (error, hasAccess) {
+    apps.hasAccessTo(req.app, req.user, async function (error, hasAccess) {
         if (error) return next(new ldap.OperationsError(error.toString()));
 
         // we return no such object, to avoid leakage of a users existence
         if (!hasAccess) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-        eventlog.upsert(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: req.app.id }, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+        eventlog.upsertLoginEvent(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: req.app.id }, { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
         res.end();
     });
@@ -537,12 +545,15 @@ function authenticateUserMailbox(req, res, next) {
             if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
             if (error) return next(new ldap.OperationsError(error.message));
 
-            verifyMailboxPassword(mailbox, req.credentials || '', function (error, result) {
+            if (!mailbox.active) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+            verifyMailboxPassword(mailbox, req.credentials || '', async function (error, result) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
 
-                eventlog.upsert(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
+                eventlog.upsertLoginEvent(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
+
                 res.end();
             });
         });
@@ -613,7 +624,7 @@ function userSearchSftp(req, res, next) {
                 var obj = {
                     dn: ldap.parseDN(`cn=${username}@${appFqdn},ou=sftp,dc=cloudron`).toString(),
                     attributes: {
-                        homeDirectory: path.join('/app/data', app.id),
+                        homeDirectory: app.dataDir ? `/mnt/${app.id}` : `/mnt/appsdata/${app.id}/data`,
                         objectclass: ['user'],
                         objectcategory: 'person',
                         cn: user.id,
@@ -677,12 +688,15 @@ function authenticateMailAddon(req, res, next) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
 
-                verifyMailboxPassword(mailbox, req.credentials || '', function (error, result) {
+                if (!mailbox.active) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+                verifyMailboxPassword(mailbox, req.credentials || '', async function (error, result) {
                     if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                     if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                     if (error) return next(new ldap.OperationsError(error.message));
 
-                    eventlog.upsert(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
+                    eventlog.upsertLoginEvent(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
+
                     res.end();
                 });
             });

src/mail.js

@@ -39,7 +39,7 @@ exports = module.exports = {
     listMailboxes,
     getMailbox,
     addMailbox,
-    updateMailboxOwner,
+    updateMailbox,
     removeMailbox,
 
     getAliases,
@@ -633,7 +633,7 @@ function configureMail(mailFqdn, mailDomain, serviceConfig, callback) {
     assert.strictEqual(typeof serviceConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    // mail (note: 2525 is hardcoded in mail container and app use this port)
+    // mail (note: 2587 is hardcoded in mail container and app use this port)
     // MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
     // MAIL_DOMAIN is the domain for which this server is relaying mails
     // mail container uses /app/data for backed up data and /run for restart-able data
@@ -643,13 +643,14 @@ function configureMail(mailFqdn, mailDomain, serviceConfig, callback) {
     const memory = system.getMemoryAllocation(memoryLimit);
     const cloudronToken = hat(8 * 128), relayToken = hat(8 * 128);
 
-    reverseProxy.getCertificate(mailFqdn, mailDomain, function (error, bundle) {
+    reverseProxy.getCertificatePath(mailFqdn, mailDomain, function (error, bundle) {
         if (error) return callback(error);
 
-        // the setup script copies dhparams.pem to /addons/mail
+        const dhparamsFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/dhparams.pem');
         const mailCertFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_cert.pem');
         const mailKeyFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_key.pem');
 
+        if (!safe.child_process.execSync(`cp ${paths.DHPARAMS_FILE} ${dhparamsFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not copy dhparams:' + safe.error.message));
         if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create cert file:' + safe.error.message));
         if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create key file:' + safe.error.message));
 
@@ -662,7 +663,7 @@ function configureMail(mailFqdn, mailDomain, serviceConfig, callback) {
             createMailConfig(mailFqdn, mailDomain, function (error, allowInbound) {
                 if (error) return callback(error);
 
-                var ports = allowInbound ? '-p 587:2525 -p 993:9993 -p 4190:4190 -p 25:2525' : '';
+                var ports = allowInbound ? '-p 587:2587 -p 993:9993 -p 4190:4190 -p 25:2587' : '';
 
                 const cmd = `docker run --restart=always -d --name="mail" \
                             --net cloudron \
@@ -722,8 +723,8 @@ function restartMail(callback) {
     services.getServiceConfig('mail', function (error, serviceConfig) {
         if (error) return callback(error);
 
-        debug(`restartMail: restarting mail container with ${settings.mailFqdn()} ${settings.adminDomain()}`);
-        configureMail(settings.mailFqdn(), settings.adminDomain(), serviceConfig, callback);
+        debug(`restartMail: restarting mail container with mailFqdn:${settings.mailFqdn()} dashboardDomain:${settings.dashboardDomain()}`);
+        configureMail(settings.mailFqdn(), settings.dashboardDomain(), serviceConfig, callback);
     });
 }
 
@@ -1182,14 +1183,18 @@ function getMailbox(name, domain, callback) {
     });
 }
 
-function addMailbox(name, domain, ownerId, ownerType, auditSource, callback) {
+function addMailbox(name, domain, data, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof ownerId, 'string');
-    assert.strictEqual(typeof ownerType, 'string');
+    assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    const { ownerId, ownerType, active } = data;
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
+    assert.strictEqual(typeof active, 'boolean');
+
     name = name.toLowerCase();
 
     var error = validateName(name);
@@ -1197,23 +1202,27 @@ function addMailbox(name, domain, ownerId, ownerType, auditSource, callback) {
 
     if (ownerType !== exports.OWNERTYPE_USER && ownerType !== exports.OWNERTYPE_GROUP) return callback(new BoxError(BoxError.BAD_FIELD, 'bad owner type'));
 
-    mailboxdb.addMailbox(name, domain, ownerId, ownerType, function (error) {
+    mailboxdb.addMailbox(name, domain, data, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType });
+        eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active });
 
         callback(null);
     });
 }
 
-function updateMailboxOwner(name, domain, ownerId, ownerType, auditSource, callback) {
+function updateMailbox(name, domain, data, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof ownerId, 'string');
-    assert.strictEqual(typeof ownerType, 'string');
+    assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    const { ownerId, ownerType, active } = data;
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
+    assert.strictEqual(typeof active, 'boolean');
+
     name = name.toLowerCase();
 
     if (ownerType !== exports.OWNERTYPE_USER && ownerType !== exports.OWNERTYPE_GROUP) return callback(new BoxError(BoxError.BAD_FIELD, 'bad owner type'));
@@ -1221,10 +1230,10 @@ function updateMailboxOwner(name, domain, ownerId, ownerType, auditSource, callb
     getMailbox(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        mailboxdb.updateMailboxOwner(name, domain, ownerId, ownerType, function (error) {
+        mailboxdb.updateMailbox(name, domain, data, function (error) {
             if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: result.userId, ownerId, ownerType });
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: result.userId, ownerId, ownerType, active });
 
             callback(null);
         });
@@ -1337,14 +1346,18 @@ function getList(name, domain, callback) {
     });
 }
 
-function addList(name, domain, members, membersOnly, auditSource, callback) {
+function addList(name, domain, data, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof name, 'string');
-    assert(Array.isArray(members));
-    assert.strictEqual(typeof membersOnly, 'boolean');
+    assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    const { members, membersOnly, active } = data;
+    assert(Array.isArray(members));
+    assert.strictEqual(typeof membersOnly, 'boolean');
+    assert.strictEqual(typeof active, 'boolean');
+
     name = name.toLowerCase();
 
     var error = validateName(name);
@@ -1354,23 +1367,27 @@ function addList(name, domain, members, membersOnly, auditSource, callback) {
         if (!validator.isEmail(members[i])) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid mail member: ' + members[i]));
     }
 
-    mailboxdb.addList(name, domain, members, membersOnly, function (error) {
+    mailboxdb.addList(name, domain, data, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly, active });
 
         callback();
     });
 }
 
-function updateList(name, domain, members, membersOnly, auditSource, callback) {
+function updateList(name, domain, data, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert(Array.isArray(members));
-    assert.strictEqual(typeof membersOnly, 'boolean');
+    assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    const { members, membersOnly, active } = data;
+    assert(Array.isArray(members));
+    assert.strictEqual(typeof membersOnly, 'boolean');
+    assert.strictEqual(typeof active, 'boolean');
+
     name = name.toLowerCase();
 
     var error = validateName(name);
@@ -1383,10 +1400,10 @@ function updateList(name, domain, members, membersOnly, auditSource, callback) {
     getList(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        mailboxdb.updateList(name, domain, members, membersOnly, function (error) {
+        mailboxdb.updateList(name, domain, data, function (error) {
             if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly });
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly, active });
 
             callback(null);
         });

src/mail_templates/app_down.ejs

@@ -1,21 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear Cloudron Admin,
-
-The application '<%= title %>' installed at <%= appFqdn %> is not responding.
-
-This is most likely a problem in the application.
-
-To resolve this, you can try the following:
-* Restart the app by opening the app's web terminal - https://docs.cloudron.io/apps/#web-terminal
-* Restore the app to the latest backup - https://docs.cloudron.io/backups/#restoring-an-app
-* Contact us via <%= supportEmail %> or https://forum.cloudron.io
-
-
-Powered by https://cloudron.io
-
-Sent at: <%= new Date().toUTCString() %>
-
-<% } else { %>
-
-<% } %>

src/mail_templates/app_up.ejs

@@ -1,14 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear Cloudron Admin,
-
-The application '<%= title %>' installed at <%= appFqdn %> is back online
-and responding to health checks.
-
-Powered by https://cloudron.io
-
-Sent at: <%= new Date().toUTCString() %>
-
-<% } else { %>
-
-<% } %>

src/mail_templates/app_updated.ejs

@@ -1,40 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear Cloudron Admin,
-
-The application '<%= title %>' installed at <%= appFqdn %> was updated to app package version <%= version %>.
-
-Changes:
-<%= changelog %>
-
-Powered by https://cloudron.io
-
-Sent at: <%= new Date().toUTCString() %>
-
-<% } else { %>
-
-    <center>
-
-        <img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
-
-        <h3>Dear <%= cloudronName %> Admin,</h3>
-
-        <br/>
-
-        <div style="width: 650px; text-align: left;">
-            The application '<%= title %>' installed at <%= appFqdn %> was updated to app package version <%= version %>.
-
-            <h5>Changelog:</h5>
-            <%- changelogHTML %>
-        </div>
-
-        <br/>
-        <br/>
-
-        <div style="font-size: 10px; color: #333333; background: #ffffff;">
-            Powered by <a href="https://cloudron.io">Cloudron</a>.<br/>
-            Sent at: <%= new Date().toUTCString() %>
-        </div>
-
-    </center>
-<% } %>

src/mail_templates/app_updates_available.ejs

@@ -1,55 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear Cloudron Admin,
-
-<% for (var i = 0; i < apps.length; i++) { -%>
-The app '<%= apps[i].app.manifest.title %>' installed at <%= apps[i].app.fqdn %> has an update available.
-
-<%= apps[i].app.manifest.title %> v<%= apps[i].updateInfo.manifest.version %> changes:
-<%= apps[i].updateInfo.manifest.changelog %>
-
-<% } -%>
-
-Update now at <%= webadminUrl %>
-
-Powered by https://cloudron.io
-
-Sent at: <%= new Date().toUTCString() %>
-
-<% } else { %>
-
-    <center>
-
-    <img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
-
-    <h3>Dear <%= cloudronName %> Admin,</h3>
-
-    <br/>
-
-    <div style="width: 650px; text-align: left;">
-<% for (var i = 0; i < apps.length; i++) { -%>
-        <p>
-            The app '<%= apps[i].app.manifest.title %>' installed at <a href="https://<%= apps[i].app.fqdn %>"><%= apps[i].app.fqdn %></a> has an update available.
-        </p>
-
-        <h5><%= apps[i].app.manifest.title %> v<%= apps[i].updateInfo.manifest.version %> changes:</h5>
-        <%- apps[i].changelogHTML %>
-
-        <br/>
-<% } -%>
-
-        <p>
-            <br/>
-            <center><a href="<%= webadminUrl %>">Update now</a></center>
-            <br/>
-        </p>
-    </div>
-
-    <div style="font-size: 10px; color: #333333; background: #ffffff;">
-        Powered by <a href="https://cloudron.io">Cloudron</a>.<br/>
-        Sent at: <%= new Date().toUTCString() %>
-    </div>
-
-    </center>
-
-<% } %>

src/mail_templates/box_update_available.ejs

@@ -1,45 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear <%= cloudronName %> Admin,
-
-Cloudron v<%= newBoxVersion %> is now available!
-
-Changes:
-<% for (var i = 0; i < changelog.length; i++) { %>
-  * <%- changelog[i] %>
-<% } %>
-
-Powered by https://cloudron.io
-
-Sent at: <%= new Date().toUTCString() %>
-
-<% } else { %>
-
-<center>
-
-<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
-
-<h3>Dear <%= cloudronName %> Admin,</h3>
-
-<div style="width: 650px; text-align: left;">
-    <p>
-        Cloudron v<%= newBoxVersion %> is now available!
-    </p>
-
-    <h5>Changes:</h5>
-    <ul>
-    <% for (var i = 0; i < changelogHTML.length; i++) { %>
-        <li><%- changelogHTML[i] %></li>
-    <% } %>
-    </ul>
-
-    <br/>
-</div>
-
-<div style="font-size: 10px; color: #333333; background: #ffffff;">
-    Powered by <a href="https://cloudron.io">Cloudron</a>.
-</div>
-
-</center>
-
-<% } %>

src/mail_templates/box_update_error.ejs

@@ -1,20 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear Cloudron Admin,
-
-Cloudron update failed because of the following reason:
-
--------------------------------------
-
-<%- message %>
-
--------------------------------------
-
-
-Powered by https://cloudron.io
-
-Sent at: <%= new Date().toUTCString() %>
-
-<% } else { %>
-
-<% } %>

src/mail_templates/new_login_location-html.ejs

@@ -0,0 +1,22 @@
+<center>
+
+<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
+
+<h3>We've noticed a new login on your Cloudron account.</h3>
+
+<p>Hi <%= user %>,</p>
+
+<p>We noticed a login on your Cloudron account from a new device.</p>
+<p>IP: <%= ip %> (<%= city %>, <%= country %>)</p>
+<p>Browser: <%= userAgent %></p>
+
+<p>If this was you, you can safely disregard this email. If this wasn't you, you should change your password immediately.</p>
+
+<br/>
+<br/>
+
+<div style="font-size: 10px; color: #333333; background: #ffffff;">
+  Powered by <a href="https://cloudron.io">Cloudron</a>
+</div>
+
+</center>

src/mail_templates/new_login_location-text.ejs

@@ -0,0 +1,15 @@
+We've noticed a new login on your Cloudron account.
+
+Hi <%= user %>,
+
+We noticed a login on your Cloudron account from a new device.
+
+IP: <%= ip %> (<%= city %>, <%= country %>)
+
+Browser: <%= userAgent %> 
+
+If this was you, you can safely disregard this email. If this wasn't you, you should change your password immediately.
+
+Powered by https://cloudron.io
+
+Sent at: <%= new Date().toUTCString() %>

src/mail_templates/oom_event.ejs

@@ -1,27 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear <%= cloudronName %> Admin,
-
-<%if (app) { %>
-The application at <%= app.fqdn %> ran out of memory. The application has been restarted automatically. If you see this notification often,
-consider increasing the memory limit - <%= webadminUrl %>/#/app/<%= app.id %>/resources .
-<% } else { %>
-The addon <%= addon.name %> service ran out of memory. The service has been restarted automatically. If you see this notification often,
-consider increasing the memory limit - <%= webadminUrl %>/#/services .
-<% } %>
-
-Out of memory event:
-
--------------------------------------
-
-<%- event %>
-
--------------------------------------
-
-Powered by https://cloudron.io
-
-Sent at: <%= new Date().toUTCString() %>
-
-<% } else { %>
-
-<% } %>