coturn -> turn in docker repo name

https://nginx.org/en/linux_packages.html#Ubuntu
http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/

CHANGES

@@ -1730,3 +1730,140 @@
 * Fix registry detection in private images
 * Make mailbox domain configurable for apps
 
+[4.3.4]
+* Do not error if fallback certs went missing
+* Add 'New Apps' section to Appstore view
+* Fix issue where graphs of some apps were not appearing
+
+[4.4.0]
+* Show swap in graphs
+* Make avatars customizable
+* Hide access tokens from logs
+* Add missing '@' sign for email address in app mailbox
+* Add app fqdn to backup progress message
+* import: add option to import app in-place
+* import: add option to import app from arbitrary backup config
+* Show download progress for rsync backups
+* Fix various repair workflows
+* acme2: Implement post-as-get
+
+[4.4.1]
+* ami: fix AWS provider validation
+
+[4.4.2]
+* Fix crash when reporting that DKIM is not setup correctly
+* Stopped apps cannot be updated or auto-updated
+* eventlog: track support ticket creation and remote support status
+
+[4.4.3]
+* Add restart button in recovery section
+* Fix issue where memory usage was not computed correctly
+* cloudflare: support API tokens
+
+[4.4.4]
+* Fix bug where restart button in terminal was not working
+* Add search field in apps view
+* Make app view tags and domain filter persistent
+* Add timezone UI
+
+[4.4.5]
+* Fix user listing regression in group edit dialog
+* Do not show error page for 503
+* Add mail list and mail box update events
+* Certs of stopped apps are not renewed anymore
+* Fix broken memory sliders in the services UI
+* Set CPU Shares
+* Update nodejs to 12.14.1
+* Update MySQL addon packet size to 64M
+
+[5.0.0]
+* Show backup disk usage in graphs
+* Add per-user app passwords
+* Make app not responding page customizable
+* Make footer customizable
+* Add UI to import backups
+* Display timestamps in browser timezone in the UI
+* Mail eventlog and usage
+* Add user roles - owner, admin, user manager and user
+* Setup logrotate configs for collectd since upstream does not set it up
+* mail: Add X-Envelope-To and X-Envelope-From headers for incoming mails
+* linode: add object storage backend
+* restore: carefully replace backup config
+* spam: add default corpus and global db
+
+[5.0.1]
+* Show backup disk usage in graphs
+* Add per-user app passwords
+* Make app not responding page customizable
+* Make footer customizable
+* Add UI to import backups
+* Display timestamps in browser timezone in the UI
+* Mail eventlog and usage
+* Add user roles - owner, admin, user manager and user
+* Setup logrotate configs for collectd since upstream does not set it up
+* mail: Add X-Envelope-To and X-Envelope-From headers for incoming mails
+* linode: add object storage backend
+* restore: carefully replace backup config
+* spam: add default corpus and global db
+
+[5.0.2]
+* Show backup disk usage in graphs
+* Add per-user app passwords
+* Make app not responding page customizable
+* Make footer customizable
+* Add UI to import backups
+* Display timestamps in browser timezone in the UI
+* Mail eventlog and usage
+* Add user roles - owner, admin, user manager and user
+* Setup logrotate configs for collectd since upstream does not set it up
+* mail: Add X-Envelope-To and X-Envelope-From headers for incoming mails
+* linode: add object storage backend
+* restore: carefully replace backup config
+* spam: per mailbox bayes db and training
+
+[5.0.3]
+* Show backup disk usage in graphs
+* Add per-user app passwords
+* Make app not responding page customizable
+* Make footer customizable
+* Add UI to import backups
+* Display timestamps in browser timezone in the UI
+* Mail eventlog and usage
+* Add user roles - owner, admin, user manager and user
+* Setup logrotate configs for collectd since upstream does not set it up
+* mail: Add X-Envelope-To and X-Envelope-From headers for incoming mails
+* linode: add object storage backend
+* restore: carefully replace backup config
+* spam: per mailbox bayes db and training
+
+[5.0.4]
+* Fix potential previlige escalation because of ghost file
+* linode: dns backend
+* make branding routes owner only
+* add branding API
+* Add app start/stop/restart events
+* Use the primary email for LE account
+* make mail eventlog more descriptive
+
+[5.0.5]
+* Fix bug where incoming mail from dynamic hostnames was rejected
+* Increase token expiry
+* Fix bug in tag UI where tag removal did not work
+
+[5.0.6]
+* Make mail eventlog only visible to owners
+* Make app password work with sftp
+
+[5.1.0]
+* Add turn addon
+* Fix disk usage display
+* Drop support for TLSv1 and TLSv1.1
+* Make cert validation work for ECC certs
+* Add type filter to mail eventlog
+* mail: Fix listing of mailboxes and aliases in the UI
+* branding: fix login page title
+* Only a Cloudron owner can install/update/exec apps with the docker addon
+* security: reset tokens are only valid for a day
+* mail: fix eventlog db perms
+* Fix various bugs in the disk graphs
+

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2019 Cloudron UG
+Copyright (c) 2020 Cloudron UG
 
 With regard to the Cloudron Software:
 

baseimage/initializeBaseUbuntuImage.sh

@@ -44,7 +44,6 @@ apt-get -y install \
     linux-generic \
     logrotate \
     mysql-server-5.7 \
-    nginx-full \
     openssh-server \
     pwgen \
     resolvconf \
@@ -54,6 +53,17 @@ apt-get -y install \
     unbound \
     xfsprogs
 
+if [[ "${ubuntu_version}" == "16.04" ]]; then
+    echo "==> installing nginx for xenial for TLSv3 support"
+
+    curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.14.0-1~xenial_amd64.deb -o /tmp/nginx.deb
+    # apt install with install deps (as opposed to dpkg -i)
+    apt install -y /tmp/nginx.deb
+    rm /tmp/nginx.deb
+else
+    apt install -y nginx-full
+fi
+
 # on some providers like scaleway the sudo file is changed and we want to keep the old one
 apt-get -o Dpkg::Options::="--force-confold" install -y sudo
 
@@ -62,10 +72,10 @@ apt-get -o Dpkg::Options::="--force-confold" install -y sudo
 cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
 echo "==> Installing node.js"
-mkdir -p /usr/local/node-10.15.1
-curl -sL https://nodejs.org/dist/v10.15.1/node-v10.15.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.15.1
-ln -sf /usr/local/node-10.15.1/bin/node /usr/bin/node
-ln -sf /usr/local/node-10.15.1/bin/npm /usr/bin/npm
+mkdir -p /usr/local/node-10.18.1
+curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
+ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
+ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
 apt-get install -y python   # Install python which is required for npm rebuild
 [[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
@@ -123,6 +133,13 @@ timedatectl set-ntp 1
 # mysql follows the system timezone
 timedatectl set-timezone UTC
 
+echo "==> Adding sshd configuration warning"
+sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://cloudron.io/documentation/security/#securing-ssh-access' -i /etc/ssh/sshd_config
+
+# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
+echo "==> Disabling motd news"
+sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
+
 # Disable bind for good measure (on online.net, kimsufi servers these are pre-installed and conflicts with unbound)
 systemctl stop bind9 || true
 systemctl disable bind9 || true

migrations/20190911193842-mailboxes-make-members-fully-qualified.js

@@ -9,7 +9,7 @@ exports.up = function(db, callback) {
             if (!mailbox.membersJson) return iteratorDone();
 
             let members = JSON.parse(mailbox.membersJson);
-            members = members.map((m) => m.indexOf('@') === -1 ? `${m}@${mailbox.domain}` : m); // only because we don't do things in a xction
+            members = members.map((m) => m && m.indexOf('@') === -1 ? `${m}@${mailbox.domain}` : m); // only because we don't do things in a xction
 
             db.runSql('UPDATE mailboxes SET membersJson=? WHERE name=? AND domain=?', [ JSON.stringify(members), mailbox.name, mailbox.domain ], iteratorDone);
         }, callback);

migrations/20200101235723-domains-set-cloudflare-tokenType.js

@@ -0,0 +1,22 @@
+'use strict';
+
+let async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('SELECT * FROM domains', function (error, domains) {
+        if (error) return callback(error);
+
+        async.eachSeries(domains, function (domain, iteratorCallback) {
+            if (domain.provider !== 'cloudflare') return iteratorCallback();
+
+            let config = JSON.parse(domain.configJson);
+            config.tokenType = 'GlobalApiKey';
+
+            db.runSql('UPDATE domains SET configJson = ? WHERE domain = ?', [ JSON.stringify(config), domain.domain ], iteratorCallback);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20200129052822-apps-add-cpuShares.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN cpuShares INTEGER DEFAULT 512', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN cpuShares', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200131232251-appPasswords-create-table.js

@@ -0,0 +1,26 @@
+'use strict';
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE appPasswords(' +
+                'id VARCHAR(128) NOT NULL UNIQUE,' +
+				'name VARCHAR(128) NOT NULL,' +
+                'userId VARCHAR(128) NOT NULL,' +
+                'identifier VARCHAR(128) NOT NULL,' +
+                'hashedPassword VARCHAR(1024) NOT NULL,' +
+				'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
+                'FOREIGN KEY(userId) REFERENCES users(id),' +
+                'UNIQUE (name, userId),' +
+				'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE appPasswords', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200205172146-remove-authcodedb.js

@@ -0,0 +1,22 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DROP TABLE authcodes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    var cmd = `CREATE TABLE IF NOT EXISTS authcodes(
+        authCode VARCHAR(128) NOT NULL UNIQUE,
+        userId VARCHAR(128) NOT NULL,
+        clientId VARCHAR(128) NOT NULL,
+        expiresAt BIGINT NOT NULL,
+        PRIMARY KEY(authCode)) CHARACTER SET utf8 COLLATE utf8_bin`;
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200206154808-remove-clients.js

@@ -0,0 +1,24 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DROP TABLE clients', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    var cmd = `CREATE TABLE IF NOT EXISTS clients(
+      id VARCHAR(128) NOT NULL UNIQUE,
+      appId VARCHAR(128) NOT NULL,
+      type VARCHAR(16) NOT NULL,
+      clientSecret VARCHAR(512) NOT NULL,
+      redirectURI VARCHAR(512) NOT NULL,
+      scope VARCHAR(512) NOT NULL,
+      PRIMARY KEY(id)) CHARACTER SET utf8 COLLATE utf8_bin`;
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200214051201-domains-drop-locked.js

@@ -0,0 +1,17 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE domains DROP COLUMN locked', function (error) {
+        if (error) return callback(error);
+
+        callback();
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE domains ADD COLUMN locked BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20200221211850-users-add-role.js

@@ -0,0 +1,40 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        db.runSql.bind(db, 'ALTER TABLE users ADD COLUMN role VARCHAR(32)'),
+        function migrateAdminFlag(done) {
+            db.all('SELECT * FROM users ORDER BY createdAt', function (error, results) {
+                if (error) return done(error);
+                let ownerFound = false;
+
+                async.eachSeries(results, function (user, next) {
+                    let role;
+                    if (!ownerFound && user.admin) {
+                        role = 'owner';
+                        ownerFound = true;
+                        console.log(`Designating ${user.username} ${user.email} ${user.id} as the owner of this cloudron`);
+                    } else {
+                        role = user.admin ? 'admin' : 'user';
+                    }
+                    db.runSql('UPDATE users SET role=? WHERE id=?', [ role, user.id ], next);
+                }, done);
+            });
+        },
+        db.runSql.bind(db, 'ALTER TABLE users DROP COLUMN admin'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY role VARCHAR(32) NOT NULL'),
+        db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN role', function (error) {
+        if (error) console.error(error);
+
+        callback(error);
+    });
+};
+

migrations/20200330233442-users-add-resetTokenCreationTime.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN resetTokenCreationTime', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200331164304-apps-alter-mailboxDomain.js

@@ -0,0 +1,28 @@
+'use strict';
+
+let async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY mailboxDomain VARCHAR(128)', [], function (error) { // make it nullable
+        if (error) console.error(error);
+
+        // clear mailboxName/Domain for apps that do not use mail addons
+        db.all('SELECT * FROM apps', function (error, apps) {
+            if (error) return callback(error);
+
+            async.eachSeries(apps, function (app, iteratorDone) {
+                var manifest = JSON.parse(app.manifestJson);
+                if (manifest.addons['sendmail'] || manifest.addons['recvmail']) return iteratorDone();
+
+                db.runSql('UPDATE apps SET mailboxName=?, mailboxDomain=? WHERE id=?', [ null, null, app.id ], iteratorDone);
+            }, callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY manifestJson VARCHAR(128) NOT NULL', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -26,8 +26,11 @@ CREATE TABLE IF NOT EXISTS users(
     fallbackEmail VARCHAR(512) DEFAULT "",
     twoFactorAuthenticationSecret VARCHAR(128) DEFAULT "",
     twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
-    admin BOOLEAN DEFAULT false,
     source VARCHAR(128) DEFAULT "",
+    role VARCHAR(32),
+    resetToken VARCHAR(128) DEFAULT "",
+    resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    active BOOLEAN DEFAULT 1,
 
     PRIMARY KEY(id));
 
@@ -52,15 +55,6 @@ CREATE TABLE IF NOT EXISTS tokens(
     expires BIGINT NOT NULL, // FIXME: make this a timestamp
     PRIMARY KEY(accessToken));
 
-CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(128) NOT NULL UNIQUE, // prefixed with cid- to identify token easily in auth routes
-    appId VARCHAR(128) NOT NULL,     // name of the client (for external apps) or id of app (for built-in apps)
-    type VARCHAR(16) NOT NULL,
-    clientSecret VARCHAR(512) NOT NULL,
-    redirectURI VARCHAR(512) NOT NULL,
-    scope VARCHAR(512) NOT NULL,
-    PRIMARY KEY(id));
-
 CREATE TABLE IF NOT EXISTS apps(
     id VARCHAR(128) NOT NULL UNIQUE,
     appStoreId VARCHAR(128) NOT NULL, // empty for custom apps
@@ -78,14 +72,15 @@ CREATE TABLE IF NOT EXISTS apps(
     updateTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,   // when the last app update was done
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, // when this db record was updated (useful for UI caching)
     memoryLimit BIGINT DEFAULT 0,
+    cpuShares INTEGER DEFAULT 512,
     xFrameOptions VARCHAR(512),
     sso BOOLEAN DEFAULT 1, // whether user chose to enable SSO
     debugModeJson TEXT, // options for development mode
     reverseProxyConfigJson TEXT, // { robotsTxt, csp }
     enableBackup BOOLEAN DEFAULT 1, // misnomer: controls automatic daily backups
     enableAutomaticUpdate BOOLEAN DEFAULT 1,
-    mailboxName VARCHAR(128), // mailbox of this app. default allocated as '.app'
-    mailboxDomain VARCHAR(128) NOT NULL, // mailbox domain of this apps
+    mailboxName VARCHAR(128), // mailbox of this app
+    mailboxDomain VARCHAR(128), // mailbox domain of this apps
     label VARCHAR(128),     // display name
     tagsJson VARCHAR(2048), // array of tags
     dataDir VARCHAR(256) UNIQUE,
@@ -104,13 +99,6 @@ CREATE TABLE IF NOT EXISTS appPortBindings(
     FOREIGN KEY(appId) REFERENCES apps(id),
     PRIMARY KEY(hostPort));
 
-CREATE TABLE IF NOT EXISTS authcodes(
-    authCode VARCHAR(128) NOT NULL UNIQUE,
-    userId VARCHAR(128) NOT NULL,
-    clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL, // ## FIXME: make this a timestamp
-    PRIMARY KEY(authCode));
-
 CREATE TABLE IF NOT EXISTS settings(
     name VARCHAR(128) NOT NULL UNIQUE,
     value TEXT,
@@ -157,7 +145,6 @@ CREATE TABLE IF NOT EXISTS domains(
     provider VARCHAR(16) NOT NULL,
     configJson TEXT, /* JSON containing the dns backend provider config */
     tlsConfigJson TEXT, /* JSON containing the tls provider config */
-    locked BOOLEAN,
 
     PRIMARY KEY (domain))
 
@@ -231,4 +218,16 @@ CREATE TABLE IF NOT EXISTS notifications(
     PRIMARY KEY (id)
 );
 
+CREATE TABLE IF NOT EXISTS appPasswords(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    name VARCHAR(128) NOT NULL,
+    userId VARCHAR(128) NOT NULL,
+    identifier VARCHAR(128) NOT NULL, // resourceId: app id or mail or webadmin
+    hashedPassword VARCHAR(1024) NOT NULL,
+    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY(userId) REFERENCES users(id),
+    UNIQUE (name, userId),
+    PRIMARY KEY (id)
+);
+
 CHARACTER SET utf8 COLLATE utf8_bin;

package.json

@@ -17,71 +17,59 @@
     "@google-cloud/dns": "^1.1.0",
     "@google-cloud/storage": "^2.5.0",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^2.6.2",
-    "aws-sdk": "^2.476.0",
+    "async": "^2.6.3",
+    "aws-sdk": "^2.610.0",
     "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^4.0.0",
+    "cloudron-manifestformat": "^5.1.1",
     "connect": "^3.7.0",
-    "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "^1.2.1",
+    "connect-lastmile": "^1.2.2",
     "connect-timeout": "^1.9.0",
-    "cookie-parser": "^1.4.4",
-    "cookie-session": "^1.3.3",
-    "cron": "^1.7.1",
-    "csurf": "^1.10.0",
+    "cookie-session": "^1.4.0",
+    "cron": "^1.8.2",
     "db-migrate": "^0.11.6",
     "db-migrate-mysql": "^1.1.10",
     "debug": "^4.1.1",
     "dockerode": "^2.5.8",
     "ejs": "^2.6.1",
-    "ejs-cli": "^2.0.1",
+    "ejs-cli": "^2.1.1",
     "express": "^4.17.1",
-    "express-session": "^1.16.2",
     "js-yaml": "^3.13.1",
     "json": "^9.0.6",
     "ldapjs": "^1.0.2",
-    "lodash": "^4.17.11",
+    "lodash": "^4.17.15",
     "lodash.chunk": "^4.2.0",
     "mime": "^2.4.4",
-    "moment-timezone": "^0.5.25",
+    "moment-timezone": "^0.5.27",
     "morgan": "^1.9.1",
     "multiparty": "^4.2.1",
-    "mysql": "^2.17.1",
-    "nodemailer": "^6.2.1",
+    "mysql": "^2.18.1",
+    "nodemailer": "^6.4.2",
     "nodemailer-smtp-transport": "^2.7.4",
-    "oauth2orize": "^1.11.0",
     "once": "^1.4.0",
     "parse-links": "^0.1.0",
-    "passport": "^0.4.0",
-    "passport-http": "^0.3.0",
-    "passport-http-bearer": "^1.0.1",
-    "passport-local": "^1.0.0",
-    "passport-oauth2-client-password": "^0.1.2",
     "pretty-bytes": "^5.3.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
-    "qrcode": "^1.3.3",
-    "readdirp": "^3.0.2",
+    "qrcode": "^1.4.4",
+    "readdirp": "^3.3.0",
     "request": "^2.88.0",
     "rimraf": "^2.6.3",
     "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^0.7.1",
+    "safetydance": "^1.0.0",
     "semver": "^6.1.1",
-    "session-file-store": "^1.3.1",
-    "showdown": "^1.9.0",
+    "showdown": "^1.9.1",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^5.0.9",
+    "superagent": "^5.2.1",
     "supererror": "^0.7.2",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.1.0",
     "tldjs": "^2.3.1",
-    "underscore": "^1.9.1",
-    "uuid": "^3.3.2",
-    "valid-url": "^1.0.9",
+    "underscore": "^1.9.2",
+    "uuid": "^3.4.0",
     "validator": "^11.0.0",
-    "ws": "^7.0.0",
-    "xml2js": "^0.4.19"
+    "ws": "^7.2.1",
+    "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",

runTests

@@ -22,7 +22,7 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
 mkdir -p platformdata/addons/mail platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
 
 # put cert

scripts/cloudron-setup

@@ -92,13 +92,14 @@ fi
 echo "Running cloudron-setup with args : $@" > "${LOG_FILE}"
 
 # validate arguments in the absence of data
-readonly AVAILABLE_PROVIDERS="azure, caas, cloudscale, contabo, digitalocean, ec2, exoscale, galaxygate, gce, hetzner, interox, lightsail, linode, netcup, ovh, rosehosting, scaleway, skysilk, time4vps, upcloud, vultr or generic"
+readonly AVAILABLE_PROVIDERS="azure, caas, cloudscale, contabo, digitalocean, ec2, exoscale, gce, hetzner, interox, lightsail, linode, netcup, ovh, rosehosting, scaleway, skysilk, time4vps, upcloud, vultr or generic"
 if [[ -z "${provider}" ]]; then
     echo "--provider is required ($AVAILABLE_PROVIDERS)"
     exit 1
 elif [[  \
             "${provider}" != "ami" && \
             "${provider}" != "azure" && \
+            "${provider}" != "azure-image" && \
             "${provider}" != "caas" && \
             "${provider}" != "cloudscale" && \
             "${provider}" != "contabo" && \
@@ -106,13 +107,13 @@ elif [[  \
             "${provider}" != "digitalocean-mp" && \
             "${provider}" != "ec2" && \
             "${provider}" != "exoscale" && \
-            "${provider}" != "galaxygate" && \
             "${provider}" != "gce" && \
             "${provider}" != "hetzner" && \
             "${provider}" != "interox" && \
             "${provider}" != "interox-image" && \
             "${provider}" != "lightsail" && \
             "${provider}" != "linode" && \
+            "${provider}" != "linode-oneclick" && \
             "${provider}" != "linode-stackscript" && \
             "${provider}" != "netcup" && \
             "${provider}" != "netcup-image" && \
@@ -202,21 +203,11 @@ if [[ "${initBaseImage}" == "true" ]]; then
     echo ""
 fi
 
-# NOTE: this install script only supports 3.x and above
+# NOTE: this install script only supports 4.2 and above
 echo "=> Installing version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
-# this file is used >= 4.2
 echo "${provider}" > /etc/cloudron/PROVIDER
 
-# this file is unused <= 4.2 and exists to make legacy installations work. the start script will remove this file anyway
-cat > "/etc/cloudron/cloudron.conf" <<CONF_END
-{
-    "apiServerOrigin": "${apiServerOrigin}",
-    "webServerOrigin": "${webServerOrigin}",
-    "provider": "${provider}"
-}
-CONF_END
-
 [[ -n "${license}" ]] && echo -n "$license" > /etc/cloudron/LICENSE
 
 if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
@@ -224,7 +215,6 @@ if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
     exit 1
 fi
 
-# only needed for >= 4.2
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('api_server_origin', '${apiServerOrigin}');" 2>/dev/null
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('web_server_origin', '${webServerOrigin}');" 2>/dev/null
 
@@ -237,7 +227,10 @@ while true; do
     sleep 10
 done
 
-echo -e "\n\n${GREEN}Visit https://<IP> and accept the self-signed certificate to finish setup.${DONE}\n"
+if ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+    ip='<IP>'
+fi
+echo -e "\n\n${GREEN}Visit https://${ip} and accept the self-signed certificate to finish setup.${DONE}\n"
 
 if [[ "${rebootServer}" == "true" ]]; then
     systemctl stop box mysql # sometimes mysql ends up having corrupt privilege tables

scripts/cloudron-support

@@ -13,7 +13,7 @@ HELP_MESSAGE="
 This script collects diagnostic information to help debug server related issues
 
  Options:
-   --admin-login   Login as administrator
+   --owner-login   Login as owner
    --enable-ssh    Enable SSH access for the Cloudron support team
    --help          Show this message
 "
@@ -26,7 +26,7 @@ fi
 
 enableSSH="false"
 
-args=$(getopt -o "" -l "help,enable-ssh,admin-login" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,enable-ssh,admin-login,owner-login" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
@@ -34,10 +34,15 @@ while true; do
     --help) echo -e "${HELP_MESSAGE}"; exit 0;;
     --enable-ssh) enableSSH="true"; shift;;
     --admin-login)
-        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE admin=1 LIMIT 1" 2>/dev/null)
+        # fall through
+        ;&
+    --owner-login)
+        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
-        printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > /tmp/cloudron_ghost.json
-        echo "Login as ${admin_username} / ${admin_password} . Remove /tmp/cloudron_ghost.json when done."
+        ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
+        printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > "${ghost_file}"
+        chown yellowtent:yellowtent "${ghost_file}" && chmod o-r,g-r "${ghost_file}"
+        echo "Login as ${admin_username} / ${admin_password} . Remove ${ghost_file} when done."
         exit 0
         ;;
     --) break;;
@@ -80,13 +85,13 @@ echo -e $LINE"Filesystem stats"$LINE >> $OUT
 df -h &>> $OUT
 
 echo -e $LINE"Appsdata stats"$LINE >> $OUT
-du -hcsL /home/yellowtent/appsdata/* &>> $OUT
+du -hcsL /home/yellowtent/appsdata/* &>> $OUT || true
 
 echo -e $LINE"Boxdata stats"$LINE >> $OUT
 du -hcsL /home/yellowtent/boxdata/* &>> $OUT
 
 echo -e $LINE"Backup stats (possibly misleading)"$LINE >> $OUT
-du -hcsL /var/backups/* &>> $OUT
+du -hcsL /var/backups/* &>> $OUT || true
 
 echo -e $LINE"System daemon status"$LINE >> $OUT
 systemctl status --lines=100 cloudron.target box mysql unbound cloudron-syslog nginx collectd docker &>> $OUT

scripts/createReleaseTarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v10.15.1" ]]; then
-    echo "This script requires node 10.15.1"
+if [[ "$(node --version)" != "v10.18.1" ]]; then
+    echo "This script requires node 10.18.1"
     exit 1
 fi
 

scripts/installer.sh

@@ -56,13 +56,22 @@ if [[ $(docker version --format {{.Client.Version}}) != "18.09.2" ]]; then
     rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
 fi
 
+readonly nginx_version=$(nginx -v)
+if [[ "${nginx_version}" != *"1.14."* && "${ubuntu_version}" == "16.04" ]]; then
+    echo "==> installer: installing nginx for xenial for TLSv3 support"
+    curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.14.0-1~xenial_amd64.deb -o /tmp/nginx.deb
+    # apt install with install deps (as opposed to dpkg -i)
+    apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
+    rm /tmp/nginx.deb
+fi
+
 echo "==> installer: updating node"
-if [[ "$(node --version)" != "v10.15.1" ]]; then
-    mkdir -p /usr/local/node-10.15.1
-    $curl -sL https://nodejs.org/dist/v10.15.1/node-v10.15.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.15.1
-    ln -sf /usr/local/node-10.15.1/bin/node /usr/bin/node
-    ln -sf /usr/local/node-10.15.1/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-8.11.2 /usr/local/node-8.9.3
+if [[ "$(node --version)" != "v10.18.1" ]]; then
+    mkdir -p /usr/local/node-10.18.1
+    $curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
+    ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
+    ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
+    rm -rf /usr/local/node-10.15.1
 fi
 
 # this is here (and not in updater.js) because rebuild requires the above node

setup/start.sh

@@ -47,10 +47,12 @@ mkdir -p "${PLATFORM_DATA_DIR}/backup"
 mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
          "${PLATFORM_DATA_DIR}/logs/updater" \
          "${PLATFORM_DATA_DIR}/logs/tasks" \
-         "${PLATFORM_DATA_DIR}/logs/crash"
+         "${PLATFORM_DATA_DIR}/logs/crash" \
+         "${PLATFORM_DATA_DIR}/logs/collectd"
 mkdir -p "${PLATFORM_DATA_DIR}/update"
 
 mkdir -p "${BOX_DATA_DIR}/appicons"
+mkdir -p "${BOX_DATA_DIR}/profileicons"
 mkdir -p "${BOX_DATA_DIR}/certs"
 mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
 mkdir -p "${BOX_DATA_DIR}/mail/dkim"
@@ -113,7 +115,7 @@ rm -f /etc/sudoers.d/${USER}
 cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER}
 
 echo "==> Configuring collectd"
-rm -rf /etc/collectd
+rm -rf /etc/collectd /var/log/collectd.log
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
 systemctl restart collectd

setup/start/cloudron-firewall.sh

@@ -12,6 +12,11 @@ iptables -t filter -I CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
 # ssh is allowed alternately on port 202
 iptables -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,25,80,202,443,587,993,4190 -j ACCEPT
 
+# turn and stun service
+iptables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
+iptables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
+iptables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
+
 iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
 iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
 iptables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT

setup/start/cloudron-motd

@@ -3,10 +3,17 @@
 printf "**********************************************************************\n\n"
 
 if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
+    if [[ -f /tmp/.cloudron-motd-cache ]]; then
+        ip=$(cat /tmp/.cloudron-motd-cache)
+    elif ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+        ip='<IP>'
+    fi
+    echo "${ip}" > /tmp/.cloudron-motd-cache
+
     printf "\t\t\tWELCOME TO CLOUDRON\n"
     printf "\t\t\t-------------------\n"
 
-    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit https://<IP> on your browser and accept the self-signed certificate to finish setup."
+    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit https://${ip} on your browser and accept the self-signed certificate to finish setup."
     printf "Cloudron overview - https://cloudron.io/documentation/ \n"
     printf "Cloudron setup - https://cloudron.io/documentation/installation/#setup \n"
 else

setup/start/collectd/collectd.conf

@@ -57,7 +57,7 @@ LoadPlugin logfile
 
 <Plugin logfile>
    LogLevel "info"
-   File "/var/log/collectd.log"
+   File "/home/yellowtent/platformdata/logs/collectd/collectd.log"
    Timestamp true
    PrintSeverity false
 </Plugin>

setup/start/collectd/du.py

@@ -6,7 +6,8 @@ PATHS = [] # { name, dir, exclude }
 INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this
 
 def du(pathinfo):
-    cmd = 'timeout 1800 du -Dsb "{}"'.format(pathinfo['dir'])
+    # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
+    cmd = 'timeout 1800 du -DsB1 "{}"'.format(pathinfo['dir'])
     if pathinfo['exclude'] != '':
         cmd += ' --exclude "{}"'.format(pathinfo['exclude'])
 
@@ -26,6 +27,7 @@ def parseSize(size):
 
 def dockerSize():
     # use --format '{{json .}}' to dump the string. '{{if eq .Type "Images"}}{{.Size}}{{end}}' still creates newlines
+    # https://godoc.org/github.com/docker/go-units#HumanSize is used. so it's 1000 (KB) and not 1024 (KiB)
     cmd = 'timeout 1800 docker system df --format "{{.Size}}" | head -n1'
     try:
         size = subprocess.check_output(cmd, shell=True).strip().decode('utf-8')

setup/start/custom.yml

@@ -1,40 +0,0 @@
-# add customizations here
-# after making changes run "sudo systemctl restart box"
-
-# appstore:
-#   blacklist:
-#     - io.wekan.cloudronapp
-#     - io.cloudron.openvpn
-#   whitelist:
-#     org.wordpress.cloudronapp: {}
-#     chat.rocket.cloudronapp: {}
-#     com.nextcloud.cloudronapp: {}
-#
-# backups:
-#   configurable: true
-#
-# domains:
-#   dynamicDns: true
-#   changeDashboardDomain: true
-#
-# subscription:
-#   configurable: true
-#
-# support:
-#   email: support@cloudron.io
-#   remoteSupport: true
-#
-#   ticketFormBody: |
-#     Use this form to open support tickets. You can also write directly to [support@cloudron.io](mailto:support@cloudron.io).
-#       * [Knowledge Base & App Docs](https://cloudron.io/documentation/apps/?support_view)
-#       * [Custom App Packaging & API](https://cloudron.io/developer/packaging/?support_view)
-#       * [Forum](https://forum.cloudron.io/)
-#
-#   submitTickets: true
-#
-# alerts:
-#   email: support@cloudron.io
-#   notifyCloudronAdmins: false
-#
-# footer:
-#   body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'

setup/start/logrotate/platform

@@ -9,6 +9,7 @@
 /home/yellowtent/platformdata/logs/sftp/*.log
 /home/yellowtent/platformdata/logs/redis-*/*.log
 /home/yellowtent/platformdata/logs/crash/*.log
+/home/yellowtent/platformdata/logs/collectd/*.log
 /home/yellowtent/platformdata/logs/updater/*.log {
     # only keep one rotated file, we currently do not send that over the api
     rotate 1

src/accesscontrol.js

@@ -1,140 +1,29 @@
 'use strict';
 
 exports = module.exports = {
-    SCOPE_APPS_READ: 'apps:read',
-    SCOPE_APPS_MANAGE: 'apps:manage',
-    SCOPE_APPSTORE: 'appstore',
-    SCOPE_CLIENTS: 'clients',
-    SCOPE_CLOUDRON: 'cloudron',
-    SCOPE_DOMAINS_READ: 'domains:read',
-    SCOPE_DOMAINS_MANAGE: 'domains:manage',
-    SCOPE_MAIL: 'mail',
-    SCOPE_PROFILE: 'profile',
-    SCOPE_SETTINGS: 'settings',
-    SCOPE_SUBSCRIPTION: 'subscription',
-    SCOPE_USERS_READ: 'users:read',
-    SCOPE_USERS_MANAGE: 'users:manage',
-    VALID_SCOPES: [ 'apps', 'appstore', 'clients', 'cloudron', 'domains', 'mail', 'profile', 'settings', 'subscription', 'users' ], // keep this sorted
-
-    SCOPE_ANY: '*',
-
-    validateScopeString: validateScopeString,
-    hasScopes: hasScopes,
-    canonicalScopeString: canonicalScopeString,
-    intersectScopes: intersectScopes,
-    validateToken: validateToken,
-    scopesForUser: scopesForUser
+    verifyToken: verifyToken
 };
 
 var assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    debug = require('debug')('box:accesscontrol'),
     tokendb = require('./tokendb.js'),
-    users = require('./users.js'),
-    _ = require('underscore');
+    users = require('./users.js');
 
-// returns scopes that does not have wildcards and is sorted
-function canonicalScopeString(scope) {
-    if (scope === exports.SCOPE_ANY) return exports.VALID_SCOPES.join(',');
-
-    return scope.split(',').sort().join(',');
-}
-
-function intersectScopes(allowedScopes, wantedScopes) {
-    assert(Array.isArray(allowedScopes), 'Expecting sorted array');
-    assert(Array.isArray(wantedScopes), 'Expecting sorted array');
-
-    if (_.isEqual(allowedScopes, wantedScopes)) return allowedScopes; // quick path
-
-    let wantedScopesMap = new Map();
-    let results = [];
-
-    // make a map of scope -> [ subscopes ]
-    for (let w of wantedScopes) {
-        let parts = w.split(':');
-        let subscopes = wantedScopesMap.get(parts[0]) || new Set();
-        subscopes.add(parts[1] || '*');
-        wantedScopesMap.set(parts[0], subscopes);
-    }
-
-    for (let a of allowedScopes) {
-        let parts = a.split(':');
-        let as = parts[1] || '*';
-
-        let subscopes = wantedScopesMap.get(parts[0]);
-        if (!subscopes) continue;
-
-        if (subscopes.has('*') || subscopes.has(as)) {
-            results.push(a);
-        } else if (as === '*') {
-            results = results.concat(Array.from(subscopes).map(function (ss) { return `${a}:${ss}`; }));
-        }
-    }
-
-    return results;
-}
-
-function validateScopeString(scope) {
-    assert.strictEqual(typeof scope, 'string');
-
-    if (scope === '') return new BoxError(BoxError.BAD_FIELD, 'Empty scope not allowed', { field: 'scope' });
-
-    // NOTE: this function intentionally does not allow '*'. This is only allowed in the db to allow
-    // us not write a migration script every time we add a new scope
-    var allValid = scope.split(',').every(function (s) { return exports.VALID_SCOPES.indexOf(s.split(':')[0]) !== -1; });
-    if (!allValid) return new BoxError(BoxError.BAD_FIELD, 'Invalid scope. Available scopes are ' + exports.VALID_SCOPES.join(', '), { field: 'scope' });
-    return null;
-}
-
-// tests if all requiredScopes are attached to the request
-function hasScopes(authorizedScopes, requiredScopes) {
-    assert(Array.isArray(authorizedScopes), 'Expecting array');
-    assert(Array.isArray(requiredScopes), 'Expecting array');
-
-    if (authorizedScopes.indexOf(exports.SCOPE_ANY) !== -1) return null;
-
-    for (var i = 0; i < requiredScopes.length; ++i) {
-        const scopeParts = requiredScopes[i].split(':');
-
-        // this allows apps:write if the token has a higher apps scope
-        if (authorizedScopes.indexOf(requiredScopes[i]) === -1 && authorizedScopes.indexOf(scopeParts[0]) === -1) {
-            debug('scope: missing scope "%s".', requiredScopes[i]);
-            return new BoxError(BoxError.NOT_FOUND, 'Missing required scope "' + requiredScopes[i] + '"');
-        }
-    }
-
-    return null;
-}
-
-function scopesForUser(user, callback) {
-    assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (user.admin) return callback(null, exports.VALID_SCOPES);
-
-    callback(null, [ 'profile', 'apps:read' ]);
-}
-
-function validateToken(accessToken, callback) {
+function verifyToken(accessToken, callback) {
     assert.strictEqual(typeof accessToken, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     tokendb.getByAccessToken(accessToken, function (error, token) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
-        if (error) return callback(error); // this triggers 'internal error' in passport
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+        if (error) return callback(error);
 
         users.get(token.identifier, function (error, user) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
+            if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
             if (error) return callback(error);
 
-            if (!user.active) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
+            if (!user.active) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
 
-            scopesForUser(user, function (error, userScopes) {
-                if (error) return callback(error);
-
-                const authorizedScopes = intersectScopes(userScopes, token.scope.split(','));
-                callback(null, user, { authorizedScopes }); // ends up in req.authInfo
-            });
+            callback(null, user);
         });
     });
 }

src/addons.js

@@ -20,27 +20,22 @@ exports = module.exports = {
     getMountsSync: getMountsSync,
     getContainerNamesSync: getContainerNamesSync,
 
-    // exported for testing
-    _setupOauth: setupOauth,
-    _teardownOauth: teardownOauth,
+    getServiceDetails: getServiceDetails,
 
     SERVICE_STATUS_STARTING: 'starting',  // container up, waiting for healthcheck
     SERVICE_STATUS_ACTIVE: 'active',
     SERVICE_STATUS_STOPPED: 'stopped'
 };
 
-var accesscontrol = require('./accesscontrol.js'),
-    appdb = require('./appdb.js'),
+var appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
-    clients = require('./clients.js'),
     constants = require('./constants.js'),
     crypto = require('crypto'),
     debug = require('debug')('box:addons'),
     docker = require('./docker.js'),
-    dockerConnection = docker.connection,
     fs = require('fs'),
     graphs = require('./graphs.js'),
     hat = require('./hat.js'),
@@ -68,6 +63,13 @@ const RMADDONDIR_CMD = path.join(__dirname, 'scripts/rmaddondir.sh');
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
+    turn: {
+        setup: setupTurn,
+        teardown: teardownTurn,
+        backup: NOOP,
+        restore: NOOP,
+        clear: NOOP
+    },
     email: {
         setup: setupEmail,
         teardown: teardownEmail,
@@ -103,13 +105,6 @@ var KNOWN_ADDONS = {
         restore: restoreMySql,
         clear: clearMySql,
     },
-    oauth: {
-        setup: setupOauth,
-        teardown: teardownOauth,
-        backup: NOOP,
-        restore: setupOauth,
-        clear: NOOP,
-    },
     postgresql: {
         setup: setupPostgreSql,
         teardown: teardownPostgreSql,
@@ -155,6 +150,11 @@ var KNOWN_ADDONS = {
 };
 
 const KNOWN_SERVICES = {
+    turn: {
+        status: statusTurn,
+        restart: restartContainer.bind(null, 'turn'),
+        defaultMemoryLimit: 256 * 1024 * 1024
+    },
     mail: {
         status: containerStatus.bind(null, 'mail', 'CLOUDRON_MAIL_TOKEN'),
         restart: mail.restartMail,
@@ -232,11 +232,26 @@ function dumpPath(addon, appId) {
     }
 }
 
-function restartContainer(serviceName, callback) {
+function rebuildService(serviceName, callback) {
     assert.strictEqual(typeof serviceName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    assert(KNOWN_SERVICES[serviceName], `Unknown service ${serviceName}`);
+    // this attempts to recreate the service docker container if they don't exist but platform infra version is unchanged
+    // passing an infra version of 'none' will not attempt to purge existing data, not sure if this is good or bad
+    if (serviceName === 'turn') return startTurn({ version: 'none' }, callback);
+    if (serviceName === 'mongodb') return startMongodb({ version: 'none' }, callback);
+    if (serviceName === 'postgresql') return startPostgresql({ version: 'none' }, callback);
+    if (serviceName === 'mysql') return startMysql({ version: 'none' }, callback);
+    if (serviceName === 'sftp') return sftp.startSftp({ version: 'none' }, callback);
+    if (serviceName === 'graphite') return graphs.startGraphite({ version: 'none' }, callback);
+
+    // nothing to rebuild for now
+    callback();
+}
+
+function restartContainer(serviceName, callback) {
+    assert.strictEqual(typeof serviceName, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     docker.stopContainer(serviceName, function (error) {
         if (error) return callback(error);
@@ -246,31 +261,12 @@ function restartContainer(serviceName, callback) {
                 callback(null); // callback early since rebuilding takes long
                 return rebuildService(serviceName, function (error) { if (error) console.error(`Unable to rebuild service ${serviceName}`, error); });
             }
-            if (error) return callback(error);
 
-            callback(null);
+            callback(error);
         });
     });
 }
 
-function rebuildService(serviceName, callback) {
-    assert.strictEqual(typeof serviceName, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    assert(KNOWN_SERVICES[serviceName], `Unknown service ${serviceName}`);
-
-    // this attempts to recreate the service docker container if they don't exist but platform infra version is unchanged
-    // passing an infra version of 'none' will not attempt to purge existing data, not sure if this is good or bad
-    if (serviceName === 'mongodb') return startMongodb({ version: 'none' }, callback);
-    if (serviceName === 'postgresql') return startPostgresql({ version: 'none' }, callback);
-    if (serviceName === 'mysql') return startMysql({ version: 'none' }, callback);
-    if (serviceName === 'sftp') return sftp.startSftp({ version: 'none' }, callback);
-    if (serviceName === 'graphite') return graphs.startGraphite({ version: 'none' }, callback);
-
-    // nothing to rebuild for now
-    callback();
-}
-
 function getServiceDetails(containerName, tokenEnvName, callback) {
     assert.strictEqual(typeof containerName, 'string');
     assert.strictEqual(typeof tokenEnvName, 'string');
@@ -280,15 +276,15 @@ function getServiceDetails(containerName, tokenEnvName, callback) {
         if (error) return callback(error);
 
         const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
-        if (!ip) return callback(new BoxError(BoxError.INACTIVE, `Error getting ${containerName} container ip`));
+        if (!ip) return callback(new BoxError(BoxError.INACTIVE, `Error getting IP of ${containerName} service`));
 
         // extract the cloudron token for auth
         const env = safe.query(result, 'Config.Env', null);
-        if (!env) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} env`));
+        if (!env) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error inspecting environment of ${containerName} service`));
         const tmp = env.find(function (e) { return e.indexOf(tokenEnvName) === 0; });
-        if (!tmp) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} cloudron token env var`));
+        if (!tmp) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`));
         const token = tmp.slice(tokenEnvName.length + 1); // +1 for the = sign
-        if (!token)  return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} cloudron token`));
+        if (!token)  return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`));
 
         callback(null, { ip: ip, token: token, state: result.State });
     });
@@ -339,6 +335,9 @@ function getService(serviceName, callback) {
     var tmp = {
         name: serviceName,
         status: null,
+        memoryUsed: 0,
+        memoryPercent: 0,
+        error: null,
         config: {
             // If a property is not set then we cannot change it through the api, see below
             // memory: 0,
@@ -483,8 +482,8 @@ function waitForService(containerName, tokenEnvName, callback) {
 
         async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
             request.get(`https://${result.ip}:3000/healthcheck?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
-                if (error) return retryCallback(new Error(`Error waiting for ${containerName}: ${error.message}`));
-                if (response.statusCode !== 200 || !response.body.status) return retryCallback(new Error(`Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Network error waiting for ${containerName}: ${error.message}`));
+                if (response.statusCode !== 200 || !response.body.status) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 retryCallback(null);
             });
@@ -502,7 +501,7 @@ function setupAddons(app, addons, callback) {
     debugApp(app, 'setupAddons: Setting up %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         debugApp(app, 'Setting up addon %s with options %j', addon, addons[addon]);
 
@@ -520,7 +519,7 @@ function teardownAddons(app, addons, callback) {
     debugApp(app, 'teardownAddons: Tearing down %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         debugApp(app, 'Tearing down addon %s with options %j', addon, addons[addon]);
 
@@ -540,7 +539,7 @@ function backupAddons(app, addons, callback) {
     debugApp(app, 'backupAddons: Backing up %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         KNOWN_ADDONS[addon].backup(app, addons[addon], iteratorCallback);
     }, callback);
@@ -558,7 +557,7 @@ function clearAddons(app, addons, callback) {
     debugApp(app, 'clearAddons: clearing %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         KNOWN_ADDONS[addon].clear(app, addons[addon], iteratorCallback);
     }, callback);
@@ -576,7 +575,7 @@ function restoreAddons(app, addons, callback) {
     debugApp(app, 'restoreAddons: restoring %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         KNOWN_ADDONS[addon].restore(app, addons[addon], iteratorCallback);
     }, callback);
@@ -587,7 +586,7 @@ function importAppDatabase(app, addon, callback) {
     assert.strictEqual(typeof addon, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!(addon in KNOWN_ADDONS)) return callback(new Error(`No such addon: ${addon}`));
+    if (!(addon in KNOWN_ADDONS)) return callback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
     async.series([
         KNOWN_ADDONS[addon].setup.bind(null, app, app.manifest.addons[addon]),
@@ -627,7 +626,6 @@ function updateServiceConfig(platformConfig, callback) {
 
     debug('updateServiceConfig: %j', platformConfig);
 
-    // TODO: this should possibly also rollback memory to default
     async.eachSeries([ 'mysql', 'postgresql', 'mail', 'mongodb', 'graphite' ], function iterator(serviceName, iteratorCallback) {
         const containerConfig = platformConfig[serviceName];
         let memory, memorySwap;
@@ -654,6 +652,7 @@ function startServices(existingInfra, callback) {
     if (existingInfra.version !== infra.version) {
         debug(`startServices: ${existingInfra.version} -> ${infra.version}. starting all services`);
         startFuncs.push(
+            startTurn.bind(null, existingInfra),
             startMysql.bind(null, existingInfra),
             startPostgresql.bind(null, existingInfra),
             startMongodb.bind(null, existingInfra),
@@ -662,6 +661,7 @@ function startServices(existingInfra, callback) {
     } else {
         assert.strictEqual(typeof existingInfra.images, 'object');
 
+        if (!existingInfra.images.turn || infra.images.turn.tag !== existingInfra.images.turn.tag) startFuncs.push(startTurn.bind(null, existingInfra));
         if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) startFuncs.push(startMysql.bind(null, existingInfra));
         if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) startFuncs.push(startPostgresql.bind(null, existingInfra));
         if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) startFuncs.push(startMongodb.bind(null, existingInfra));
@@ -681,7 +681,7 @@ function getEnvironment(app, callback) {
     appdb.getAddonConfigByAppId(app.id, function (error, result) {
         if (error) return callback(error);
 
-        if (app.manifest.addons['docker']) result.push({ name: 'DOCKER_HOST', value: `tcp://172.18.0.1:${constants.DOCKER_PROXY_PORT}` });
+        if (app.manifest.addons['docker']) result.push({ name: 'CLOUDRON_DOCKER_HOST', value: `tcp://172.18.0.1:${constants.DOCKER_PROXY_PORT}` });
 
         return callback(null, result.map(function (e) { return e.name + '=' + e.value; }));
     });
@@ -772,52 +772,37 @@ function teardownLocalStorage(app, options, callback) {
     ], callback);
 }
 
-function setupOauth(app, options, callback) {
+function setupTurn(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debugApp(app, 'setupOauth');
+    var turnSecret = safe.fs.readFileSync(paths.ADDON_TURN_SECRET_FILE, 'utf8');
+    if (!turnSecret) console.error('No turn secret set. Will leave emtpy, but this is a problem!');
 
-    if (!app.sso) return callback(null);
+    const env = [
+        { name: 'CLOUDRON_STUN_SERVER', value: settings.adminFqdn() },
+        { name: 'CLOUDRON_STUN_PORT', value: '3478' },
+        { name: 'CLOUDRON_STUN_TLS_PORT', value: '5349' },
+        { name: 'CLOUDRON_TURN_SERVER', value: settings.adminFqdn() },
+        { name: 'CLOUDRON_TURN_PORT', value: '3478' },
+        { name: 'CLOUDRON_TURN_TLS_PORT', value: '5349' },
+        { name: 'CLOUDRON_TURN_SECRET', value: turnSecret }
+    ];
 
-    var appId = app.id;
-    var redirectURI = 'https://' + app.fqdn;
-    var scope = accesscontrol.SCOPE_PROFILE;
+    debugApp(app, 'Setting up TURN');
 
-    clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds
-        if (error && error.reason !== BoxError.NOT_FOUND) return callback(error);
-
-        clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) {
-            if (error) return callback(error);
-
-            const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
-            var env = [
-                { name: `${envPrefix}OAUTH_CLIENT_ID`, value: result.id },
-                { name: `${envPrefix}OAUTH_CLIENT_SECRET`, value: result.clientSecret },
-                { name: `${envPrefix}OAUTH_ORIGIN`, value: settings.adminOrigin() }
-            ];
-
-            debugApp(app, 'Setting oauth addon config to %j', env);
-
-            appdb.setAddonConfig(appId, 'oauth', env, callback);
-        });
-    });
+    appdb.setAddonConfig(app.id, 'turn', env, callback);
 }
 
-function teardownOauth(app, options, callback) {
+function teardownTurn(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debugApp(app, 'teardownOauth');
+    debugApp(app, 'Tearing down TURN');
 
-    clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) {
-        if (error && error.reason !== BoxError.NOT_FOUND) debug(error);
-
-        appdb.unsetAddonConfig(app.id, 'oauth', callback);
-    });
+    appdb.unsetAddonConfig(app.id, 'turn', callback);
 }
 
 function setupEmail(app, options, callback) {
@@ -1052,8 +1037,8 @@ function setupMySql(app, options, callback) {
             if (error) return callback(error);
 
             request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up mysql: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mysql: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
@@ -1090,9 +1075,10 @@ function clearMySql(app, options, callback) {
     getServiceDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mysql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+
             callback();
         });
     });
@@ -1109,9 +1095,9 @@ function teardownMySql(app, options, callback) {
     getServiceDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.delete(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mysql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             appdb.unsetAddonConfig(app.id, 'mysql', callback);
         });
@@ -1130,14 +1116,15 @@ function pipeRequestToFile(url, filename, callback) {
         callback(error);
     });
 
-    writeStream.on('error', done);
+    writeStream.on('error', (error) => done(new BoxError(BoxError.FS_ERROR, `Error writing to ${filename}: ${error.message}`)));
+
     writeStream.on('open', function () {
         // note: do not attach to post callback handler because this will buffer the entire reponse!
         // see https://github.com/request/request/issues/2270
         const req = request.post(url, { rejectUnauthorized: false });
-        req.on('error', done); // network error, dns error, request errored in middle etc
+        req.on('error', (error) => done(new BoxError(BoxError.NETWORK_ERROR, `Request error writing to ${filename}: ${error.message}`))); // network error, dns error, request errored in middle etc
         req.on('response', function (response) {
-            if (response.statusCode !== 200) return done(new Error(`Unexpected response code: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`));
+            if (response.statusCode !== 200) return done(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code when piping ${url}: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`));
 
             response.pipe(writeStream).on('finish', done); // this is hit after data written to disk
         });
@@ -1176,11 +1163,11 @@ function restoreMySql(app, options, callback) {
         if (error) return callback(error);
 
         var input = fs.createReadStream(dumpPath('mysql', app.id));
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mysql: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from mysql addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1265,8 +1252,8 @@ function setupPostgreSql(app, options, callback) {
             if (error) return callback(error);
 
             request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up postgresql: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up postgresql: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
@@ -1298,9 +1285,9 @@ function clearPostgreSql(app, options, callback) {
     getServiceDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing postgresql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1317,9 +1304,9 @@ function teardownPostgreSql(app, options, callback) {
     getServiceDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error tearing down postgresql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error tearing down postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error tearing down postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             appdb.unsetAddonConfig(app.id, 'postgresql', callback);
         });
@@ -1358,11 +1345,11 @@ function restorePostgreSql(app, options, callback) {
         if (error) return callback(error);
 
         var input = fs.createReadStream(dumpPath('postgresql', app.id));
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring postgresql: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from postgresql addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1371,6 +1358,43 @@ function restorePostgreSql(app, options, callback) {
     });
 }
 
+function startTurn(existingInfra, callback) {
+    assert.strictEqual(typeof existingInfra, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    // get and ensure we have a turn secret
+    var turnSecret = safe.fs.readFileSync(paths.ADDON_TURN_SECRET_FILE, 'utf8');
+    if (!turnSecret) {
+        turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter
+        safe.fs.writeFileSync(paths.ADDON_TURN_SECRET_FILE, turnSecret, 'utf8');
+    }
+
+    const tag = infra.images.turn.tag;
+    const memoryLimit = 256;
+    const realm = settings.adminFqdn();
+
+    if (existingInfra.version === infra.version && existingInfra.images.turn && infra.images.turn.tag === existingInfra.images.turn.tag) return callback();
+
+    // this exports 3478/tcp, 5349/tls and 50000-51000/udp
+    const cmd = `docker run --restart=always -d --name="turn" \
+                --hostname turn \
+                --net host \
+                --log-driver syslog \
+                --log-opt syslog-address=udp://127.0.0.1:2514 \
+                --log-opt syslog-format=rfc5424 \
+                --log-opt tag=turn \
+                -m ${memoryLimit}m \
+                --memory-swap ${memoryLimit * 2}m \
+                --dns 172.18.0.1 \
+                --dns-search=. \
+                -e CLOUDRON_TURN_SECRET="${turnSecret}" \
+                -e CLOUDRON_REALM="${realm}" \
+                --label isCloudronManaged=true \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.exec('startTurn', cmd, callback);
+}
+
 function startMongodb(existingInfra, callback) {
     assert.strictEqual(typeof existingInfra, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -1441,8 +1465,8 @@ function setupMongoDb(app, options, callback) {
             if (error) return callback(error);
 
             request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up mongodb: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up mongodb. Status code: ${response.statusCode}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mongodb: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
@@ -1476,9 +1500,9 @@ function clearMongodb(app, options, callback) {
     getServiceDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/databases/${app.id}/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mongodb: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/databases/${app.id}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback();
         });
@@ -1495,9 +1519,9 @@ function teardownMongoDb(app, options, callback) {
     getServiceDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.delete(`https://${result.ip}:3000/databases/${app.id}?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error tearing down mongodb: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error tearing down mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/databases/${app.id}?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             appdb.unsetAddonConfig(app.id, 'mongodb', callback);
         });
@@ -1532,11 +1556,11 @@ function restoreMongoDb(app, options, callback) {
         if (error) return callback(error);
 
         const readStream = fs.createReadStream(dumpPath('mongodb', app.id));
-        readStream.on('error', callback);
+        readStream.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mongodb: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${app.id}/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from mongodb addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/databases/${app.id}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1654,9 +1678,9 @@ function clearRedis(app, options, callback) {
     getServiceDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing redis: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing redis. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing redis: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing redis. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1668,18 +1692,11 @@ function teardownRedis(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = dockerConnection.getContainer('redis-' + app.id);
-
-    var removeOptions = {
-        force: true, // kill container if it's running
-        v: true // removes volumes associated with the container
-    };
-
-    container.remove(removeOptions, function (error) {
-        if (error && error.statusCode !== 404) return callback(new Error('Error removing container:' + error));
+    docker.deleteContainer(`redis-${app.id}`, function (error) {
+        if (error) return callback(error);
 
         shell.sudo('removeVolume', [ RMADDONDIR_CMD, 'redis', app.id ], {}, function (error) {
-            if (error) return callback(new Error('Error removing redis data:' + error));
+            if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`));
 
             rimraf(path.join(paths.LOG_DIR, `redis-${app.id}`), function (error) {
                 if (error) debugApp(app, 'cannot cleanup logs: %s', error);
@@ -1712,6 +1729,8 @@ function restoreRedis(app, options, callback) {
 
     debugApp(app, 'Restoring redis');
 
+    callback = once(callback); // protect from multiple returns with streams
+
     getServiceDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) {
         if (error) return callback(error);
 
@@ -1722,11 +1741,11 @@ function restoreRedis(app, options, callback) {
         } else { // old location of dumps
             input = fs.createReadStream(path.join(paths.APPS_DATA_DIR, app.id, 'redis/dump.rdb'));
         }
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring redis: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from redis addon: ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1735,6 +1754,27 @@ function restoreRedis(app, options, callback) {
     });
 }
 
+function statusTurn(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.inspect('turn', function (error, container) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, { status: exports.SERVICE_STATUS_STOPPED });
+        if (error) return callback(error);
+
+        docker.memoryUsage(container.Id, function (error, result) {
+            if (error) return callback(error);
+
+            var tmp = {
+                status: container.State.Running ? exports.SERVICE_STATUS_ACTIVE : exports.SERVICE_STATUS_STOPPED,
+                memoryUsed: result.memory_stats.usage,
+                memoryPercent: parseInt(100 * result.memory_stats.usage / result.memory_stats.limit)
+            };
+
+            callback(null, tmp);
+        });
+    });
+}
+
 function statusDocker(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -1811,7 +1851,7 @@ function statusGraphite(callback) {
         if (error && error.reason === BoxError.NOT_FOUND) return callback(null, { status: exports.SERVICE_STATUS_STOPPED });
         if (error) return callback(error);
 
-        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { timeout: 3000 }, function (error, response) {
+        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { json: true, timeout: 3000 }, function (error, response) {
             if (error) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite: ${error.message}` });
             if (response.statusCode !== 200) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite. Status code: ${response.statusCode} message: ${response.body.message}` });
 

src/appdb.js

@@ -40,7 +40,7 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'subdomains.subdomain AS location', 'subdomains.domain',
-    'apps.accessRestrictionJson', 'apps.memoryLimit',
+    'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup',
     'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
@@ -242,6 +242,7 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
     const accessRestriction = data.accessRestriction || null;
     const accessRestrictionJson = JSON.stringify(accessRestriction);
     const memoryLimit = data.memoryLimit || 0;
+    const cpuShares = data.cpuShares || 512;
     const installationState = data.installationState;
     const runState = data.runState;
     const sso = 'sso' in data ? data.sso : null;
@@ -256,10 +257,10 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
     var queries = [];
 
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, '
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
             + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit,
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
             sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson ]
     });
 
@@ -348,12 +349,13 @@ function del(id, callback) {
         { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
         { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
     ];
 
     database.transaction(queries, function (error, results) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[3].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+        if (results[4].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
 
         callback(null);
     });

src/apphealthmonitor.js

@@ -186,9 +186,10 @@ function processApp(callback) {
         async.each(result, checkAppHealth, function (error) {
             if (error) console.error(error);
 
-            var alive = result
+            const alive = result
                 .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; })
-                .map(function (a) { return (a.location || 'naked_domain') + '|' + (a.manifest.id || 'customapp'); }).join(', ');
+                .map(a => a.fqdn)
+                .join(', ');
 
             debug('apps alive: [%s]', alive);
 

src/appstore.js

@@ -1,16 +1,22 @@
 'use strict';
 
 exports = module.exports = {
+    getFeatures: getFeatures,
+
     getApps: getApps,
     getApp: getApp,
     getAppVersion: getAppVersion,
 
+    trackBeginSetup: trackBeginSetup,
+    trackFinishedSetup: trackFinishedSetup,
+
     registerWithLoginCredentials: registerWithLoginCredentials,
     registerWithLicense: registerWithLicense,
 
     purchaseApp: purchaseApp,
     unpurchaseApp: unpurchaseApp,
 
+    getUserToken: getUserToken,
     getSubscription: getSubscription,
     isFreePlan: isFreePlan,
 
@@ -27,13 +33,13 @@ var apps = require('./apps.js'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    custom = require('./custom.js'),
     debug = require('debug')('box:appstore'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     groups = require('./groups.js'),
     mail = require('./mail.js'),
     os = require('os'),
+    paths = require('./paths.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     settings = require('./settings.js'),
@@ -43,6 +49,38 @@ var apps = require('./apps.js'),
 
 const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
+// These are the default options and will be adjusted once a subscription state is obtained
+// Keep in sync with appstore/routes/cloudrons.js
+let gFeatures = {
+    userMaxCount: null,
+    externalLdap: true,
+    eventLog: true,
+    privateDockerRegistry: true,
+    branding: true,
+    userManager: true,
+    multiAdmin: true,
+    support: true
+};
+
+// attempt to load feature cache in case appstore would be down
+let tmp = safe.JSON.parse(safe.fs.readFileSync(paths.FEATURES_INFO_FILE, 'utf8'));
+if (tmp) gFeatures = tmp;
+
+function getFeatures() {
+    return gFeatures;
+}
+
+function isAppAllowed(appstoreId, listingConfig) {
+    assert.strictEqual(typeof listingConfig, 'object');
+    assert.strictEqual(typeof appstoreId, 'string');
+
+    if (listingConfig.blacklist && listingConfig.blacklist.includes(appstoreId)) return false;
+
+    if (listingConfig.whitelist) return listingConfig.whitelist.includes(appstoreId);
+
+    return true;
+}
+
 function getCloudronToken(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -96,6 +134,25 @@ function registerUser(email, password, callback) {
     });
 }
 
+function getUserToken(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (settings.isDemo()) return callback(new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode'));
+
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
+
+        const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
+
+        superagent.post(url).send({}).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${result.status}`));
+
+            callback(null, result.body.accessToken);
+        });
+    });
+}
+
 function getSubscription(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -110,7 +167,11 @@ function getSubscription(callback) {
             if (result.statusCode === 502) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`));
             if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`));
 
-            callback(null, result.body); // { email, subscription }
+            // update the features cache
+            gFeatures = result.body.features;
+            safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
+
+            callback(null, result.body);
         });
     });
 }
@@ -375,6 +436,35 @@ function registerCloudron(data, callback) {
     });
 }
 
+// This works without a Cloudron token as this Cloudron was not yet registered
+let gBeginSetupAlreadyTracked = false;
+function trackBeginSetup(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
+    // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
+    if (gBeginSetupAlreadyTracked) return;
+    gBeginSetupAlreadyTracked = true;
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
+
+    superagent.post(url).send({ provider }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
+// This works without a Cloudron token as this Cloudron was not yet registered
+function trackFinishedSetup(domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;
+
+    superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
 function registerWithLicense(license, domain, callback) {
     assert.strictEqual(typeof license, 'string');
     assert.strictEqual(typeof domain, 'string');
@@ -383,7 +473,10 @@ function registerWithLicense(license, domain, callback) {
     getCloudronToken(function (error, token) {
         if (token) return callback(new BoxError(BoxError.CONFLICT));
 
-        registerCloudron({ license, domain }, callback);
+        const provider = settings.provider();
+        const version = constants.VERSION;
+
+        registerCloudron({ license, domain, provider, version }, callback);
     });
 }
 
@@ -406,19 +499,20 @@ function registerWithLoginCredentials(options, callback) {
             login(options.email, options.password, options.totpToken || '', function (error, result) {
                 if (error) return callback(error);
 
-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken }, callback);
+                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, provider: settings.provider(), version: constants.VERSION, purpose: options.purpose || '' }, callback);
             });
         });
     });
 }
 
-function createTicket(info, callback) {
+function createTicket(info, auditSource, callback) {
     assert.strictEqual(typeof info, 'object');
     assert.strictEqual(typeof info.email, 'string');
     assert.strictEqual(typeof info.displayName, 'string');
     assert.strictEqual(typeof info.type, 'string');
     assert.strictEqual(typeof info.subject, 'string');
     assert.strictEqual(typeof info.description, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     function collectAppInfoIfNeeded(callback) {
@@ -435,7 +529,7 @@ function createTicket(info, callback) {
 
             let url = settings.apiServerOrigin() + '/api/v1/ticket';
 
-            info.supportEmail = custom.spec().support.email; // destination address for tickets
+            info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
 
             superagent.post(url).query({ accessToken: token }).send(info).timeout(10 * 1000).end(function (error, result) {
                 if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
@@ -443,7 +537,9 @@ function createTicket(info, callback) {
                 if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
                 if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-                callback(null);
+                eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+
+                callback(null, { message: `An email for sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
             });
         });
     });
@@ -466,7 +562,13 @@ function getApps(callback) {
                 if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
                 if (!result.body.apps) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-                callback(null, result.body.apps);
+                settings.getAppstoreListingConfig(function (error, listingConfig) {
+                    if (error) return callback(error);
+
+                    const filteredApps = result.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
+
+                    callback(null, filteredApps);
+                });
             });
         });
     });
@@ -477,20 +579,26 @@ function getAppVersion(appId, version, callback) {
     assert.strictEqual(typeof version, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
+    settings.getAppstoreListingConfig(function (error, listingConfig) {
         if (error) return callback(error);
 
-        let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
-        if (version !== 'latest') url += `/versions/${version}`;
+        if (!isAppAllowed(appId, listingConfig)) return callback(new BoxError(BoxError.FEATURE_DISABLED));
 
-        superagent.get(url).query({ accessToken: token }).timeout(10 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
+        getCloudronToken(function (error, token) {
+            if (error) return callback(error);
 
-            callback(null, result.body);
+            let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
+            if (version !== 'latest') url += `/versions/${version}`;
+
+            superagent.get(url).query({ accessToken: token }).timeout(10 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
+
+                callback(null, result.body);
+            });
         });
     });
 }

src/apptask.js

@@ -27,6 +27,7 @@ var addons = require('./addons.js'),
     auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
+    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:apptask'),
     df = require('@sindresorhus/df'),
@@ -51,8 +52,7 @@ var addons = require('./addons.js'),
     util = require('util'),
     _ = require('underscore');
 
-const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
-    CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh'),
+const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/app.ejs', { encoding: 'utf8' }),
     MV_VOLUME_CMD = path.join(__dirname, 'scripts/mvvolume.sh'),
     LOGROTATE_CONFIG_EJS = fs.readFileSync(__dirname + '/logrotate.ejs', { encoding: 'utf8' }),
     CONFIGURE_LOGROTATE_CMD = path.join(__dirname, 'scripts/configurelogrotate.sh');
@@ -64,11 +64,13 @@ function debugApp(app) {
 }
 
 function makeTaskError(error, app) {
-    let boxError = error instanceof BoxError ? error : new BoxError(BoxError.UNKNOWN_ERROR, error.message); // until we port everything to BoxError
+    assert.strictEqual(typeof error, 'object');
+    assert.strictEqual(typeof app, 'object');
+
     // track a few variables which helps 'repair' restart the task (see also scheduleTask in apps.js)
-    boxError.details.taskId = app.taskId;
-    boxError.details.installationState = app.installationState;
-    return boxError.toPlainObject();
+    error.details.taskId = app.taskId;
+    error.details.installationState = app.installationState;
+    return error.toPlainObject();
 }
 
 // updates the app object and the database
@@ -140,7 +142,15 @@ function createContainer(app, callback) {
     docker.createContainer(app, function (error, container) {
         if (error) return callback(error);
 
-        updateApp(app, { containerId: container.id }, callback);
+        updateApp(app, { containerId: container.id }, function (error) {
+            if (error) return callback(error);
+
+            // re-generate configs that rely on container id
+            async.series([
+                addLogrotateConfig.bind(null, app),
+                addCollectdProfile.bind(null, app)
+            ], callback);
+        });
     });
 }
 
@@ -151,11 +161,14 @@ function deleteContainers(app, options, callback) {
 
     debugApp(app, 'deleting app containers (app, scheduler)');
 
-    docker.deleteContainers(app.id, options, function (error) {
-        if (error) return callback(error);
-
-        updateApp(app, { containerId: null }, callback);
-    });
+    async.series([
+        // remove configs that rely on container id
+        removeCollectdProfile.bind(null, app),
+        removeLogrotateConfig.bind(null, app),
+        docker.stopContainers.bind(null, app.id),
+        docker.deleteContainers.bind(null, app.id, options),
+        updateApp.bind(null, app, { containerId: null })
+    ], callback);
 }
 
 function createAppDir(app, callback) {
@@ -216,29 +229,14 @@ function addCollectdProfile(app, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir: apps.getDataDir(app, app.dataDir) });
-    fs.writeFile(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), collectdConf, function (error) {
-        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
-
-        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', app.id ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
-
-            callback(null);
-        });
-    });
+    collectd.addProfile(app.id, collectdConf, callback);
 }
 
 function removeCollectdProfile(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), function (error) {
-        if (error && error.code !== 'ENOENT') debugApp(app, 'Error removing collectd profile', error);
-        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', app.id ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
-
-            callback(null);
-        });
-    });
+    collectd.removeProfile(app.id, callback);
 }
 
 function addLogrotateConfig(app, callback) {
@@ -433,12 +431,12 @@ function waitForDnsPropagation(app, callback) {
     sysinfo.getServerIp(function (error, ip) {
         if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Error getting public IP: ${error.message}`));
 
-        domains.waitForDnsRecord(app.location, app.domain, 'A', ip, { interval: 5000, times: 240 }, function (error) {
+        domains.waitForDnsRecord(app.location, app.domain, 'A', ip, { times: 240 }, function (error) {
             if (error) return callback(new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: app.location, domain: app.domain }));
 
             // now wait for alternateDomains, if any
             async.eachSeries(app.alternateDomains, function (domain, iteratorCallback) {
-                domains.waitForDnsRecord(domain.subdomain, domain.domain, 'A', ip, { interval: 5000, times: 240 }, function (error) {
+                domains.waitForDnsRecord(domain.subdomain, domain.domain, 'A', ip, { times: 240 }, function (error) {
                     if (error) return callback(new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: domain.subdomain, domain: domain.domain }));
 
                     iteratorCallback();
@@ -448,16 +446,18 @@ function waitForDnsPropagation(app, callback) {
     });
 }
 
-function moveDataDir(app, sourceDir, callback) {
+function moveDataDir(app, targetDir, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert(sourceDir === null || typeof sourceDir === 'string');
+    assert(targetDir === null || typeof targetDir === 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let resolvedSourceDir = apps.getDataDir(app, sourceDir);
-    let resolvedTargetDir = apps.getDataDir(app, app.dataDir);
+    let resolvedSourceDir = apps.getDataDir(app, app.dataDir);
+    let resolvedTargetDir = apps.getDataDir(app, targetDir);
 
     debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
 
+    if (resolvedSourceDir === resolvedTargetDir) return callback();
+
     shell.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}, function (error) {
         if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`));
 
@@ -492,17 +492,6 @@ function startApp(app, callback){
     docker.startContainer(app.id, callback);
 }
 
-// Ordering is based on the following rationale:
-//   - configure nginx, icon, oauth
-//   - register subdomain.
-//          at this point, the user can visit the site and the above nginx config can show some install screen.
-//          the icon can be displayed in this nginx page and oauth proxy means the page can be protected
-//   - download image
-//   - setup volumes
-//   - setup addons (requires the above volume)
-//   - setup the container (requires image, volumes, addons)
-//   - setup collectd (requires container id)
-// restore is also handled here since restore is just an install with some oldConfig to clean up
 function install(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
@@ -511,6 +500,7 @@ function install(app, args, progressCallback, callback) {
 
     const restoreConfig = args.restoreConfig; // has to be set when restoring
     const overwriteDns = args.overwriteDns;
+    const oldManifest = args.oldManifest;
 
     async.series([
         // this protects against the theoretical possibility of an app being marked for install/restore from
@@ -520,29 +510,29 @@ function install(app, args, progressCallback, callback) {
         // teardown for re-installs
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
         unconfigureReverseProxy.bind(null, app),
-        removeCollectdProfile.bind(null, app),
-        removeLogrotateConfig.bind(null, app),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
         function teardownAddons(next) {
             // when restoring, app does not require these addons anymore. remove carefully to preserve the db passwords
             let addonsToRemove;
-            if (restoreConfig && restoreConfig.oldManifest) { // oldManifest is null for clone
-                addonsToRemove = _.omit(restoreConfig.oldManifest.addons, Object.keys(app.manifest.addons));
+            if (oldManifest) {
+                addonsToRemove = _.omit(oldManifest.addons, Object.keys(app.manifest.addons));
             } else {
                 addonsToRemove = app.manifest.addons;
             }
 
             addons.teardownAddons(app, addonsToRemove, next);
         },
-        deleteAppDir.bind(null, app, { removeDirectory: false }), // do not remove any symlinked appdata dir
+
+        function deleteAppDirIfNeeded(done) {
+            if (restoreConfig && !restoreConfig.backupId) return done(); // in-place import should not delete data dir
+
+            deleteAppDir(app, { removeDirectory: false }, done); // do not remove any symlinked appdata dir
+        },
 
         function deleteImageIfChanged(done) {
-            if (!restoreConfig || !restoreConfig.oldManifest) return done();
+            if (!oldManifest || oldManifest.dockerImage === app.manifest.dockerImage) return done();
 
-            if (restoreConfig.oldManifest.dockerImage === app.manifest.dockerImage) return done();
-
-            docker.deleteImage(restoreConfig.oldManifest, done);
+            docker.deleteImage(oldManifest, done);
         },
 
         reserveHttpPort.bind(null, app),
@@ -565,14 +555,22 @@ function install(app, args, progressCallback, callback) {
                     progressCallback.bind(null, { percent: 60, message: 'Setting up addons' }),
                     addons.setupAddons.bind(null, app, app.manifest.addons),
                 ], next);
+            } else if (!restoreConfig.backupId) { // in-place import
+                async.series([
+                    progressCallback.bind(null, { percent: 60, message: 'Importing addons in-place' }),
+                    addons.setupAddons.bind(null, app, app.manifest.addons),
+                    addons.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
+                    addons.restoreAddons.bind(null, app, app.manifest.addons),
+                ], next);
             } else {
                 async.series([
                     progressCallback.bind(null, { percent: 65, message: 'Download backup and restoring addons' }),
                     addons.setupAddons.bind(null, app, app.manifest.addons),
                     addons.clearAddons.bind(null, app, app.manifest.addons),
-                    backups.restoreApp.bind(null, app, app.manifest.addons, restoreConfig, (progress) => {
-                        progressCallback({ percent: 65, message: `Restore - ${progress.message}` });
-                    })
+                    backups.downloadApp.bind(null, app, restoreConfig, (progress) => {
+                        progressCallback({ percent: 65, message: progress.message });
+                    }),
+                    addons.restoreAddons.bind(null, app, app.manifest.addons)
                 ], next);
             }
         },
@@ -580,12 +578,6 @@ function install(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 70, message: 'Creating container' }),
         createContainer.bind(null, app),
 
-        progressCallback.bind(null, { percent: 75, message: 'Setting up logrotate config' }),
-        addLogrotateConfig.bind(null, app),
-
-        progressCallback.bind(null, { percent: 80, message: 'Setting up collectd profile' }),
-        addCollectdProfile.bind(null, app),
-
         startApp.bind(null, app),
 
         progressCallback.bind(null, { percent: 85, message: 'Waiting for DNS propagation' }),
@@ -622,8 +614,8 @@ function backup(app, args, progressCallback, callback) {
     ], function seriesDone(error) {
         if (error) {
             debugApp(app, 'error backing up app: %s', error);
-            // return to installed state intentionally
-            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: error.toPlainObject ? error.toPlainObject() : error.message }, callback.bind(null, error));
+            // return to installed state intentionally. the error is stashed only in the task and not the app (the UI shows error state otherwise)
+            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }, callback.bind(null, makeTaskError(error, app)));
         }
         callback(null);
     });
@@ -637,7 +629,6 @@ function create(app, args, progressCallback, callback) {
 
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
 
         // FIXME: re-setup addons only because sendmail addon to re-inject env vars on mailboxName change
@@ -673,7 +664,6 @@ function changeLocation(app, args, progressCallback, callback) {
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
         unconfigureReverseProxy.bind(null, app),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
         function (next) {
             let obsoleteDomains = oldConfig.alternateDomains.filter(function (o) {
@@ -709,7 +699,7 @@ function changeLocation(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error reconfiguring : %s', error);
+            debugApp(app, 'error changing location : %s', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -722,12 +712,11 @@ function migrateDataDir(app, args, progressCallback, callback) {
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    let oldDataDir = args.oldDataDir;
-    assert(oldDataDir === null || typeof oldDataDir === 'string');
+    let newDataDir = args.newDataDir;
+    assert(newDataDir === null || typeof newDataDir === 'string');
 
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
 
         progressCallback.bind(null, { percent: 45, message: 'Ensuring app data directory' }),
@@ -735,72 +724,43 @@ function migrateDataDir(app, args, progressCallback, callback) {
 
         // re-setup addons since this creates the localStorage volume
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        addons.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),
 
-        // migrate dataDir
-        function (next) {
-            const dataDirChanged = oldDataDir !== app.dataDir;
+        progressCallback.bind(null, { percent: 60, message: 'Moving data dir' }),
+        moveDataDir.bind(null, app, newDataDir),
 
-            if (!dataDirChanged) return next();
-
-            moveDataDir(app, oldDataDir, next);
-        },
-
-        progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
+        progressCallback.bind(null, { percent: 90, message: 'Creating container' }),
         createContainer.bind(null, app),
 
         startApp.bind(null, app),
 
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
-        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
+        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error reconfiguring : %s', error);
+            debugApp(app, 'error migrating data dir : %s', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
     });
 }
 
-// configure is called for an infra update and repair
+// configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local"
 function configure(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    const oldConfig = args.oldConfig || null;
-    const overwriteDns = args.overwriteDns;
-
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
         unconfigureReverseProxy.bind(null, app),
-        removeCollectdProfile.bind(null, app),
-        removeLogrotateConfig.bind(null, app),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
-        function (next) {
-            if (!oldConfig) return next();
-
-            let obsoleteDomains = oldConfig.alternateDomains.filter(function (o) {
-                return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
-            });
-
-            if (oldConfig.fqdn !== app.fqdn) obsoleteDomains.push({ subdomain: oldConfig.location, domain: oldConfig.domain });
-
-            if (obsoleteDomains.length === 0) return next();
-
-            unregisterSubdomains(app, obsoleteDomains, next);
-        },
-
         reserveHttpPort.bind(null, app),
 
         progressCallback.bind(null, { percent: 20, message: 'Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
-        registerSubdomains.bind(null, app, overwriteDns),
-
         progressCallback.bind(null, { percent: 40, message: 'Downloading image' }),
         downloadImage.bind(null, app.manifest),
 
@@ -814,17 +774,8 @@ function configure(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
         createContainer.bind(null, app),
 
-        progressCallback.bind(null, { percent: 65, message: 'Setting up logrotate config' }),
-        addLogrotateConfig.bind(null, app),
-
-        progressCallback.bind(null, { percent: 70, message: 'Add collectd profile' }),
-        addCollectdProfile.bind(null, app),
-
         startApp.bind(null, app),
 
-        progressCallback.bind(null, { percent: 80, message: 'Waiting for DNS propagation' }),
-        exports._waitForDnsPropagation.bind(null, app),
-
         progressCallback.bind(null, { percent: 90, message: 'Configuring reverse proxy' }),
         configureReverseProxy.bind(null, app),
 
@@ -856,7 +807,7 @@ function update(app, args, progressCallback, callback) {
     async.series([
         // this protects against the theoretical possibility of an app being marked for update from
         // a previous version of box code
-        progressCallback.bind(null, { percent: 0, message: 'Verify manifest' }),
+        progressCallback.bind(null, { percent: 5, message: 'Verify manifest' }),
         verifyManifest.bind(null, updateConfig.manifest),
 
         function (next) {
@@ -882,7 +833,6 @@ function update(app, args, progressCallback, callback) {
         // note: we cleanup first and then backup. this is done so that the app is not running should backup fail
         // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
         progressCallback.bind(null, { percent: 35, message: 'Cleaning up old install' }),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
         function deleteImageIfChanged(done) {
             if (app.manifest.dockerImage === updateConfig.manifest.dockerImage) return done();
@@ -952,6 +902,10 @@ function start(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 20, message: 'Starting container' }),
         docker.startContainer.bind(null, app.id),
 
+        // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
+        progressCallback.bind(null, { percent: 60, message: 'Configuring reverse proxy' }),
+        configureReverseProxy.bind(null, app),
+
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
@@ -984,6 +938,27 @@ function stop(app, args, progressCallback, callback) {
     });
 }
 
+function restart(app, args, progressCallback, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        progressCallback.bind(null, { percent: 20, message: 'Restarting container' }),
+        docker.restartContainer.bind(null, app.id),
+
+        progressCallback.bind(null, { percent: 100, message: 'Done' }),
+        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error starting app: %s', error);
+            return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
+        }
+        callback(null);
+    });
+}
+
 function uninstall(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
@@ -991,16 +966,8 @@ function uninstall(app, args, progressCallback, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     async.series([
-        progressCallback.bind(null, { percent: 0, message: 'Remove collectd profile' }),
-        removeCollectdProfile.bind(null, app),
-
-        progressCallback.bind(null, { percent: 5, message: 'Remove logrotate config' }),
-        removeLogrotateConfig.bind(null, app),
-
-        progressCallback.bind(null, { percent: 10, message: 'Stopping app' }),
-        docker.stopContainers.bind(null, app.id),
-
         progressCallback.bind(null, { percent: 20, message: 'Deleting container' }),
+        unconfigureReverseProxy.bind(null, app),
         deleteContainers.bind(null, app, {}),
 
         progressCallback.bind(null, { percent: 30, message: 'Teardown addons' }),
@@ -1018,9 +985,6 @@ function uninstall(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 70, message: 'Cleanup icon' }),
         removeIcon.bind(null, app),
 
-        progressCallback.bind(null, { percent: 80, message: 'Unconfiguring reverse proxy' }),
-        unconfigureReverseProxy.bind(null, app),
-
         progressCallback.bind(null, { percent: 90, message: 'Cleanup logs' }),
         cleanupLogs.bind(null, app),
 
@@ -1072,9 +1036,11 @@ function run(appId, args, progressCallback, callback) {
             return start(app, args, progressCallback, callback);
         case apps.ISTATE_PENDING_STOP:
             return stop(app, args, progressCallback, callback);
+        case apps.ISTATE_PENDING_RESTART:
+            return restart(app, args, progressCallback, callback);
         default:
             debugApp(app, 'apptask launched with invalid command');
-            return callback(new Error('Unknown install command in apptask:' + app.installationState));
+            return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState));
         }
     });
 }

src/apptaskmanager.js

@@ -5,6 +5,7 @@ exports = module.exports = {
 };
 
 let assert = require('assert'),
+    BoxError = require('./boxerror.js'),
     debug = require('debug')('box:apptaskmanager'),
     fs = require('fs'),
     locker = require('./locker.js'),
@@ -42,12 +43,12 @@ function scheduleTask(appId, taskId, callback) {
     if (!gInitialized) initializeSync();
 
     if (appId in gActiveTasks) {
-        return callback(new Error(`Task for %s is already active: ${appId}`));
+        return callback(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
     }
 
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 0, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
+        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
         gPendingTasks.push({ appId, taskId, callback });
         return;
     }
@@ -56,7 +57,7 @@ function scheduleTask(appId, taskId, callback) {
 
     if (lockError) {
         debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 0, message: waitText(lockError.operation) }, NOOP_CALLBACK);
+        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
         gPendingTasks.push({ appId, taskId, callback });
         return;
     }

src/auditsource.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     HEALTH_MONITOR: { userId: null, username: 'healthmonitor' },
     APP_TASK: { userId: null, username: 'apptask' },
     EXTERNAL_LDAP_TASK: { userId: null, username: 'externalldap' },
+    EXTERNAL_LDAP_AUTO_CREATE: { userId: null, username: 'externalldap' },
 
     fromRequest: fromRequest
 };

src/authcodedb.js

@@ -1,78 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-exports = module.exports = {
-    get: get,
-    add: add,
-    del: del,
-    delExpired: delExpired,
-
-    _clear: clear
-};
-
-var assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js');
-
-var AUTHCODES_FIELDS = [ 'authCode', 'userId', 'clientId', 'expiresAt' ].join(',');
-
-function get(authCode, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + AUTHCODES_FIELDS + ' FROM authcodes WHERE authCode = ? AND expiresAt > ?', [ authCode, Date.now() ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Authcode not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function add(authCode, clientId, userId, expiresAt, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof expiresAt, 'number');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('INSERT INTO authcodes (authCode, clientId, userId, expiresAt) VALUES (?, ?, ?, ?)',
-        [ authCode, clientId, userId, expiresAt ], function (error, result) {
-            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-            if (error || result.affectedRows !== 1) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            callback(null);
-        });
-}
-
-function del(authCode, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes WHERE authCode = ?', [ authCode ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Authcode not found'));
-
-        callback(null);
-    });
-}
-
-function delExpired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes WHERE expiresAt <= ?', [ Date.now() ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        return callback(null, result.affectedRows);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes', function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-

src/backups.js

@@ -2,6 +2,7 @@
 
 exports = module.exports = {
     testConfig: testConfig,
+    testProviderConfig: testProviderConfig,
 
     getByStatePaged: getByStatePaged,
     getByAppIdPaged: getByAppIdPaged,
@@ -14,7 +15,7 @@ exports = module.exports = {
     restore: restore,
 
     backupApp: backupApp,
-    restoreApp: restoreApp,
+    downloadApp: downloadApp,
 
     backupBoxAndApps: backupBoxAndApps,
 
@@ -29,6 +30,8 @@ exports = module.exports = {
 
     checkConfiguration: checkConfiguration,
 
+    configureCollectd: configureCollectd,
+
     SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8),
 
     // for testing
@@ -43,12 +46,14 @@ var addons = require('./addons.js'),
     assert = require('assert'),
     backupdb = require('./backupdb.js'),
     BoxError = require('./boxerror.js'),
+    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     crypto = require('crypto'),
     database = require('./database.js'),
     DataLayout = require('./datalayout.js'),
     debug = require('debug')('box:backups'),
     df = require('@sindresorhus/df'),
+    ejs = require('ejs'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
     locker = require('./locker.js'),
@@ -68,6 +73,7 @@ var addons = require('./addons.js'),
     zlib = require('zlib');
 
 const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');
+const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/cloudron-backup.ejs', { encoding: 'utf8' });
 
 function debugApp(app) {
     assert(typeof app === 'object');
@@ -87,6 +93,7 @@ function api(provider) {
     case 'exoscale-sos': return require('./storage/s3.js');
     case 'wasabi': return require('./storage/s3.js');
     case 'scaleway-objectstorage': return require('./storage/s3.js');
+    case 'linode-objectstorage': return require('./storage/s3.js');
     case 'noop': return require('./storage/noop.js');
     default: return null;
     }
@@ -118,6 +125,17 @@ function testConfig(backupConfig, callback) {
     api(backupConfig.provider).testConfig(backupConfig, callback);
 }
 
+
+function testProviderConfig(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var func = api(backupConfig.provider);
+    if (!func) return callback(new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' }));
+
+    api(backupConfig.provider).testConfig(backupConfig, callback);
+}
+
 function getByStatePaged(state, page, perPage, callback) {
     assert.strictEqual(typeof state, 'string');
     assert(typeof page === 'number' && page > 0);
@@ -217,14 +235,14 @@ function createReadStream(sourceFile, key) {
 
     stream.on('error', function (error) {
         debug('createReadStream: read stream error.', error);
-        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, error.message));
     });
 
     if (key !== null) {
         var encrypt = crypto.createCipher('aes-256-cbc', key);
         encrypt.on('error', function (error) {
             debug('createReadStream: encrypt stream error.', error);
-            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
         });
         return stream.pipe(encrypt).pipe(ps);
     } else {
@@ -237,17 +255,25 @@ function createWriteStream(destFile, key) {
     assert(key === null || typeof key === 'string');
 
     var stream = fs.createWriteStream(destFile);
+    var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+
+    stream.on('error', function (error) {
+        debug('createWriteStream: write stream error.', error);
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, error.message));
+    });
 
     if (key !== null) {
         var decrypt = crypto.createDecipher('aes-256-cbc', key);
         decrypt.on('error', function (error) {
             debug('createWriteStream: decrypt stream error.', error);
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
         });
-        decrypt.pipe(stream);
-        return decrypt;
+        ps.pipe(decrypt).pipe(stream);
     } else {
-        return stream;
+        ps.pipe(stream);
     }
+
+    return ps;
 }
 
 function tarPack(dataLayout, key, callback) {
@@ -338,7 +364,7 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
                     debug(`read stream error for ${task.path}: ${error.message}`);
                     retryCallback();
                 }); // ignore error if file disappears
-                stream.on('progress', function(progress) {
+                stream.on('progress', function (progress) {
                     const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
                     if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0Mbps looks wrong
                     progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}Mbps` }); // 0M@0Mbps looks wrong
@@ -437,7 +463,7 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback)
                     tarPack(dataLayout, backupConfig.key || null, function (error, tarStream) {
                         if (error) return retryCallback(error);
 
-                        tarStream.on('progress', function(progress) {
+                        tarStream.on('progress', function (progress) {
                             const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
                             if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0Mbps looks wrong
                             progressCallback({ message: `Uploading backup ${transferred}M@${speed}Mbps` });
@@ -545,24 +571,30 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
 
     debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
 
-    function downloadFile(entry, callback) {
+    function downloadFile(entry, done) {
         let relativePath = path.relative(backupFilePath, entry.fullPath);
         if (backupConfig.key) {
             relativePath = decryptFilePath(relativePath, backupConfig.key);
-            if (!relativePath) return callback(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file'));
+            if (!relativePath) return done(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file'));
         }
         const destFilePath = dataLayout.toLocalPath('./' + relativePath);
 
         mkdirp(path.dirname(destFilePath), function (error) {
-            if (error) return callback(new BoxError(BoxError.FS_ERROR, error.message));
+            if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));
 
             async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
                 let destStream = createWriteStream(destFilePath, backupConfig.key || null);
 
+                destStream.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0Mbps looks wrong
+                    progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}Mbps` });
+                });
+
                 // protect against multiple errors. must destroy the write stream so that a previous retry does not write
                 let closeAndRetry = once((error) => {
-                    if (error) progressCallback({ message: `Download ${entry.fullPath} errored: ${error.message}` });
-                    else progressCallback({ message: `Download ${entry.fullPath} finished` });
+                    if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                    else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` });
                     destStream.destroy();
                     retryCallback(error);
                 });
@@ -571,21 +603,21 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
                     if (error) return closeAndRetry(error);
 
                     sourceStream.on('error', closeAndRetry);
-                    destStream.on('error', closeAndRetry);
+                    destStream.on('error', closeAndRetry); // already emits BoxError
 
                     progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
 
                     sourceStream.pipe(destStream, { end: true }).on('finish', closeAndRetry);
                 });
-            }, callback);
+            }, done);
         });
     }
 
-    api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, done) {
+    api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
         // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
         const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
 
-        async.eachLimit(entries, concurrency, downloadFile, done);
+        async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
     }, callback);
 }
 
@@ -597,7 +629,7 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback,
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`download - Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`);
+    debug(`download: Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`);
 
     const backupFilePath = getBackupFilePath(backupConfig, backupId, format);
 
@@ -651,9 +683,8 @@ function restore(backupConfig, backupId, progressCallback, callback) {
     });
 }
 
-function restoreApp(app, addonsToRestore, restoreConfig, progressCallback, callback) {
+function downloadApp(app, restoreConfig, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof addonsToRestore, 'object');
     assert.strictEqual(typeof restoreConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
@@ -662,30 +693,32 @@ function restoreApp(app, addonsToRestore, restoreConfig, progressCallback, callb
     if (!appDataDir) return callback(safe.error);
     const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
 
-    var startTime = new Date();
+    const startTime = new Date();
+    const getBackupConfigFunc = restoreConfig.backupConfig ? (next) => next(null, restoreConfig.backupConfig) : settings.getBackupConfig;
 
-    settings.getBackupConfig(function (error, backupConfig) {
+    getBackupConfigFunc(function (error, backupConfig) {
         if (error) return callback(error);
 
-        async.series([
-            download.bind(null, backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback),
-            addons.restoreAddons.bind(null, app, addonsToRestore)
-        ], function (error) {
-            debug('restoreApp: time: %s', (new Date() - startTime)/1000);
+        download(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback, function (error) {
+            debug('downloadApp: time: %s', (new Date() - startTime)/1000);
 
             callback(error);
         });
     });
 }
 
-function runBackupUpload(backupId, format, dataLayout, progressCallback, callback) {
-    assert.strictEqual(typeof backupId, 'string');
-    assert.strictEqual(typeof format, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+function runBackupUpload(uploadConfig, progressCallback, callback) {
+    assert.strictEqual(typeof uploadConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    let result = '';
+    const { backupId, format, dataLayout, progressTag } = uploadConfig;
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof format, 'string');
+    assert.strictEqual(typeof progressTag, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    let result = ''; // the script communicates error result as a string
 
     shell.sudo(`backup-${backupId}`, [ BACKUP_UPLOAD_CMD, backupId, format, dataLayout.toString() ], { preserveEnv: true, ipc: true }, function (error) {
         if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
@@ -695,10 +728,10 @@ function runBackupUpload(backupId, format, dataLayout, progressCallback, callbac
         }
 
         callback();
-    }).on('message', function (message) {
-        if (!message.result) return progressCallback(message);
-        debug(`runBackupUpload: result - ${JSON.stringify(message)}`);
-        result = message.result;
+    }).on('message', function (progress) { // this is { message } or { result }
+        if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
+        debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
+        result = progress.result;
     });
 }
 
@@ -752,8 +785,14 @@ function uploadBoxSnapshot(backupConfig, progressCallback, callback) {
         const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
         if (!boxDataDir) return callback(safe.error);
 
-        const dataLayout = new DataLayout(boxDataDir, []);
-        runBackupUpload('snapshot/box', backupConfig.format, dataLayout, progressCallback, function (error) {
+        const uploadConfig = {
+            backupId: 'snapshot/box',
+            format: backupConfig.format,
+            dataLayout: new DataLayout(boxDataDir, []),
+            progressTag: 'box'
+        };
+
+        runBackupUpload(uploadConfig, progressCallback, function (error) {
             if (error) return callback(error);
 
             debug('uploadBoxSnapshot: time: %s secs', (new Date() - startTime)/1000);
@@ -782,7 +821,7 @@ function rotateBoxBackup(backupConfig, tag, appBackupIds, progressCallback, call
         if (error) return callback(error);
 
         var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message }));
+        copy.on('progress', (message) => progressCallback({ message: `box: ${message}` }));
         copy.on('done', function (copyBackupError) {
             const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
 
@@ -863,7 +902,7 @@ function rotateAppBackup(backupConfig, app, tag, options, progressCallback, call
         if (error) return callback(error);
 
         var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message }));
+        copy.on('progress', (message) => progressCallback({ message: `${message} (${app.fqdn})` }));
         copy.on('done', function (copyBackupError) {
             const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
 
@@ -898,7 +937,14 @@ function uploadAppSnapshot(backupConfig, app, progressCallback, callback) {
 
         const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
 
-        runBackupUpload(backupId, backupConfig.format, dataLayout, progressCallback, function (error) {
+        const uploadConfig = {
+            backupId,
+            format: backupConfig.format,
+            dataLayout,
+            progressTag: app.fqdn
+        };
+
+        runBackupUpload(uploadConfig, progressCallback, function (error) {
             if (error) return callback(error);
 
             debugApp(app, 'uploadAppSnapshot: %s done time: %s secs', backupId, (new Date() - startTime)/1000);
@@ -1262,3 +1308,15 @@ function checkConfiguration(callback) {
         callback(null, message);
     });
 }
+
+function configureCollectd(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (backupConfig.provider === 'filesystem') {
+        const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { backupDir: backupConfig.backupFolder });
+        collectd.addProfile('cloudron-backup', collectdConf, callback);
+    } else {
+        collectd.removeProfile('cloudron-backup', callback);
+    }
+}

src/boxerror.js

@@ -33,6 +33,7 @@ function BoxError(reason, errorOrMessage, details) {
 }
 util.inherits(BoxError, Error);
 BoxError.ACCESS_DENIED = 'Access Denied';
+BoxError.ADDONS_ERROR = 'Addons Error';
 BoxError.ALREADY_EXISTS = 'Already Exists';
 BoxError.BAD_FIELD = 'Bad Field';
 BoxError.BAD_STATE = 'Bad State';
@@ -44,6 +45,7 @@ BoxError.DATABASE_ERROR = 'Database Error';
 BoxError.DNS_ERROR = 'DNS Error';
 BoxError.DOCKER_ERROR = 'Docker Error';
 BoxError.EXTERNAL_ERROR = 'External Error'; // use this for external API errors
+BoxError.FEATURE_DISABLED = 'Feature Disabled';
 BoxError.FS_ERROR = 'FileSystem Error';
 BoxError.INACTIVE = 'Inactive';
 BoxError.INTERNAL_ERROR = 'Internal Error';
@@ -58,9 +60,10 @@ BoxError.NOT_IMPLEMENTED = 'Not implemented';
 BoxError.NOT_SIGNED = 'Not Signed';
 BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.PLAN_LIMIT = 'Plan Limit';
+BoxError.SPAWN_ERROR = 'Spawn Error';
 BoxError.TASK_ERROR = 'Task Error';
+BoxError.TIMEOUT = 'Timeout';
 BoxError.TRY_AGAIN = 'Try Again';
-BoxError.UNKNOWN_ERROR = 'Unknown Error'; // only used for porting
 
 BoxError.prototype.toPlainObject = function () {
     return _.extend({}, { message: this.message, reason: this.reason }, this.details);
@@ -75,6 +78,8 @@ BoxError.toHttpError = function (error) {
         return new HttpError(402, error);
     case BoxError.NOT_FOUND:
         return new HttpError(404, error);
+    case BoxError.FEATURE_DISABLED:
+        return new HttpError(405, error);
     case BoxError.ALREADY_EXISTS:
     case BoxError.BAD_STATE:
     case BoxError.CONFLICT:
@@ -86,6 +91,7 @@ BoxError.toHttpError = function (error) {
     case BoxError.FS_ERROR:
     case BoxError.MAIL_ERROR:
     case BoxError.DOCKER_ERROR:
+    case BoxError.ADDONS_ERROR:
         return new HttpError(424, error);
     case BoxError.DATABASE_ERROR:
     case BoxError.INTERNAL_ERROR:

src/cert/acme2.js

@@ -9,8 +9,8 @@ var assert = require('assert'),
     fs = require('fs'),
     path = require('path'),
     paths = require('../paths.js'),
+    request = require('request'),
     safe = require('safetydance'),
-    superagent = require('superagent'),
     util = require('util'),
     _ = require('underscore');
 
@@ -41,15 +41,6 @@ function Acme2(options) {
     this.wildcard = !!options.wildcard;
 }
 
-Acme2.prototype.getNonce = function (callback) {
-    superagent.get(this.directory.newNonce).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(error);
-        if (response.statusCode !== 204) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
-
-        return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
-    });
-};
-
 // urlsafe base64 encoding (jose)
 function urlBase64Encode(string) {
     return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
@@ -96,8 +87,12 @@ Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
 
     var payload64 = b64(payload);
 
-    this.getNonce(function (error, nonce) {
-        if (error) return callback(error);
+    request.get(this.directory.newNonce, { json: true, timeout: 30000 }, function (error, response) {
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`));
+        if (response.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching nonce : ' + response.statusCode));
+
+        const nonce = response.headers['Replay-Nonce'.toLowerCase()];
+        if (!nonce) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response'));
 
         debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
 
@@ -113,14 +108,23 @@ Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
             signature: signature64
         };
 
-        superagent.post(url).set('Content-Type', 'application/jose+json').set('User-Agent', 'acme-cloudron').send(JSON.stringify(data)).timeout(30 * 1000).end(function (error, res) {
-            if (error && !error.response) return callback(error); // network errors
+        request.post(url, { headers: { 'Content-Type': 'application/jose+json', 'User-Agent': 'acme-cloudron' }, body: JSON.stringify(data), timeout: 30000 }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`)); // network error
 
-            callback(null, res);
+            // we don't set json: true in request because it ends up mangling the content-type
+            // we don't set json: true in request because it ends up mangling the content-type
+            if (response.headers['content-type'] === 'application/json') response.body = safe.JSON.parse(response.body);
+
+            callback(null, response);
         });
     });
 };
 
+// https://tools.ietf.org/html/rfc8555#section-6.3
+Acme2.prototype.postAsGet = function (url, callback) {
+    this.sendSignedRequest(url, '', callback);
+};
+
 Acme2.prototype.updateContact = function (registrationUri, callback) {
     assert.strictEqual(typeof registrationUri, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -134,7 +138,7 @@ Acme2.prototype.updateContact = function (registrationUri, callback) {
 
     const that = this;
     this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when updating contact: ${error.message}`));
+        if (error) return callback(error);
         if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 200, got %s %s', result.statusCode, result.text)));
 
         debug(`updateContact: contact of user updated to ${that.email}`);
@@ -154,7 +158,7 @@ Acme2.prototype.registerUser = function (callback) {
 
     var that = this;
     this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when registering user: ${error.message}`));
+        if (error) return callback(error);
         // 200 if already exists. 201 for new accounts
         if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register new account. Expecting 200 or 201, got %s %s', result.statusCode, result.text)));
 
@@ -180,7 +184,7 @@ Acme2.prototype.newOrder = function (domain, callback) {
     debug('newOrder: %s', domain);
 
     this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when creating new order: ${error.message}`));
+        if (error) return callback(error);
         if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending signed request: ${result.body.detail}`));
         if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
 
@@ -201,14 +205,15 @@ Acme2.prototype.waitForOrder = function (orderUrl, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     debug(`waitForOrder: ${orderUrl}`);
+    const that = this;
 
     async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
         debug('waitForOrder: getting status');
 
-        superagent.get(orderUrl).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) {
+        that.postAsGet(orderUrl, function (error, result) {
+            if (error) {
                 debug('waitForOrder: network error getting uri %s', orderUrl);
-                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for order: ${error.message}`)); // network error
+                return retryCallback(error);
             }
             if (result.statusCode !== 200) {
                 debug('waitForOrder: invalid response code getting uri %s', result.statusCode);
@@ -253,7 +258,7 @@ Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
     };
 
     this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when notifying challenge: ${error.message}`));
+        if (error) return callback(error);
         if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 200, got %s %s', result.statusCode, result.text)));
 
         callback();
@@ -265,21 +270,22 @@ Acme2.prototype.waitForChallenge = function (challenge, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     debug('waitingForChallenge: %j', challenge);
+    const that = this;
 
     async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
         debug('waitingForChallenge: getting status');
 
-        superagent.get(challenge.url).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) {
+        that.postAsGet(challenge.url, function (error, result) {
+            if (error) {
                 debug('waitForChallenge: network error getting uri %s', challenge.url);
-                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for challenge: ${error.message}`));
+                return retryCallback(error);
             }
             if (result.statusCode !== 200) {
                 debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
                 return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
             }
 
-            debug('waitForChallenge: status is "%s %j', result.body.status, result.body);
+            debug('waitForChallenge: status is "%s" %j', result.body.status, result.body);
 
             if (result.body.status === 'pending') return retryCallback(new BoxError(BoxError.TRY_AGAIN));
             else if (result.body.status === 'valid') return retryCallback();
@@ -305,7 +311,7 @@ Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, cal
     debug('signCertificate: sending sign request');
 
     this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when signing certificate: ${error.message}`));
+        if (error) return callback(error);
         // 429 means we reached the cert limit for this domain
         if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 200, got %s %s', result.statusCode, result.text)));
 
@@ -348,20 +354,17 @@ Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var outdir = paths.APP_CERTS_DIR;
+    const that = this;
 
     async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
         debug('downloadCertificate: downloading certificate');
 
-        superagent.get(certUrl).buffer().parse(function (res, done) {
-            var data = [ ];
-            res.on('data', function(chunk) { data.push(chunk); });
-            res.on('end', function () { res.text = Buffer.concat(data); done(); });
-        }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
-            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry'));
+        that.postAsGet(certUrl, function (error, result) {
+            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
+            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
             if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
 
-            const fullChainPem = result.text;
+            const fullChainPem = result.body; // buffer
 
             const certName = hostname.replace('*.', '_.');
             var certificateFile = path.join(outdir, `${certName}.cert`);
@@ -449,7 +452,7 @@ Acme2.prototype.prepareDnsChallenge = function (hostname, domain, authorization,
     domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
         if (error) return callback(error);
 
-        domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { interval: 5000, times: 200 }, function (error) {
+        domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 }, function (error) {
             if (error) return callback(error);
 
             callback(null, challenge);
@@ -488,8 +491,8 @@ Acme2.prototype.prepareChallenge = function (hostname, domain, authorizationUrl,
     debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
 
     const that = this;
-    superagent.get(authorizationUrl).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when preparing challenge: ${error.message}`));
+    this.postAsGet(authorizationUrl, function (error, response) {
+        if (error) return callback(error);
         if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code getting authorization : ' + response.statusCode));
 
         const authorization = response.body;
@@ -569,13 +572,13 @@ Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
 Acme2.prototype.getDirectory = function (callback) {
     const that = this;
 
-    superagent.get(this.caDirectory).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
+    request.get(this.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
         if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
 
         if (typeof response.body.newNonce !== 'string' ||
             typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') return callback(new Error(`Invalid response body : ${response.body}`));
+            typeof response.body.newAccount !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));
 
         that.directory = response.body;
 

src/cert/interface.js

@@ -10,7 +10,8 @@ exports = module.exports = {
     getCertificate: getCertificate
 };
 
-var assert = require('assert');
+var assert = require('assert'),
+    BoxError = require('../boxerror.js');
 
 function getCertificate(hostname, domain, options, callback) {
     assert.strictEqual(typeof hostname, 'string');
@@ -18,6 +19,6 @@ function getCertificate(hostname, domain, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    return callback(new Error('Not implemented'));
+    return callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'getCertificate is not implemented'));
 }
 

src/clientdb.js

@@ -1,179 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get: get,
-    getAll: getAll,
-    getAllWithTokenCount: getAllWithTokenCount,
-    getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
-    add: add,
-    del: del,
-    getByAppId: getByAppId,
-    getByAppIdAndType: getByAppIdAndType,
-
-    upsert: upsert,
-
-    delByAppId: delByAppId,
-    delByAppIdAndType: delByAppIdAndType,
-
-    _clear: clear
-};
-
-var assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js');
-
-var CLIENTS_FIELDS = [ 'id', 'appId', 'type', 'clientSecret', 'redirectURI', 'scope' ].join(',');
-var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.type', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE id = ?', [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, `Client not found: ${id}`));
-
-        callback(null, result[0]);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients ORDER BY appId', function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAllWithTokenCount(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAllWithTokenCountByIdentifier(identifier, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? LIMIT 1', [ appId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function getByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? AND type = ? LIMIT 1', [ appId, type ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function add(id, appId, type, clientSecret, redirectURI, scope, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof clientSecret, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
-
-    database.query('INSERT INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-        if (error || result.affectedRows === 0) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function upsert(id, appId, type, clientSecret, redirectURI, scope, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof clientSecret, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
-
-    database.query('REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-        if (error || result.affectedRows === 0) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients WHERE id = ?', [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, `Client not found: ${id}`));
-
-        callback(null);
-    });
-}
-
-function delByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients WHERE appId=?', [ appId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null);
-    });
-}
-
-function delByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients WHERE appId=? AND type=?', [ appId, type ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients', function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}

src/clients.js

@@ -1,329 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    add: add,
-    get: get,
-    del: del,
-    getAll: getAll,
-    getByAppIdAndType: getByAppIdAndType,
-    getTokensByUserId: getTokensByUserId,
-    delTokensByUserId: delTokensByUserId,
-    delByAppIdAndType: delByAppIdAndType,
-    addTokenByUserId: addTokenByUserId,
-    delToken: delToken,
-
-    issueDeveloperToken: issueDeveloperToken,
-
-    addDefaultClients: addDefaultClients,
-
-    removeTokenPrivateFields: removeTokenPrivateFields,
-
-    // client ids. we categorize them so we can have different restrictions based on the client
-    ID_WEBADMIN: 'cid-webadmin',    // dashboard oauth
-    ID_SDK: 'cid-sdk',              // created by user via dashboard
-    ID_CLI: 'cid-cli',              // created via cli tool
-
-    // client type enums
-    TYPE_EXTERNAL: 'external',
-    TYPE_BUILT_IN: 'built-in',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_PROXY: 'addon-proxy'
-};
-
-var apps = require('./apps.js'),
-    assert = require('assert'),
-    async = require('async'),
-    BoxError = require('./boxerror.js'),
-    clientdb = require('./clientdb.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:clients'),
-    eventlog = require('./eventlog.js'),
-    hat = require('./hat.js'),
-    accesscontrol = require('./accesscontrol.js'),
-    tokendb = require('./tokendb.js'),
-    users = require('./users.js'),
-    uuid = require('uuid'),
-    _ = require('underscore');
-
-function validateClientName(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 character', { field: 'name' });
-    if (name.length > 128) return new BoxError(BoxError.BAD_FIELD, 'name too long', { field: 'name' });
-
-    if (/[^a-zA-Z0-9-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'name can only contain alphanumerals and dash', { field: 'name' });
-
-    return null;
-}
-
-function validateTokenName(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (name.length > 64) return new BoxError(BoxError.BAD_FIELD, 'name too long', { field: 'name' });
-
-    return null;
-}
-
-function add(appId, type, redirectURI, scope, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = accesscontrol.validateScopeString(scope);
-    if (error) return callback(error);
-
-    error = validateClientName(appId);
-    if (error) return callback(error);
-
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(8 * 128);
-
-    clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
-        if (error) return callback(error);
-
-        var client = {
-            id: id,
-            appId: appId,
-            type: type,
-            clientSecret: clientSecret,
-            redirectURI: redirectURI,
-            scope: scope
-        };
-
-        callback(null, client);
-    });
-}
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.get(id, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.del(id, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.getAll(function (error, results) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, []);
-        if (error) return callback(error);
-
-        var tmp = [];
-        async.each(results, function (record, callback) {
-            if (record.type === exports.TYPE_EXTERNAL || record.type === exports.TYPE_BUILT_IN) {
-                // the appId in this case holds the name
-                record.name = record.appId;
-
-                tmp.push(record);
-
-                return callback(null);
-            }
-
-            apps.get(record.appId, function (error, result) {
-                if (error) {
-                    console.error('Failed to get app details for oauth client', record.appId, error);
-                    return callback(null);  // ignore error so we continue listing clients
-                }
-
-                if (record.type === exports.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === exports.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-
-                record.domain = result.fqdn;
-
-                tmp.push(record);
-
-                callback(null);
-            });
-        }, function (error) {
-            if (error) return callback(error);
-            callback(null, tmp);
-        });
-    });
-}
-
-function getByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.getByAppIdAndType(appId, type, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function getTokensByUserId(clientId, userId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.getByIdentifierAndClientId(userId, clientId, function (error, result) {
-        if (error && error.reason === BoxError.NOT_FOUND) {
-            // this can mean either that there are no tokens or the clientId is actually unknown
-            get(clientId, function (error/*, result*/) {
-                if (error) return callback(error);
-                callback(null, []);
-            });
-            return;
-        }
-        if (error) return callback(error);
-        callback(null, result || []);
-    });
-}
-
-function delTokensByUserId(clientId, userId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.delByIdentifierAndClientId(userId, clientId, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) {
-            // this can mean either that there are no tokens or the clientId is actually unknown
-            get(clientId, function (error/*, result*/) {
-                if (error) return callback(error);
-                callback(null);
-            });
-            return;
-        }
-        if (error) return callback(error);
-        callback(null);
-    });
-}
-
-function delByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    getByAppIdAndType(appId, type, function (error, result) {
-        if (error) return callback(error);
-
-        tokendb.delByClientId(result.id, function (error) {
-            if (error && error.reason !== BoxError.NOT_FOUND) return callback(error);
-
-            clientdb.delByAppIdAndType(appId, type, function (error) {
-                if (error) return callback(error);
-
-                callback(null);
-            });
-        });
-    });
-}
-
-function addTokenByUserId(clientId, userId, expiresAt, options, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof expiresAt, 'number');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const name = options.name || '';
-    let error = validateTokenName(name);
-    if (error) return callback(error);
-
-    get(clientId, function (error, result) {
-        if (error) return callback(error);
-
-        users.get(userId, function (error, user) {
-            if (error) return callback(error);
-
-            accesscontrol.scopesForUser(user, function (error, userScopes) {
-                if (error) return callback(error);
-
-                const scope = accesscontrol.canonicalScopeString(result.scope);
-                const authorizedScopes = accesscontrol.intersectScopes(userScopes, scope.split(','));
-
-                const token = {
-                    id: 'tid-' + uuid.v4(),
-                    accessToken: hat(8 * 32),
-                    identifier: userId,
-                    clientId: result.id,
-                    expires: expiresAt,
-                    scope: authorizedScopes.join(','),
-                    name: name
-                };
-
-                tokendb.add(token, function (error) {
-                    if (error) return callback(error);
-
-                    callback(null, {
-                        accessToken: token.accessToken,
-                        tokenScopes: authorizedScopes,
-                        identifier: userId,
-                        clientId: result.id,
-                        expires: expiresAt
-                    });
-                });
-            });
-        });
-    });
-}
-
-function issueDeveloperToken(userObject, auditSource, callback) {
-    assert.strictEqual(typeof userObject, 'object');
-    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const expiresAt = Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
-
-    addTokenByUserId(exports.ID_CLI, userObject.id, expiresAt, {}, function (error, result) {
-        if (error) return callback(error);
-
-        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: userObject.id, user: users.removePrivateFields(userObject) });
-
-        callback(null, result);
-    });
-}
-
-function delToken(clientId, tokenId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof tokenId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    get(clientId, function (error) {
-        if (error) return callback(error);
-
-        tokendb.del(tokenId, function (error) {
-            if (error) return callback(error);
-
-            callback(null);
-        });
-    });
-}
-
-function addDefaultClients(origin, callback) {
-    assert.strictEqual(typeof origin, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Adding default clients');
-
-    // The domain might have changed, therefor we have to update the record
-    // id, appId, type, clientSecret, redirectURI, scope
-    async.series([
-        clientdb.upsert.bind(null, exports.ID_WEBADMIN, 'Settings', 'built-in', 'secret-webadmin', origin, '*'),
-        clientdb.upsert.bind(null, exports.ID_SDK, 'SDK', 'built-in', 'secret-sdk', origin, '*'),
-        clientdb.upsert.bind(null, exports.ID_CLI, 'Cloudron Tool', 'built-in', 'secret-cli', origin, '*')
-    ], callback);
-}
-
-function removeTokenPrivateFields(token) {
-    return _.pick(token, 'id', 'identifier', 'clientId', 'scope', 'expires', 'name');
-}

src/cloudron.js

@@ -21,23 +21,22 @@ exports = module.exports = {
     runSystemChecks: runSystemChecks,
 };
 
-var apps = require('./apps.js'),
+var addons = require('./addons.js'),
+    apps = require('./apps.js'),
+    appstore = require('./appstore.js'),
     assert = require('assert'),
     async = require('async'),
     auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
-    clients = require('./clients.js'),
     constants = require('./constants.js'),
     cron = require('./cron.js'),
     debug = require('debug')('box:cloudron'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
-    custom = require('./custom.js'),
     fs = require('fs'),
     mail = require('./mail.js'),
     notifications = require('./notifications.js'),
-    os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
     platform = require('./platform.js'),
@@ -107,6 +106,12 @@ function runStartupTasks() {
     // configure nginx to be reachable by IP
     reverseProxy.writeDefaultConfig(NOOP_CALLBACK);
 
+    // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return console.error('Failed to read backup config.', error);
+        backups.configureCollectd(backupConfig, NOOP_CALLBACK);
+    });
+
     // always generate webadmin config since we have no versioning mechanism for the ejs
     if (settings.adminDomain()) reverseProxy.writeAdminConfig(settings.adminDomain(), NOOP_CALLBACK);
 
@@ -134,16 +139,20 @@ function getConfig(callback) {
             mailFqdn: settings.mailFqdn(),
             version: constants.VERSION,
             isDemo: settings.isDemo(),
-            memory: os.totalmem(),
             provider: settings.provider(),
             cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
-            uiSpec: custom.uiSpec()
+            footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
+            features: appstore.getFeatures()
         });
     });
 }
 
 function reboot(callback) {
-    shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
+    notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '', function (error) {
+        if (error) console.error('Failed to clear reboot notification.', error);
+
+        shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
+    });
 }
 
 function isRebootRequired(callback) {
@@ -154,20 +163,20 @@ function isRebootRequired(callback) {
 }
 
 // called from cron.js
-function runSystemChecks() {
+function runSystemChecks(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     async.parallel([
         checkBackupConfiguration,
         checkMailStatus,
         checkRebootRequired
-    ], function (error) {
-        debug('runSystemChecks: done', error);
-    });
+    ], callback);
 }
 
 function checkBackupConfiguration(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    debug('Checking backup configuration');
+    debug('checking backup configuration');
 
     backups.checkConfiguration(function (error, message) {
         if (error) return callback(error);
@@ -196,7 +205,7 @@ function checkRebootRequired(callback) {
     isRebootRequired(function (error, rebootRequired) {
         if (error) return callback(error);
 
-        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish security updates, a [reboot](/#/system) is necessary.' : '', callback);
+        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '', callback);
     });
 }
 
@@ -253,6 +262,8 @@ function prepareDashboardDomain(domain, auditSource, callback) {
 
     debug(`prepareDashboardDomain: ${domain}`);
 
+    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+
     domains.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
@@ -291,10 +302,7 @@ function setDashboardDomain(domain, auditSource, callback) {
 
             const fqdn = domains.fqdn(constants.ADMIN_LOCATION, domainObject);
 
-            async.series([
-                (done) => settings.setAdmin(domain, fqdn, done),
-                (done) => clients.addDefaultClients(settings.adminOrigin(), done)
-            ], function (error) {
+            settings.setAdmin(domain, fqdn, function (error) {
                 if (error) return callback(error);
 
                 eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain: domain, fqdn: fqdn });
@@ -313,10 +321,13 @@ function setDashboardAndMailDomain(domain, auditSource, callback) {
 
     debug(`setDashboardAndMailDomain: ${domain}`);
 
+    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+
     setDashboardDomain(domain, auditSource, function (error) {
         if (error) return callback(error);
 
         mail.onMailFqdnChanged(NOOP_CALLBACK); // this will update dns and re-configure mail server
+        addons.restartService('turn', NOOP_CALLBACK); // to update the realm variable
 
         callback(null);
     });

src/collectd.js

@@ -0,0 +1,55 @@
+'use strict';
+
+exports = module.exports = {
+    addProfile,
+    removeProfile
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    debug = require('debug')('collectd'),
+    fs = require('fs'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js');
+
+const CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh');
+
+function addProfile(name, profile, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof profile, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const configFilePath = path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`);
+
+    // skip restarting collectd if the profile already exists with the same contents
+    const currentProfile = safe.fs.readFileSync(configFilePath, 'utf8') || '';
+    if (currentProfile === profile) return callback(null);
+
+    fs.writeFile(configFilePath, profile, function (error) {
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
+
+        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}, function (error) {
+            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
+
+            callback(null);
+        });
+    });
+
+}
+
+function removeProfile(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`), function (error) {
+        if (error && error.code !== 'ENOENT') debug('Error removing collectd profile', error);
+
+        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}, function (error) {
+            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
+
+            callback(null);
+        });
+    });
+}

src/collectd/cloudron-backup.ejs

@@ -0,0 +1,9 @@
+<Plugin python>
+  <Module du>
+    <Path>
+        Instance "cloudron-backup"
+        Dir "<%= backupDir %>"
+    </Path>
+  </Module>
+</Plugin>
+

src/constants.js

@@ -32,9 +32,7 @@ exports = module.exports = {
 
     NGINX_DEFAULT_CONFIG_FILE_NAME: 'default.conf',
 
-    GHOST_USER_FILE: '/tmp/cloudron_ghost.json',
-
-    DEFAULT_TOKEN_EXPIRATION: 7 * 24 * 60 * 60 * 1000, // 1 week
+    DEFAULT_TOKEN_EXPIRATION: 365 * 24 * 60 * 60 * 1000, // 1 year
 
     DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024), // see also client.js
 
@@ -47,6 +45,10 @@ exports = module.exports = {
     CLOUDRON: CLOUDRON,
     TEST: TEST,
 
+    SUPPORT_EMAIL: 'support@cloudron.io',
+
+    FOOTER: '&copy; 2020  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
+
     VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '4.2.0-test'
 };
 

src/cron.js

@@ -12,18 +12,19 @@ var appHealthMonitor = require('./apphealthmonitor.js'),
     apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
+    async = require('async'),
     auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     cloudron = require('./cloudron.js'),
     constants = require('./constants.js'),
     CronJob = require('cron').CronJob,
     debug = require('debug')('box:cron'),
-    disks = require('./disks.js'),
     dyndns = require('./dyndns.js'),
     eventlog = require('./eventlog.js'),
     janitor = require('./janitor.js'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
+    system = require('./system.js'),
     updater = require('./updater.js'),
     updateChecker = require('./updatechecker.js');
 
@@ -59,150 +60,141 @@ var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 function startJobs(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    var randomHourMinute = Math.floor(60*Math.random());
+    const randomMinute = Math.floor(60*Math.random());
     gJobs.alive = new CronJob({
-        cronTime: '00 ' + randomHourMinute + ' * * * *', // every hour on a random minute
+        cronTime: '00 ' + randomMinute + ' * * * *', // every hour on a random minute
         onTick: appstore.sendAliveStatus,
         start: true
     });
 
+    gJobs.systemChecks = new CronJob({
+        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
+        onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.diskSpaceChecker = new CronJob({
+        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
+        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.boxUpdateCheckerJob = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkBoxUpdates(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.appUpdateChecker = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkAppUpdates(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.cleanupTokens = new CronJob({
+        cronTime: '00 */30 * * * *', // every 30 minutes
+        onTick: janitor.cleanupTokens,
+        start: true
+    });
+
+    gJobs.cleanupBackups = new CronJob({
+        cronTime: '00 45 1,3,5,23 * * *', // every 6 hours. try not to overlap with ensureBackup job
+        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.cleanupEventlog = new CronJob({
+        cronTime: '00 */30 * * * *', // every 30 minutes
+        onTick: eventlog.cleanup,
+        start: true
+    });
+
+    gJobs.dockerVolumeCleaner = new CronJob({
+        cronTime: '00 00 */12 * * *', // every 12 hours
+        onTick: janitor.cleanupDockerVolumes,
+        start: true
+    });
+
+    gJobs.schedulerSync = new CronJob({
+        cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
+        onTick: scheduler.sync,
+        start: true
+    });
+
+    gJobs.certificateRenew = new CronJob({
+        cronTime: '00 00 */12 * * *', // every 12 hours
+        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.appHealthMonitor = new CronJob({
+        cronTime: '*/10 * * * * *', // every 10 seconds
+        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
+        start: true
+    });
+
     settings.getAll(function (error, allSettings) {
         if (error) return callback(error);
 
-        recreateJobs(allSettings[settings.TIME_ZONE_KEY]);
-        appAutoupdatePatternChanged(allSettings[settings.APP_AUTOUPDATE_PATTERN_KEY]);
-        boxAutoupdatePatternChanged(allSettings[settings.BOX_AUTOUPDATE_PATTERN_KEY]);
+        const tz = allSettings[settings.TIME_ZONE_KEY];
+        backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
+        appAutoupdatePatternChanged(allSettings[settings.APP_AUTOUPDATE_PATTERN_KEY], tz);
+        boxAutoupdatePatternChanged(allSettings[settings.BOX_AUTOUPDATE_PATTERN_KEY], tz);
         dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
 
         callback();
     });
 }
 
+// eslint-disable-next-line no-unused-vars
 function handleSettingsChanged(key, value) {
     assert.strictEqual(typeof key, 'string');
-
     // value is a variant
+
     switch (key) {
-    case settings.TIME_ZONE_KEY: recreateJobs(value); break;
-    case settings.APP_AUTOUPDATE_PATTERN_KEY: appAutoupdatePatternChanged(value); break;
-    case settings.BOX_AUTOUPDATE_PATTERN_KEY: boxAutoupdatePatternChanged(value); break;
-    case settings.DYNAMIC_DNS_KEY: dynamicDnsChanged(value); break;
-    default: break;
+    case settings.TIME_ZONE_KEY:
+    case settings.BACKUP_CONFIG_KEY:
+    case settings.APP_AUTOUPDATE_PATTERN_KEY:
+    case settings.BOX_AUTOUPDATE_PATTERN_KEY:
+    case settings.DYNAMIC_DNS_KEY:
+        debug('handleSettingsChanged: recreating all jobs');
+        async.series([
+            stopJobs,
+            startJobs
+        ], NOOP_CALLBACK);
+        break;
+    default:
+        break;
     }
 }
 
-function recreateJobs(tz) {
+function backupConfigChanged(value, tz) {
+    assert.strictEqual(typeof value, 'object');
     assert.strictEqual(typeof tz, 'string');
 
-    debug('Creating jobs with timezone %s', tz);
+    debug(`backupConfigChanged: interval ${value.intervalSecs} (${tz})`);
 
     if (gJobs.backup) gJobs.backup.stop();
+    let pattern;
+    if (value.intervalSecs <= 6 * 60 * 60) {
+        pattern = '00 00 1,7,13,19 * * *'; // no option but to backup in the middle of the day
+    } else {
+        pattern = '00 00 1,3,5,23 * * *'; // avoid middle of the day backups
+    }
+
     gJobs.backup = new CronJob({
-        cronTime: '00 00 */6 * * *', // check every 6 hours
+        cronTime: pattern,
         onTick: backups.ensureBackup.bind(null, auditSource.CRON, NOOP_CALLBACK),
         start: true,
         timeZone: tz
     });
-
-    if (gJobs.systemChecks) gJobs.systemChecks.stop();
-    gJobs.systemChecks = new CronJob({
-        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: cloudron.runSystemChecks,
-        start: true,
-        runOnInit: true, // run system check immediately
-        timeZone: tz
-    });
-
-    if (gJobs.diskSpaceChecker) gJobs.diskSpaceChecker.stop();
-    gJobs.diskSpaceChecker = new CronJob({
-        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: () => disks.checkDiskSpace(NOOP_CALLBACK),
-        start: true,
-        runOnInit: true, // run system check immediately
-        timeZone: tz
-    });
-
-    // randomized pattern per cloudron every hour
-    var randomMinute = Math.floor(60*Math.random());
-
-    if (gJobs.boxUpdateCheckerJob) gJobs.boxUpdateCheckerJob.stop();
-    gJobs.boxUpdateCheckerJob = new CronJob({
-        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
-        onTick: () => updateChecker.checkBoxUpdates(NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.appUpdateChecker) gJobs.appUpdateChecker.stop();
-    gJobs.appUpdateChecker = new CronJob({
-        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
-        onTick: () => updateChecker.checkAppUpdates(NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.cleanupTokens) gJobs.cleanupTokens.stop();
-    gJobs.cleanupTokens = new CronJob({
-        cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: janitor.cleanupTokens,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.cleanupBackups) gJobs.cleanupBackups.stop();
-    gJobs.cleanupBackups = new CronJob({
-        cronTime: '00 45 */6 * * *', // every 6 hours. try not to overlap with ensureBackup job
-        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.cleanupEventlog) gJobs.cleanupEventlog.stop();
-    gJobs.cleanupEventlog = new CronJob({
-        cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: eventlog.cleanup,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.dockerVolumeCleaner) gJobs.dockerVolumeCleaner.stop();
-    gJobs.dockerVolumeCleaner = new CronJob({
-        cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: janitor.cleanupDockerVolumes,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.schedulerSync) gJobs.schedulerSync.stop();
-    gJobs.schedulerSync = new CronJob({
-        cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
-        onTick: scheduler.sync,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.certificateRenew) gJobs.certificateRenew.stop();
-    gJobs.certificateRenew = new CronJob({
-        cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.appHealthMonitor) gJobs.appHealthMonitor.stop();
-    gJobs.appHealthMonitor = new CronJob({
-        cronTime: '*/10 * * * * *', // every 10 seconds
-        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
 }
 
-function boxAutoupdatePatternChanged(pattern) {
+function boxAutoupdatePatternChanged(pattern, tz) {
     assert.strictEqual(typeof pattern, 'string');
-    assert(gJobs.boxUpdateCheckerJob);
+    assert.strictEqual(typeof tz, 'string');
 
-    debug('Box auto update pattern changed to %s', pattern);
+    debug(`boxAutoupdatePatternChanged: pattern - ${pattern} (${tz})`);
 
     if (gJobs.boxAutoUpdater) gJobs.boxAutoUpdater.stop();
 
@@ -220,15 +212,15 @@ function boxAutoupdatePatternChanged(pattern) {
             }
         },
         start: true,
-        timeZone: gJobs.boxUpdateCheckerJob.cronTime.zone // hack
+        timeZone: tz
     });
 }
 
-function appAutoupdatePatternChanged(pattern) {
+function appAutoupdatePatternChanged(pattern, tz) {
     assert.strictEqual(typeof pattern, 'string');
-    assert(gJobs.boxUpdateCheckerJob);
+    assert.strictEqual(typeof tz, 'string');
 
-    debug('Apps auto update pattern changed to %s', pattern);
+    debug(`appAutoupdatePatternChanged: pattern ${pattern} (${tz})`);
 
     if (gJobs.appAutoUpdater) gJobs.appAutoUpdater.stop();
 
@@ -246,13 +238,12 @@ function appAutoupdatePatternChanged(pattern) {
             }
         },
         start: true,
-        timeZone: gJobs.boxUpdateCheckerJob.cronTime.zone // hack
+        timeZone: tz
     });
 }
 
 function dynamicDnsChanged(enabled) {
     assert.strictEqual(typeof enabled, 'boolean');
-    assert(gJobs.boxUpdateCheckerJob);
 
     debug('Dynamic DNS setting changed to %s', enabled);
 
@@ -260,8 +251,7 @@ function dynamicDnsChanged(enabled) {
         gJobs.dynamicDns = new CronJob({
             cronTime: '5 * * * * *',    // we only update the records if the ip has changed.
             onTick: dyndns.sync.bind(null, auditSource.CRON, NOOP_CALLBACK),
-            start: true,
-            timeZone: gJobs.boxUpdateCheckerJob.cronTime.zone // hack
+            start: true
         });
     } else {
         if (gJobs.dynamicDns) gJobs.dynamicDns.stop();

src/custom.js

@@ -1,66 +0,0 @@
-'use strict';
-
-let debug = require('debug')('box:custom'),
-    lodash = require('lodash'),
-    paths = require('./paths.js'),
-    safe = require('safetydance'),
-    yaml = require('js-yaml');
-
-exports = module.exports = {
-    uiSpec: uiSpec,
-    spec: spec
-};
-
-const DEFAULT_SPEC = {
-    appstore: {
-        blacklist: [],
-        whitelist: null // null imples, not set. this is an object and not an array
-    },
-    backups: {
-        configurable: true
-    },
-    domains: {
-        dynamicDns: true,
-        changeDashboardDomain: true
-    },
-    subscription: {
-        configurable: true
-    },
-    support: {
-        email: 'support@cloudron.io',
-        remoteSupport: true,
-        ticketFormBody:
-            'Use this form to open support tickets. You can also write directly to [support@cloudron.io](mailto:support@cloudron.io).\n\n'
-            + '* [Knowledge Base & App Docs](https://cloudron.io/documentation/apps/?support_view)\n'
-            + '* [Custom App Packaging & API](https://cloudron.io/developer/packaging/?support_view)\n'
-            + '* [Forum](https://forum.cloudron.io/)\n\n',
-        submitTickets: true
-    },
-    alerts: {
-        email: '',
-        notifyCloudronAdmins: true
-    },
-    footer: {
-        body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
-    }
-};
-
-const gSpec = (function () {
-    try {
-        if (!safe.fs.existsSync(paths.CUSTOM_FILE)) return DEFAULT_SPEC;
-        const c = yaml.safeLoad(safe.fs.readFileSync(paths.CUSTOM_FILE, 'utf8'));
-        return lodash.merge({}, DEFAULT_SPEC, c);
-    } catch (e) {
-        debug(`Error loading features file from ${paths.CUSTOM_FILE} : ${e.message}`);
-        return DEFAULT_SPEC;
-    }
-})();
-
-// flags sent to the UI. this is separate because we have values that are secret to the backend
-function uiSpec() {
-    return gSpec;
-}
-
-function spec() {
-    return gSpec;
-}

src/database.js

@@ -14,6 +14,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
+    BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
     constants = require('./constants.js'),
     mysql = require('mysql'),
@@ -103,16 +104,13 @@ function clear(callback) {
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name,
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name);
 
-    async.series([
-        child_process.exec.bind(null, cmd),
-        require('./clients.js').addDefaultClients.bind(null, 'https://admin-localhost')
-    ], callback);
+    child_process.exec(cmd, callback);
 }
 
 function beginTransaction(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (gConnectionPool === null) return callback(new Error('No database connection pool.'));
+    if (gConnectionPool === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No database connection pool.'));
 
     gConnectionPool.getConnection(function (error, connection) {
         if (error) {
@@ -156,7 +154,7 @@ function query() {
     var callback = args[args.length - 1];
     assert.strictEqual(typeof callback, 'function');
 
-    if (gDefaultConnection === null) return callback(new Error('No connection to database'));
+    if (gDefaultConnection === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No connection to database'));
 
     args[args.length -1 ] = function (error, result) {
         if (error && error.fatal) {
@@ -203,7 +201,11 @@ function exportToFile(file, callback) {
     assert.strictEqual(typeof file, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
+    // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
+    // this option must not be set in production cloudrons which still use the old mysqldump
+    const disableColStats = (constants.TEST && process.env.DESKTOP_SESSION !== 'ubuntu') ? '--column-statistics=0' : '';
+
+    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${disableColStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
 
     child_process.exec(cmd, callback);
 }

src/dns/cloudflare.js

@@ -48,15 +48,29 @@ function translateRequestError(result, callback) {
     callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
 }
 
+function createRequest(method, url, dnsConfig) {
+    assert.strictEqual(typeof method, 'string');
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof dnsConfig, 'object');
+
+    let request = superagent(method, url)
+        .timeout(30 * 1000);
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        request.set('X-Auth-Key', dnsConfig.token).set('X-Auth-Email', dnsConfig.email);
+    } else {
+        request.set('Authorization', 'Bearer ' + dnsConfig.token);
+    }
+
+    return request;
+}
+
 function getZoneByName(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active')
-        .set('X-Auth-Key', dnsConfig.token)
-        .set('X-Auth-Email', dnsConfig.email)
-        .timeout(30 * 1000)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active', dnsConfig)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -74,11 +88,8 @@ function getDnsRecords(dnsConfig, zoneId, fqdn, type, callback) {
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-        .set('X-Auth-Key',dnsConfig.token)
-        .set('X-Auth-Email',dnsConfig.email)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
         .query({ type: type, name: fqdn })
-        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -132,11 +143,8 @@ function upsert(domainObject, location, type, values, callback) {
                 if (i >= dnsRecords.length) { // create a new record
                     debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: false`);
 
-                    superagent.post(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('POST', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             if (error && !error.response) return iteratorCallback(error);
                             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, iteratorCallback);
@@ -148,11 +156,8 @@ function upsert(domainObject, location, type, values, callback) {
 
                     debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
 
-                    superagent.put(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id)
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('PUT', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id, dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             ++i; // increment, as we have consumed the record
 
@@ -217,10 +222,7 @@ function del(domainObject, location, type, values, callback) {
             if (tmp.length === 0) return callback(null);
 
             async.eachSeries(tmp, function (record, callback) {
-                superagent.del(CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id)
-                    .set('X-Auth-Key', dnsConfig.token)
-                    .set('X-Auth-Email', dnsConfig.email)
-                    .timeout(30 * 1000)
+                createRequest('DELETE', CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id, dnsConfig)
                     .end(function (error, result) {
                         if (error && !error.response) return callback(error);
                         if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -277,14 +279,20 @@ function verifyDnsConfig(domainObject, callback) {
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
+    // token can be api token or global api key
     if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
-    if (!dnsConfig.email || typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    if (dnsConfig.tokenType !== 'GlobalApiKey' && dnsConfig.tokenType !== 'ApiToken')  return callback(new BoxError(BoxError.BAD_FIELD, 'tokenType is required', { field: 'tokenType' }));
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        if ('email' in dnsConfig && typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    }
 
     const ip = '127.0.0.1';
 
     var credentials = {
         token: dnsConfig.token,
-        email: dnsConfig.email
+        tokenType: dnsConfig.tokenType,
+        email: dnsConfig.email || null
     };
 
     if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here

src/dns/godaddy.js

@@ -126,7 +126,7 @@ function del(domainObject, location, type, values, callback) {
 
     debug(`get: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, new Error('Record deletion is not supported by GoDaddy API')));
+    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Record deletion is not supported by GoDaddy API'));
 
     // check if the record exists at all so that we don't insert the "Dead" record for no reason
     get(domainObject, location, type, function (error, values) {

src/dns/interface.js

@@ -17,6 +17,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
     util = require('util');
 
 function removePrivateFields(domainObject) {
@@ -24,6 +25,7 @@ function removePrivateFields(domainObject) {
     return domainObject;
 }
 
+// eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
     // in-place injection of tokens and api keys which came in with domains.SECRET_PLACEHOLDER
 }
@@ -37,7 +39,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     // Result: none
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upsert is not implemented'));
 }
 
 function get(domainObject, location, type, callback) {
@@ -48,7 +50,7 @@ function get(domainObject, location, type, callback) {
 
     // Result: Array of matching DNS records in string format
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'get is not implemented'));
 }
 
 function del(domainObject, location, type, values, callback) {
@@ -60,7 +62,7 @@ function del(domainObject, location, type, values, callback) {
 
     // Result: none
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'del is not implemented'));
 }
 
 function wait(domainObject, location, type, value, options, callback) {
@@ -80,5 +82,5 @@ function verifyDnsConfig(domainObject, callback) {
 
     // Result: dnsConfig object
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'verifyDnsConfig is not implemented'));
 }

src/dns/linode.js

@@ -0,0 +1,309 @@
+'use strict';
+
+exports = module.exports = {
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
+};
+
+let async = require('async'),
+    assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    debug = require('debug')('box:dns/linode'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
+    superagent = require('superagent'),
+    util = require('util'),
+    waitForDns = require('./waitfordns.js');
+
+const LINODE_ENDPOINT = 'https://api.linode.com/v4';
+
+function formatError(response) {
+    return util.format('Linode DNS error [%s] %j', response.statusCode, response.body);
+}
+
+function removePrivateFields(domainObject) {
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
+    return domainObject;
+}
+
+function injectPrivateFields(newConfig, currentConfig) {
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+}
+
+function getZoneId(dnsConfig, zoneName, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // returns 100 at a time
+    superagent.get(`${LINODE_ENDPOINT}/domains`)
+        .set('Authorization', 'Bearer ' + dnsConfig.token)
+        .timeout(30 * 1000)
+        .retry(5)
+        .end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
+            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+            if (!Array.isArray(result.body.data)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
+
+            const zone = result.body.data.find(d => d.domain === zoneName);
+
+            if (!zone || !zone.id) return callback(new BoxError(BoxError.NOT_FOUND, 'Zone not found'));
+
+            debug(`getZoneId: zone id of ${zoneName} is ${zone.id}`);
+
+            callback(null, zone.id);
+        });
+}
+
+function getZoneRecords(dnsConfig, zoneName, name, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`getInternal: getting dns records of ${zoneName} with ${name} and type ${type}`);
+
+    getZoneId(dnsConfig, zoneName, function (error, zoneId) {
+        if (error) return callback(error);
+
+        let page = 0, more = false;
+        let records = [];
+
+        async.doWhilst(function (iteratorDone) {
+            const url = `${LINODE_ENDPOINT}/domains/${zoneId}/records?page=${++page}`;
+
+            superagent.get(url)
+                .set('Authorization', 'Bearer ' + dnsConfig.token)
+                .timeout(30 * 1000)
+                .retry(5)
+                .end(function (error, result) {
+                    if (error && !error.response) return iteratorDone(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                    if (result.statusCode === 404) return iteratorDone(new BoxError(BoxError.NOT_FOUND, formatError(result)));
+                    if (result.statusCode === 403 || result.statusCode === 401) return iteratorDone(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
+                    if (result.statusCode !== 200) return iteratorDone(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                    records = records.concat(result.body.data.filter(function (record) {
+                        return (record.type === type && record.name === name);
+                    }));
+
+                    more = result.body.page !== result.body.pages;
+
+                    iteratorDone();
+                });
+        }, function () { return more; }, function (error) {
+            debug('getZoneRecords:', error, JSON.stringify(records));
+
+            if (error) return callback(error);
+
+            callback(null, { zoneId, records });
+        });
+    });
+}
+
+function get(domainObject, location, type, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '';
+
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, { records }) {
+        if (error) return callback(error);
+
+        var tmp = records.map(function (record) { return record.target; });
+
+        debug('get: %j', tmp);
+
+        return callback(null, tmp);
+    });
+}
+
+function upsert(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '';
+
+    debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
+
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, { zoneId, records }) {
+        if (error) return callback(error);
+
+        let i = 0, recordIds = []; // used to track available records to update instead of create
+
+        async.eachSeries(values, function (value, iteratorCallback) {
+            let data = {
+                type: type,
+                ttl_sec: 300 // lowest
+            };
+
+            if (type === 'MX') {
+                data.priority = parseInt(value.split(' ')[0], 10);
+                data.target = value.split(' ')[1];
+            } else if (type === 'TXT') {
+                data.target = value.replace(/^"(.*)"$/, '$1'); // strip any double quotes
+            } else {
+                data.target = value;
+            }
+
+            if (i >= records.length) {
+                data.name = name; // only set for new records
+
+                superagent.post(`${LINODE_ENDPOINT}/domains/${zoneId}/records`)
+                    .set('Authorization', 'Bearer ' + dnsConfig.token)
+                    .send(data)
+                    .timeout(30 * 1000)
+                    .retry(5)
+                    .end(function (error, result) {
+                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
+                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
+                        if (result.statusCode !== 200) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                        recordIds.push(result.body.id);
+
+                        return iteratorCallback(null);
+                    });
+            } else {
+                superagent.put(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${records[i].id}`)
+                    .set('Authorization', 'Bearer ' + dnsConfig.token)
+                    .send(data)
+                    .timeout(30 * 1000)
+                    .retry(5)
+                    .end(function (error, result) {
+                        // increment, as we have consumed the record
+                        ++i;
+
+                        if (error && !error.response) return iteratorCallback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                        if (result.statusCode === 400) return iteratorCallback(new BoxError(BoxError.BAD_FIELD, formatError(result)));
+                        if (result.statusCode === 403 || result.statusCode === 401) return iteratorCallback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
+                        if (result.statusCode !== 200) return iteratorCallback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                        recordIds.push(result.body.id);
+
+                        return iteratorCallback(null);
+                    });
+            }
+        }, function (error) {
+            if (error) return callback(error);
+
+            debug('upsert: completed with recordIds:%j', recordIds);
+
+            callback();
+        });
+    });
+}
+
+function del(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '';
+
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, { zoneId, records }) {
+        if (error) return callback(error);
+
+        if (records.length === 0) return callback(null);
+
+        var tmp = records.filter(function (record) { return values.some(function (value) { return value === record.target; }); });
+
+        debug('del: %j', tmp);
+
+        if (tmp.length === 0) return callback(null);
+
+        // FIXME we only handle the first one currently
+
+        superagent.del(`${LINODE_ENDPOINT}/domains/${zoneId}/records/${tmp[0].id}`)
+            .set('Authorization', 'Bearer ' + dnsConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 404) return callback(null);
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.ACCESS_DENIED, formatError(result)));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                debug('del: done');
+
+                return callback(null);
+            });
+    });
+}
+
+function wait(domainObject, location, type, value, options, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+    assert.strictEqual(typeof callback, 'function');
+
+    const fqdn = domains.fqdn(location, domainObject);
+
+    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+}
+
+function verifyDnsConfig(domainObject, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+
+    const ip = '127.0.0.1';
+
+    var credentials = {
+        token: dnsConfig.token
+    };
+
+    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+
+    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
+        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
+        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+
+        if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) {
+            debug('verifyDnsConfig: %j does not contains DO NS', nameservers);
+            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode', { field: 'nameservers' }));
+        }
+
+        const location = 'cloudrontestdns';
+
+        upsert(domainObject, location, 'A', [ ip ], function (error) {
+            if (error) return callback(error);
+
+            debug('verifyDnsConfig: Test A record added');
+
+            del(domainObject, location, 'A', [ ip ], function (error) {
+                if (error) return callback(error);
+
+                debug('verifyDnsConfig: Test A record removed again');
+
+                callback(null, credentials);
+            });
+        });
+    });
+}

src/docker.js

@@ -1,8 +1,6 @@
 'use strict';
 
 exports = module.exports = {
-    connection: connectionInstance(),
-
     testRegistryConfig: testRegistryConfig,
     setRegistryConfig: setRegistryConfig,
     injectPrivateFields: injectPrivateFields,
@@ -16,6 +14,7 @@ exports = module.exports = {
     downloadImage: downloadImage,
     createContainer: createContainer,
     startContainer: startContainer,
+    restartContainer: restartContainer,
     stopContainer: stopContainer,
     stopContainerByName: stopContainer,
     stopContainers: stopContainers,
@@ -27,6 +26,7 @@ exports = module.exports = {
     getContainerIdByIp: getContainerIdByIp,
     inspect: inspect,
     inspectByName: inspect,
+    execContainer: execContainer,
     getEvents: getEvents,
     memoryUsage: memoryUsage,
     createVolume: createVolume,
@@ -34,20 +34,14 @@ exports = module.exports = {
     clearVolume: clearVolume
 };
 
-// timeout is optional
-function connectionInstance(timeout) {
-    var Docker = require('dockerode');
-    var docker = new Docker({ socketPath: '/var/run/docker.sock', timeout: timeout });
-    return docker;
-}
-
 var addons = require('./addons.js'),
     async = require('async'),
     assert = require('assert'),
     BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
     constants = require('./constants.js'),
-    debug = require('debug')('box:docker.js'),
+    debug = require('debug')('box:docker'),
+    Docker = require('dockerode'),
     path = require('path'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
@@ -58,6 +52,9 @@ var addons = require('./addons.js'),
 const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
     MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');
 
+const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
+const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
+
 function debugApp(app) {
     assert(typeof app === 'object');
 
@@ -68,8 +65,7 @@ function testRegistryConfig(auth, callback) {
     assert.strictEqual(typeof auth, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-    docker.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
+    gConnection.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
         if (error) return callback(new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' }));
 
         callback();
@@ -108,13 +104,14 @@ function ping(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     // do not let the request linger
-    var docker = connectionInstance(1000);
+    const connection = new Docker({ socketPath: DOCKER_SOCKET_PATH, timeout: 1000 });
 
-    docker.ping(function (error, result) {
+    connection.ping(function (error, result) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
-        if (result !== 'OK') return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon'));
+        if (Buffer.isBuffer(result) && result.toString('utf8') === 'OK') return callback(null); // sometimes it returns buffer
+        if (result === 'OK') return callback(null);
 
-        callback(null);
+        callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon'));
     });
 }
 
@@ -140,15 +137,14 @@ function getRegistryConfig(image, callback) {
 }
 
 function pullImage(manifest, callback) {
-    var docker = exports.connection;
-
     getRegistryConfig(manifest.dockerImage, function (error, authConfig) {
         if (error) return callback(error);
 
         debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${authConfig ? 'yes' : 'no'}`);
 
-        docker.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
-            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to pull image. Please check the network or if the image needs authentication. statusCode: ' + error.statusCode));
+        gConnection.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
+            if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${manifest.dockerImage}. message: ${error.message} statusCode: ${error.statusCode}`));
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`));
 
             // https://github.com/dotcloud/docker/issues/1074 says each status message
             // is emitted as a chunk
@@ -185,14 +181,10 @@ function downloadImage(manifest, callback) {
 
     var attempt = 1;
 
-    async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
+    async.retry({ times: 10, interval: 5000, errorFilter: e => e.reason !== BoxError.NOT_FOUND }, function (retryCallback) {
         debug('Downloading image %s. attempt: %s', manifest.dockerImage, attempt++);
 
-        pullImage(manifest, function (error) {
-            if (error) console.error(error);
-
-            retryCallback(error);
-        });
+        pullImage(manifest, retryCallback);
     }, callback);
 }
 
@@ -203,8 +195,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection,
-        isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers
+    let isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers
 
     var manifest = app.manifest;
     var exposedPorts = {}, dockerPortBindings = { };
@@ -303,7 +294,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                     'Name': isAppContainer ? 'unless-stopped' : 'no',
                     'MaximumRetryCount': 0
                 },
-                CpuShares: 512, // relative to 1024 for system processes
+                CpuShares: app.cpuShares,
                 VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ],
                 NetworkMode: 'cloudron', // user defined bridge network
                 Dns: ['172.18.0.1'], // use internal dns
@@ -330,7 +321,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
 
         debugApp(app, 'Creating container for %s', app.manifest.dockerImage);
 
-        docker.createContainer(containerOptions, function (error, container) {
+        gConnection.createContainer(containerOptions, function (error, container) {
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
             callback(null, container);
@@ -346,15 +337,29 @@ function startContainer(containerId, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
     debug('Starting container %s', containerId);
 
     container.start(function (error) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
         if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
-        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); // 304 means already started
+
+        return callback(null);
+    });
+}
+
+function restartContainer(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = gConnection.getContainer(containerId);
+    debug('Restarting container %s', containerId);
+
+    container.restart(function (error) {
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+        if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
+        if (error && error.statusCode !== 204) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         return callback(null);
     });
@@ -369,8 +374,7 @@ function stopContainer(containerId, callback) {
         return callback();
     }
 
-    var docker = exports.connection;
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
     debug('Stopping container %s', containerId);
 
     var options = {
@@ -400,8 +404,7 @@ function deleteContainer(containerId, callback) {
 
     if (containerId === null) return callback(null);
 
-    var docker = exports.connection;
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
 
     var removeOptions = {
         force: true, // kill container if it's running
@@ -425,14 +428,12 @@ function deleteContainers(appId, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
     debug('deleting containers of %s', appId);
 
     let labels = [ 'appId=' + appId ];
     if (options.managedOnly) labels.push('isCloudronManaged=true');
 
-    docker.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) {
+    gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         async.eachSeries(containers, function (container, iteratorDone) {
@@ -445,11 +446,9 @@ function stopContainers(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
+    debug('Stopping containers of %s', appId);
 
-    debug('stopping containers of %s', appId);
-
-    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+    gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         async.eachSeries(containers, function (container, iteratorDone) {
@@ -465,8 +464,6 @@ function deleteImage(manifest, callback) {
     var dockerImage = manifest ? manifest.dockerImage : null;
     if (!dockerImage) return callback(null);
 
-    var docker = exports.connection;
-
     var removeOptions = {
         force: false, // might be shared with another instance of this app
         noprune: false // delete untagged parents
@@ -475,7 +472,7 @@ function deleteImage(manifest, callback) {
     // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
     // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
     // not created anymore after https://github.com/docker/docker/pull/10571
-    docker.getImage(dockerImage).remove(removeOptions, function (error) {
+    gConnection.getImage(dockerImage).remove(removeOptions, function (error) {
         if (error && error.statusCode === 400) return callback(null); // invalid image format. this can happen if user installed with a bad --docker-image
         if (error && error.statusCode === 404) return callback(null); // not found
         if (error && error.statusCode === 409) return callback(null); // another container using the image
@@ -493,10 +490,8 @@ function getContainerIdByIp(ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
-    docker.getNetwork('cloudron').inspect(function (error, bridge) {
-        if (error && error.statusCode === 404) return callback(new Error('Unable to find the cloudron network'));
+    gConnection.getNetwork('cloudron').inspect(function (error, bridge) {
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to find the cloudron network'));
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         var containerId;
@@ -506,7 +501,7 @@ function getContainerIdByIp(ip, callback) {
                 break;
             }
         }
-        if (!containerId) return callback(new Error('No container with that ip'));
+        if (!containerId) return callback(new BoxError(BoxError.DOCKER_ERROR, 'No container with that ip'));
 
         callback(null, containerId);
     });
@@ -516,7 +511,7 @@ function inspect(containerId, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = exports.connection.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
 
     container.inspect(function (error, result) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -526,13 +521,38 @@ function inspect(containerId, callback) {
     });
 }
 
+function execContainer(containerId, options, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = gConnection.getContainer(containerId);
+
+    container.exec(options.execOptions, function (error, exec) {
+        if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running
+        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+
+        exec.start(options.startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+
+            if (options.rows && options.columns) {
+                // there is a race where resizing too early results in a 404 "no such exec"
+                // https://git.cloudron.io/cloudron/box/issues/549
+                setTimeout(function () {
+                    exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
+                }, 2000);
+            }
+
+            callback(null, stream);
+        });
+    });
+}
+
 function getEvents(options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
-    docker.getEvents(options, function (error, stream) {
+    gConnection.getEvents(options, function (error, stream) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         callback(null, stream);
@@ -543,7 +563,7 @@ function memoryUsage(containerId, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = exports.connection.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
 
     container.stats({ stream: false }, function (error, result) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -559,8 +579,6 @@ function createVolume(app, name, volumeDataDir, callback) {
     assert.strictEqual(typeof volumeDataDir, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
     const volumeOptions = {
         Name: name,
         Driver: 'local',
@@ -577,9 +595,9 @@ function createVolume(app, name, volumeDataDir, callback) {
 
     // requires sudo because the path can be outside appsdata
     shell.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}, function (error) {
-        if (error) return callback(new Error(`Error creating app data dir: ${error.message}`));
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`));
 
-        docker.createVolume(volumeOptions, function (error) {
+        gConnection.createVolume(volumeOptions, function (error) {
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
             callback();
@@ -593,8 +611,7 @@ function clearVolume(app, name, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-    let volume = docker.getVolume(name);
+    let volume = gConnection.getVolume(name);
     volume.inspect(function (error, v) {
         if (error && error.statusCode === 404) return callback();
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
@@ -614,9 +631,7 @@ function removeVolume(app, name, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
-    let volume = docker.getVolume(name);
+    let volume = gConnection.getVolume(name);
     volume.remove(function (error) {
         if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume of ${app.id} ${error.message}`));
 
@@ -627,9 +642,7 @@ function removeVolume(app, name, callback) {
 function info(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
-    docker.info(function (error, result) {
+    gConnection.info(function (error, result) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker'));
 
         callback(null, result);

src/dockerproxy.js

@@ -23,11 +23,6 @@ var apps = require('./apps.js'),
 var gHttpServer = null;
 
 function authorizeApp(req, res, next) {
-    // TODO add here some authorization
-    // - block apps not using the docker addon
-    // - block calls regarding platform containers
-    // - only allow managing and inspection of containers belonging to the app
-
     // make the tests pass for now
     if (constants.TEST) {
         req.app = { id: 'testappid' };
@@ -64,6 +59,8 @@ function attachDockerRequest(req, res, next) {
         dockerResponse.pipe(res, { end: true });
     });
 
+    req.dockerRequest.on('error', () => {}); // abort() throws
+
     next();
 }
 
@@ -74,22 +71,21 @@ function containersCreate(req, res, next) {
     safe.set(req.body, 'Labels',  _.extend({ }, safe.query(req.body, 'Labels'), { appId: req.app.id, isCloudronManaged: String(false) }));    // overwrite the app id to track containers of an app
     safe.set(req.body, 'HostConfig.LogConfig', { Type: 'syslog', Config: { 'tag': req.app.id, 'syslog-address': 'udp://127.0.0.1:2514', 'syslog-format': 'rfc5424' }});
 
-    const appDataDir = path.join(paths.APPS_DATA_DIR, req.app.id, 'data'),
-        dockerDataDir = path.join(paths.APPS_DATA_DIR, req.app.id, 'docker');
+    const appDataDir = path.join(paths.APPS_DATA_DIR, req.app.id, 'data');
 
-    debug('Original volume binds:', req.body.HostConfig.Binds);
+    debug('Original bind mounts:', req.body.HostConfig.Binds);
 
     let binds = [];
     for (let bind of (req.body.HostConfig.Binds || [])) {
-        if (bind.startsWith(appDataDir)) binds.push(bind); // eclipse will inspect docker to find out the host folders and pass that to child containers
-        else if (bind.startsWith('/app/data')) binds.push(bind.replace(new RegExp('^/app/data'), appDataDir));
-        else binds.push(`${dockerDataDir}/${bind}`);
+        if (!bind.startsWith('/app/data/')) {
+            req.dockerRequest.abort();
+            return next(new HttpError(400, 'Binds must be under /app/data/'));
+        }
+
+        binds.push(bind.replace(new RegExp('^/app/data/'), appDataDir + '/'));
     }
 
-    // cleanup the paths from potential double slashes
-    binds = binds.map(function (bind) { return bind.replace(/\/+/g, '/'); });
-
-    debug('Rewritten volume binds:', binds);
+    debug('Rewritten bind mounts:', binds);
     safe.set(req.body, 'HostConfig.Binds', binds);
 
     let plainBody = JSON.stringify(req.body);
@@ -117,6 +113,9 @@ function start(callback) {
     assert(gHttpServer === null, 'Already started');
 
     let json = middleware.json({ strict: true });
+
+    // we protect container create as the app/admin can otherwise mount random paths (like the ghost file)
+    // protected other paths is done by preventing install/exec access of apps using docker addon
     let router = new express.Router();
     router.post('/:version/containers/create', containersCreate);
 
@@ -137,7 +136,7 @@ function start(callback) {
         .use(middleware.lastMile());
 
     gHttpServer = http.createServer(proxyServer);
-    gHttpServer.listen(constants.DOCKER_PROXY_PORT, '0.0.0.0', callback);
+    gHttpServer.listen(constants.DOCKER_PROXY_PORT, '172.18.0.1', callback);
 
     // Overwrite the default 2min request timeout. This is required for large builds for example
     gHttpServer.setTimeout(60 * 60 * 1000);

src/domaindb.js

@@ -16,7 +16,7 @@ var assert = require('assert'),
     database = require('./database.js'),
     safe = require('safetydance');
 
-var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'locked' ].join(',');
+var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson' ].join(',');
 
 function postProcess(data) {
     data.config = safe.JSON.parse(data.configJson);
@@ -24,8 +24,6 @@ function postProcess(data) {
     delete data.configJson;
     delete data.tlsConfigJson;
 
-    data.locked = !!data.locked; // make it bool
-
     return data;
 }
 
@@ -53,16 +51,21 @@ function getAll(callback) {
     });
 }
 
-function add(name, domain, callback) {
+function add(name, data, callback) {
     assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof domain, 'object');
-    assert.strictEqual(typeof domain.zoneName, 'string');
-    assert.strictEqual(typeof domain.provider, 'string');
-    assert.strictEqual(typeof domain.config, 'object');
-    assert.strictEqual(typeof domain.tlsConfig, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof data.zoneName, 'string');
+    assert.strictEqual(typeof data.provider, 'string');
+    assert.strictEqual(typeof data.config, 'object');
+    assert.strictEqual(typeof data.tlsConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson) VALUES (?, ?, ?, ?, ?)', [ name, domain.zoneName, domain.provider, JSON.stringify(domain.config), JSON.stringify(domain.tlsConfig) ], function (error) {
+    let queries = [
+        { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson) VALUES (?, ?, ?, ?, ?)', args: [ name, data.zoneName, data.provider, JSON.stringify(data.config), JSON.stringify(data.tlsConfig) ] },
+        { query: 'INSERT INTO mail (domain, dkimSelector) VALUES (?, ?)', args: [ name, data.dkimSelector || 'cloudron' ] },
+    ];
+
+    database.transaction(queries, function (error) {
         if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
@@ -102,10 +105,22 @@ function del(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('DELETE FROM domains WHERE domain=?', [ domain ], function (error, result) {
-        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') return callback(new BoxError(BoxError.CONFLICT));
+    let queries = [
+        { query: 'DELETE FROM mail WHERE domain = ?', args: [ domain ] },
+        { query: 'DELETE FROM domains WHERE domain = ?', args: [ domain ] },
+    ];
+
+    database.transaction(queries, function (error, results) {
+        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') {
+            if (error.message.indexOf('apps_mailDomain_constraint') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by an app or the mailbox of an app. Check the domains of apps and the Email section of each app.'));
+            if (error.message.indexOf('subdomains') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more app(s).'));
+            if (error.message.indexOf('mail') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more mailboxes. Delete them first in the Email view.'));
+
+            return callback(new BoxError(BoxError.CONFLICT, error.message));
+        }
+
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
+        if (results[1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
 
         callback(null);
     });

src/domains.js

@@ -40,6 +40,7 @@ var assert = require('assert'),
     debug = require('debug')('box:domains'),
     domaindb = require('./domaindb.js'),
     eventlog = require('./eventlog.js'),
+    mail = require('./mail.js'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
@@ -48,6 +49,8 @@ var assert = require('assert'),
     util = require('util'),
     _ = require('underscore');
 
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
 // choose which subdomain backend we use for test purpose we use route53
 function api(provider) {
     assert.strictEqual(typeof provider, 'string');
@@ -60,6 +63,7 @@ function api(provider) {
     case 'digitalocean': return require('./dns/digitalocean.js');
     case 'gandi': return require('./dns/gandi.js');
     case 'godaddy': return require('./dns/godaddy.js');
+    case 'linode': return require('./dns/linode.js');
     case 'namecom': return require('./dns/namecom.js');
     case 'namecheap': return require('./dns/namecheap.js');
     case 'noop': return require('./dns/noop.js');
@@ -86,9 +90,9 @@ function verifyDnsConfig(dnsConfig, domain, zoneName, provider, callback) {
 
     const domainObject = { config: dnsConfig, domain: domain, zoneName: zoneName };
     api(provider).verifyDnsConfig(domainObject, function (error, result) {
-        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, 'Incorrect configuration. Access denied'));
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, 'Zone not found'));
-        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, 'Configuration error: ' + error.message));
+        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`));
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`));
+        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`));
         if (error) return callback(error);
 
         result.hyphenatedSubdomains = !!dnsConfig.hyphenatedSubdomains;
@@ -170,7 +174,7 @@ function add(domain, data, auditSource, callback) {
     assert.strictEqual(typeof data.tlsConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
+    let { zoneName, provider, config, fallbackCertificate, tlsConfig, dkimSelector } = data;
 
     if (!tld.isValid(domain)) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
     if (domain.endsWith('.')) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
@@ -193,10 +197,12 @@ function add(domain, data, auditSource, callback) {
     let error = validateTlsConfig(tlsConfig, provider);
     if (error) return callback(error);
 
+    if (!dkimSelector) dkimSelector = 'cloudron-' + settings.adminDomain().replace(/\./g, '');
+
     verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
         if (error) return callback(error);
 
-        domaindb.add(domain, { zoneName: zoneName, provider: provider, config: sanitizedConfig, tlsConfig: tlsConfig }, function (error) {
+        domaindb.add(domain, { zoneName, provider, config: sanitizedConfig, tlsConfig, dkimSelector }, function (error) {
             if (error) return callback(error);
 
             reverseProxy.setFallbackCertificate(domain, fallbackCertificate, function (error) {
@@ -204,6 +210,8 @@ function add(domain, data, auditSource, callback) {
 
                 eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
 
+                mail.onDomainAdded(domain, NOOP_CALLBACK);
+
                 callback();
             });
         });
@@ -215,14 +223,14 @@ function get(domain, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     domaindb.get(domain, function (error, result) {
-        // TODO try to find subdomain entries maybe based on zoneNames or so
         if (error) return callback(error);
 
         reverseProxy.getFallbackCertificate(domain, function (_, bundle) { // never returns an error
             var cert = safe.fs.readFileSync(bundle.certFilePath, 'utf-8');
             var key = safe.fs.readFileSync(bundle.keyFilePath, 'utf-8');
 
-            if (!cert || !key) return callback(new BoxError(BoxError.FS_ERROR, 'unable to read certificates from disk'));
+            // do not error here. otherwise, there is no way to fix things up from the UI
+            if (!cert || !key) debug(`Unable to read fallback certificates of ${domain} from disk`);
 
             result.fallbackCertificate = { cert: cert, key: key };
 
@@ -253,6 +261,8 @@ function update(domain, data, auditSource, callback) {
 
     let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
 
+    if (settings.isDemo() && (domain === settings.adminDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+
     domaindb.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
@@ -311,6 +321,8 @@ function del(domain, auditSource, callback) {
 
         eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
 
+        mail.onDomainRemoved(domain, NOOP_CALLBACK);
+
         return callback(null);
     });
 }
@@ -434,19 +446,22 @@ function waitForDnsRecord(location, domain, type, value, options, callback) {
     get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
+        // linode DNS takes ~15mins
+        if (!options.interval) options.interval = domainObject.provider === 'linode' ? 20000 : 5000;
+
         api(domainObject.provider).wait(domainObject, location, type, value, options, callback);
     });
 }
 
 // removes all fields that are strictly private and should never be returned by API calls
 function removePrivateFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate', 'locked');
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate');
     return api(result.provider).removePrivateFields(result);
 }
 
 // removes all fields that are not accessible by a normal user
 function removeRestrictedFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'locked');
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider');
 
     // always ensure config object
     result.config = { hyphenatedSubdomains: !!domain.config.hyphenatedSubdomains };

src/eventlog.js

@@ -7,7 +7,7 @@ exports = module.exports = {
     getByCreationTime: getByCreationTime,
     cleanup: cleanup,
 
-    // keep in sync with webadmin index.js filter and CLI tool
+    // keep in sync with webadmin index.js filter
     ACTION_ACTIVATE: 'cloudron.activate',
     ACTION_APP_CLONE: 'app.clone',
     ACTION_APP_CONFIGURE: 'app.configure',
@@ -21,6 +21,9 @@ exports = module.exports = {
     ACTION_APP_OOM: 'app.oom',
     ACTION_APP_UP: 'app.up',
     ACTION_APP_DOWN: 'app.down',
+    ACTION_APP_START: 'app.start',
+    ACTION_APP_STOP: 'app.stop',
+    ACTION_APP_RESTART: 'app.restart',
 
     ACTION_BACKUP_FINISH: 'backup.finish',
     ACTION_BACKUP_START: 'backup.start',
@@ -40,8 +43,10 @@ exports = module.exports = {
     ACTION_MAIL_DISABLED: 'mail.disabled',
     ACTION_MAIL_MAILBOX_ADD: 'mail.box.add',
     ACTION_MAIL_MAILBOX_REMOVE: 'mail.box.remove',
+    ACTION_MAIL_MAILBOX_UPDATE: 'mail.box.update',
     ACTION_MAIL_LIST_ADD: 'mail.list.add',
     ACTION_MAIL_LIST_REMOVE: 'mail.list.remove',
+    ACTION_MAIL_LIST_UPDATE: 'mail.list.update',
 
     ACTION_PROVISION: 'cloudron.provision',
     ACTION_RESTORE: 'cloudron.restore', // unused
@@ -57,6 +62,9 @@ exports = module.exports = {
 
     ACTION_DYNDNS_UPDATE: 'dyndns.update',
 
+    ACTION_SUPPORT_TICKET: 'support.ticket',
+    ACTION_SUPPORT_SSH: 'support.ssh',
+
     ACTION_PROCESS_CRASH: 'system.crash'
 };
 
@@ -82,11 +90,9 @@ function add(action, source, data, callback) {
     api(uuid.v4(), action, source, data, function (error, id) {
         if (error) return callback(error);
 
-        notifications.onEvent(id, action, source, data, function (error) {
-            if (error) return callback(error);
+        callback(null, { id: id });
 
-            callback(null, { id: id });
-        });
+        notifications.onEvent(id, action, source, data, NOOP_CALLBACK);
     });
 }
 

src/externalldap.js

@@ -1,7 +1,9 @@
 'use strict';
 
 exports = module.exports = {
+    search: search,
     verifyPassword: verifyPassword,
+    createAndVerifyUserIfNotExist: createAndVerifyUserIfNotExist,
 
     testConfig: testConfig,
     startSyncer: startSyncer,
@@ -33,6 +35,25 @@ function removePrivateFields(ldapConfig) {
     return ldapConfig;
 }
 
+function translateUser(ldapConfig, ldapUser) {
+    assert.strictEqual(typeof ldapConfig, 'object');
+
+    return {
+        username: ldapUser[ldapConfig.usernameField],
+        email: ldapUser.mail,
+        displayName: ldapUser.cn // user.giveName + ' ' + user.sn
+    };
+}
+
+function validUserRequirements(user) {
+    if (!user.username || !user.email || !user.displayName) {
+        debug(`[LDAP user empty username/email/displayName] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+        return false;
+    } else {
+        return true;
+    }
+}
+
 // performs service bind if required
 function getClient(externalLdapConfig, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
@@ -60,6 +81,8 @@ function getClient(externalLdapConfig, callback) {
     });
 }
 
+
+// TODO support search by email
 function ldapSearch(externalLdapConfig, options, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof options, 'object');
@@ -136,6 +159,58 @@ function testConfig(config, callback) {
     });
 }
 
+function search(identifier, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
+        if (error) return callback(error);
+        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+
+        ldapSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
+            if (error) return callback(error);
+
+            // translate ldap properties to ours
+            let users = ldapUsers.map(function (u) { return translateUser(externalLdapConfig, u); });
+
+            callback(null, users);
+        });
+    });
+}
+
+function createAndVerifyUserIfNotExist(identifier, password, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
+        if (error) return callback(error);
+        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+        if (!externalLdapConfig.autoCreate) return callback(new BoxError(BoxError.BAD_STATE, 'auto create not enabled'));
+
+        ldapSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
+            if (error) return callback(error);
+            if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
+            if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
+
+            let user = translateUser(externalLdapConfig, ldapUsers[0]);
+            if (!validUserRequirements(user)) return callback(new BoxError(BoxError.BAD_FIELD));
+
+            users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_AUTO_CREATE, function (error, user) {
+                if (error) {
+                    console.error('Failed to auto create user', user.username, error);
+                    return callback(new BoxError(BoxError.INTERNAL_ERROR));
+                }
+
+                verifyPassword(user, password, function (error) {
+                    if (error) return callback(error);
+                    callback(null, user);
+                });
+            });
+        });
+    });
+}
+
 function verifyPassword(user, password, callback) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof password, 'string');
@@ -150,14 +225,12 @@ function verifyPassword(user, password, callback) {
             if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
             if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
 
-            const userDn = ldapUsers[0].dn;
             let client = ldap.createClient({ url: externalLdapConfig.url });
-
-            client.bind(userDn, password, function (error) {
+            client.bind(ldapUsers[0].dn, password, function (error) {
                 if (error instanceof ldap.InvalidCredentialsError) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
                 if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
-                callback();
+                callback(null, translateUser(externalLdapConfig, ldapUsers[0]));
             });
         });
     });
@@ -201,46 +274,41 @@ function sync(progressCallback, callback) {
 
             // we ignore all errors here and just log them for now
             async.eachSeries(ldapUsers, function (user, iteratorCallback) {
-                const username = user[externalLdapConfig.usernameField];
-                const email = user.mail;
-                const displayName = user.cn; // user.giveName + ' ' + user.sn
+                user = translateUser(externalLdapConfig, user);
 
-                if (!username || !email || !displayName) {
-                    debug(`[empty username/email/displayName] username=${username} email=${email} displayName=${displayName} usernameField=${externalLdapConfig.usernameField}`);
-                    return iteratorCallback();
-                }
+                if (!validUserRequirements(user)) return iteratorCallback();
 
                 percent += step;
-                progressCallback({ percent, message: `Syncing... ${username}` });
+                progressCallback({ percent, message: `Syncing... ${user.username}` });
 
-                users.getByUsername(username, function (error, result) {
+                users.getByUsername(user.username, function (error, result) {
                     if (error && error.reason !== BoxError.NOT_FOUND) {
-                        debug(`Could not find user with username ${username}: ${error.message}`);
+                        debug(`Could not find user with username ${user.username}: ${error.message}`);
                         return iteratorCallback();
                     }
 
                     if (error) {
-                        debug(`[adding user] username=${username} email=${email} displayName=${displayName}`);
+                        debug(`[adding user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
 
-                        users.create(username, null /* password */, email, displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
+                        users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
                             if (error) console.error('Failed to create user', user, error);
                             iteratorCallback();
                         });
                     } else if (result.source !== 'ldap') {
-                        debug(`[conflicting user] username=${username} email=${email} displayName=${displayName}`);
+                        debug(`[conflicting user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
 
                         iteratorCallback();
-                    } else if (result.email !== email || result.displayName !== displayName) {
-                        debug(`[updating user] username=${username} email=${email} displayName=${displayName}`);
+                    } else if (result.email !== user.email || result.displayName !== user.displayName) {
+                        debug(`[updating user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
 
-                        users.update(result.id, { email: email, fallbackEmail: email, displayName: displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
+                        users.update(result, { email: user.email, fallbackEmail: user.email, displayName: user.displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
                             if (error) debug('Failed to update user', user, error);
 
                             iteratorCallback();
                         });
                     } else {
                         // user known and up-to-date
-                        debug(`[up-to-date user] username=${username} email=${email} displayName=${displayName}`);
+                        debug(`[up-to-date user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
 
                         iteratorCallback();
                     }

src/infra_version.js

@@ -6,7 +6,7 @@
 
 exports = module.exports = {
     // a version change recreates all containers with latest docker config
-    'version': '48.16.0',
+    'version': '48.17.0',
 
     'baseImages': [
         { repo: 'cloudron/base', tag: 'cloudron/base:1.0.0@sha256:147a648a068a2e746644746bbfb42eb7a50d682437cead3c67c933c546357617' }
@@ -15,12 +15,13 @@ exports = module.exports = {
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.0.2@sha256:a28320f313785816be60e3f865e09065504170a3d20ed37de675c719b32b01eb' },
+        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.0.1@sha256:d7e8b3a88e30986a459a4da9742fbd50a1d150f07408942b91e279c41e6af059' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.2.0@sha256:440c8a9ca4d2958d51a375359f8158ef702b83395aa9ac4f450c51825ec09239' },
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:2.0.2@sha256:6dcee0731dfb9b013ed94d56205eee219040ee806c7e251db3b3886eaa4947ff' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:2.1.0@sha256:6d1bf221cfe6124957e2c58b57c0a47214353496009296acb16adf56df1da9d5' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.1.0@sha256:f2cda21bd15c21bbf44432df412525369ef831a2d53860b5c5b1675e6f384de2' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.5.0@sha256:086ae1c9433d90a820326aa43914a2afe94ad707074ef2bc05a7ef4798e83655' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.7.2@sha256:f20d112ff9a97e052a9187063eabbd8d484ce369114d44186e344169a1b3ef6b' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.2.0@sha256:fc9ca69d16e6ebdbd98ed53143d4a0d2212eef60cb638dc71219234e6f427a2c' },
-        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:0.1.0@sha256:e177c5bf5f38c84ce1dea35649c22a1b05f96eec67a54a812c5a35e585670f0f' }
+        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:1.0.0@sha256:3b70aac36700225945a4a39b5a400c28e010e980879d0dcca76e4a37b04a16ed' }
     }
 };

src/janitor.js

@@ -2,9 +2,9 @@
 
 var assert = require('assert'),
     async = require('async'),
-    authcodedb = require('./authcodedb.js'),
+    BoxError = require('./boxerror.js'),
     debug = require('debug')('box:janitor'),
-    docker = require('./docker.js').connection,
+    Docker = require('dockerode'),
     tokendb = require('./tokendb.js');
 
 exports = module.exports = {
@@ -12,7 +12,9 @@ exports = module.exports = {
     cleanupDockerVolumes: cleanupDockerVolumes
 };
 
-var NOOP_CALLBACK = function () { };
+const NOOP_CALLBACK = function () { };
+
+const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });
 
 function ignoreError(func) {
     return function (callback) {
@@ -36,26 +38,13 @@ function cleanupExpiredTokens(callback) {
     });
 }
 
-function cleanupExpiredAuthCodes(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    authcodedb.delExpired(function (error, result) {
-        if (error) return callback(error);
-
-        debug('Cleaned up %s expired authcodes.', result);
-
-        callback(null);
-    });
-}
-
 function cleanupTokens(callback) {
     assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
 
     debug('Cleaning up expired tokens');
 
     async.series([
-        ignoreError(cleanupExpiredTokens),
-        ignoreError(cleanupExpiredAuthCodes)
+        ignoreError(cleanupExpiredTokens)
     ], callback);
 }
 
@@ -67,16 +56,16 @@ function cleanupTmpVolume(containerInfo, callback) {
 
     debug('cleanupTmpVolume %j', containerInfo.Names);
 
-    docker.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
-        if (error) return callback(new Error('Failed to exec container : ' + error.message));
+    gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
+        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`));
 
         execContainer.start({ hijack: true }, function (error, stream) {
-            if (error) return callback(new Error('Failed to start exec container : ' + error.message));
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to start exec container: ${error.message}`));
 
             stream.on('error', callback);
             stream.on('end', callback);
 
-            docker.modem.demuxStream(stream, process.stdout, process.stderr);
+            gConnection.modem.demuxStream(stream, process.stdout, process.stderr);
         });
     });
 }
@@ -88,7 +77,7 @@ function cleanupDockerVolumes(callback) {
 
     debug('Cleaning up docker volumes');
 
-    docker.listContainers({ all: 0 }, function (error, containers) {
+    gConnection.listContainers({ all: 0 }, function (error, containers) {
         if (error) return callback(error);
 
         async.eachSeries(containers, function (container, iteratorDone) {

src/ldap.js

@@ -126,16 +126,16 @@ function userSearch(req, res, next) {
         var results = [];
 
         // send user objects
-        result.forEach(function (entry) {
+        result.forEach(function (user) {
             // skip entries with empty username. Some apps like owncloud can't deal with this
-            if (!entry.username) return;
+            if (!user.username) return;
 
-            var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+            var dn = ldap.parseDN('cn=' + user.id + ',ou=users,dc=cloudron');
 
             var groups = [ GROUP_USERS_DN ];
-            if (entry.admin) groups.push(GROUP_ADMINS_DN);
+            if (users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) groups.push(GROUP_ADMINS_DN);
 
-            var displayName = entry.displayName || entry.username || ''; // displayName can be empty and username can be null
+            var displayName = user.displayName || user.username || ''; // displayName can be empty and username can be null
             var nameParts = displayName.split(' ');
             var firstName = nameParts[0];
             var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
@@ -143,17 +143,18 @@ function userSearch(req, res, next) {
             var obj = {
                 dn: dn.toString(),
                 attributes: {
-                    objectclass: ['user'],
+                    objectclass: ['user', 'inetorgperson', 'person' ],
                     objectcategory: 'person',
-                    cn: entry.id,
-                    uid: entry.id,
-                    mail: entry.email,
-                    mailAlternateAddress: entry.fallbackEmail,
+                    cn: user.id,
+                    uid: user.id,
+                    entryuuid: user.id, // to support OpenLDAP clients
+                    mail: user.email,
+                    mailAlternateAddress: user.fallbackEmail,
                     displayname: displayName,
                     givenName: firstName,
-                    username: entry.username,
-                    samaccountname: entry.username,      // to support ActiveDirectory clients
-                    isadmin: entry.admin,
+                    username: user.username,
+                    samaccountname: user.username,      // to support ActiveDirectory clients
+                    isadmin: users.compareRoles(user.role, users.ROLE_ADMIN) >= 0,
                     memberof: groups
                 }
             };
@@ -193,7 +194,7 @@ function groupSearch(req, res, next) {
 
         groups.forEach(function (group) {
             var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-            var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+            var members = group.admin ? result.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : result;
 
             var obj = {
                 dn: dn.toString(),
@@ -241,8 +242,8 @@ function groupAdminsCompare(req, res, next) {
 
         // we only support memberuid here, if we add new group attributes later add them here
         if (req.attribute === 'memberuid') {
-            var found = result.find(function (u) { return u.id === req.value; });
-            if (found && found.admin) return res.end(true);
+            var user = result.find(function (u) { return u.id === req.value; });
+            if (user && users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) return res.end(true);
         }
 
         res.end(false);
@@ -283,10 +284,11 @@ function mailboxSearch(req, res, next) {
                 res.end();
             }
         });
-    } else if (req.dn.rdns[0].attrs.domain) {
+    } else if (req.dn.rdns[0].attrs.domain) { // legacy ldap mailbox search for old sogo
         var domain = req.dn.rdns[0].attrs.domain.value.toLowerCase();
 
         mailboxdb.listMailboxes(domain, 1, 1000, function (error, result) {
+
             if (error) return next(new ldap.OperationsError(error.toString()));
 
             var results = [];
@@ -309,7 +311,6 @@ function mailboxSearch(req, res, next) {
                 // ensure all filter values are also lowercase
                 var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
                 if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
                 if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
                     results.push(obj);
                 }
@@ -317,8 +318,55 @@ function mailboxSearch(req, res, next) {
 
             finalSend(results, req, res, next);
         });
-    } else {
-        return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    } else { // new sogo
+        mailboxdb.listAllMailboxes(1, 1000, function (error, mailboxes) {
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            var results = [];
+
+            // send mailbox objects
+            async.eachSeries(mailboxes, function (mailbox, callback) {
+                var dn = ldap.parseDN(`cn=${mailbox.name}@${mailbox.domain},ou=mailboxes,dc=cloudron`);
+
+                users.get(mailbox.ownerId, function (error, userObject) {
+                    if (error) return callback(); // skip mailboxes with unknown owner
+
+                    var obj = {
+                        dn: dn.toString(),
+                        attributes: {
+                            objectclass: ['mailbox'],
+                            objectcategory: 'mailbox',
+                            displayname: userObject.displayName,
+                            cn: `${mailbox.name}@${mailbox.domain}`,
+                            uid: `${mailbox.name}@${mailbox.domain}`,
+                            mail: `${mailbox.name}@${mailbox.domain}`
+                        }
+                    };
+
+                    mailboxdb.getAliasesForName(mailbox.name, mailbox.domain, function (error, aliases) {
+                        if (error) return callback(error);
+
+                        aliases.forEach(function (a, idx) {
+                            obj.attributes['mail' + idx] = `${a}@${mailbox.domain}`;
+                        });
+
+                        // ensure all filter values are also lowercase
+                        var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+                        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+                        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                            results.push(obj);
+                        }
+
+                        callback();
+                    });
+                });
+            }, function (error) {
+                if (error) return next(new ldap.OperationsError(error.toString()));
+
+                finalSend(results, req, res, next);
+            });
+        });
     }
 }
 
@@ -419,7 +467,7 @@ function authenticateUser(req, res, next) {
         api = users.verifyWithUsername;
     }
 
-    api(commonName, req.credentials || '', function (error, user) {
+    api(commonName, req.credentials || '', req.app.id, function (error, user) {
         if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.message));
@@ -465,7 +513,7 @@ function authenticateUserMailbox(req, res, next) {
             if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
             if (error) return next(new ldap.OperationsError(error.message));
 
-            users.verify(mailbox.ownerId, req.credentials || '', function (error, result) {
+            users.verify(mailbox.ownerId, req.credentials || '', users.AP_MAIL, function (error, result) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
@@ -486,13 +534,16 @@ function authenticateSftp(req, res, next) {
     var parts = email.split('@');
     if (parts.length !== 2) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-    // actual user bind
-    users.verifyWithUsername(parts[0], req.credentials, function (error) {
+    apps.getByFqdn(parts[1], function (error, app) {
         if (error) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
 
-        debug('sftp auth: success');
+        users.verifyWithUsername(parts[0], req.credentials, app.id, function (error) {
+            if (error) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
 
-        res.end();
+            debug('sftp auth: success');
+
+            res.end();
+        });
     });
 }
 
@@ -576,7 +627,7 @@ function authenticateMailAddon(req, res, next) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
 
-                users.verify(mailbox.ownerId, req.credentials || '', function (error, result) {
+                users.verify(mailbox.ownerId, req.credentials || '', users.AP_MAIL, function (error, result) {
                     if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                     if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                     if (error) return next(new ldap.OperationsError(error.message));

src/locker.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
     debug = require('debug')('box:locker'),
     EventEmitter = require('events').EventEmitter,
     util = require('util');
@@ -23,7 +24,7 @@ Locker.prototype.lock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
     if (this._operation !== null) {
-        let error = new Error(`Locked for ${this._operation}`);
+        let error = new BoxError(BoxError.CONFLICT, `Locked for ${this._operation}`);
         error.operation = this._operation;
         return error;
     }
@@ -54,7 +55,7 @@ Locker.prototype.recursiveLock = function (operation) {
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
-    if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
+    if (this._operation !== operation) throw BoxError(BoxError.BAD_STATE, 'Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
     if (--this._lockDepth === 0) {
         debug('Released : %s', this._operation);

src/mail.js

@@ -7,10 +7,11 @@ exports = module.exports = {
     getDomains: getDomains,
 
     getDomain: getDomain,
-    addDomain: addDomain,
-    removeDomain: removeDomain,
     clearDomains: clearDomains,
 
+    onDomainAdded: onDomainAdded,
+    onDomainRemoved: onDomainRemoved,
+
     removePrivateFields: removePrivateFields,
 
     setDnsRecords: setDnsRecords,
@@ -101,7 +102,6 @@ function checkOutboundPort25(callback) {
         'smtp.gmail.com',
         'smtp.live.com',
         'smtp.mail.yahoo.com',
-        'smtp.comcast.net',
         'smtp.1und1.de',
     ]);
 
@@ -123,13 +123,13 @@ function checkOutboundPort25(callback) {
         relay.status = false;
         relay.value = `Connect to ${smtpServer} timed out. Check if port 25 (outbound) is blocked`;
         client.destroy();
-        callback(new Error('Timeout'), relay);
+        callback(new BoxError(BoxError.TIMEOUT, `Connect to ${smtpServer} timed out.`), relay);
     });
     client.on('error', function (error) {
         relay.status = false;
         relay.value = `Connect to ${smtpServer} failed: ${error.message}. Check if port 25 (outbound) is blocked`;
         client.destroy();
-        callback(error, relay);
+        callback(new BoxError(BoxError.NETWORK_ERROR, `Connect to ${smtpServer} failed.`), relay);
     });
 }
 
@@ -199,7 +199,7 @@ function checkDkim(mailDomain, callback) {
     };
 
     var dkimKey = readDkimPublicKeySync(domain);
-    if (!dkimKey) return callback(new Error('Failed to read dkim public key'), dkim);
+    if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, `Failed to read dkim public key of ${domain}`), dkim);
 
     dkim.expected = 'v=DKIM1; t=s; p=' + dkimKey;
 
@@ -313,9 +313,8 @@ function checkDmarc(domain, callback) {
 
         if (txtRecords.length !== 0) {
             dmarc.value = txtRecords[0].join('');
-            // allow extra fields in dmarc like rua
-            const actual = txtToDict(dmarc.value), expected = txtToDict(dmarc.expected);
-            dmarc.status = Object.keys(expected).every(k => expected[k] === actual[k]);
+            const actual = txtToDict(dmarc.value);
+            dmarc.status = actual.v === 'DMARC1'; // see box#666
         }
 
         callback(null, dmarc);
@@ -456,7 +455,7 @@ function getStatus(domain, callback) {
 
     // ensure we always have a valid toplevel properties for the api
     var results = {
-        dns: {}, // { mx/dmar/dkim/spf/ptr: { expected, value, name, domain, type } }
+        dns: {}, // { mx/dmarc/dkim/spf/ptr: { expected, value, name, domain, type, status } }
         rbl: {}, // { status, ip, servers: [{name,site,dns}]} optional. only for cloudron-smtp
         relay: {} // { status, value } always checked
     };
@@ -466,7 +465,7 @@ function getStatus(domain, callback) {
             func(function (error, result) {
                 if (error) debug('Ignored error - ' + what + ':', error);
 
-                safe.set(results, what, result);
+                safe.set(results, what, result || {});
 
                 callback();
             });
@@ -567,12 +566,12 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
         // mail_domain is used for SRS
         if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
             `mail_in_domains=${mailInDomains}\nmail_out_domains=${mailOutDomains}\nmail_server_name=${mailFqdn}\nmail_domain=${mailDomain}\n\n`, 'utf8')) {
-            return callback(new Error('Could not create mail var file:' + safe.error.message));
+            return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
         }
 
         // enable_outbound makes plugin forward email for relayed mail. non-relayed mail always hits LMTP plugin first
         if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/smtp_forward.ini'), 'enable_outbound=false\ndomain_selector=mail_from\n', 'utf8')) {
-            return callback(new Error('Could not create smtp forward file:' + safe.error.message));
+            return callback(new BoxError(BoxError.FS_ERROR, 'Could not create smtp forward file:' + safe.error.message));
         }
 
         // create sections for per-domain configuration
@@ -582,7 +581,7 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
 
             if (!safe.fs.appendFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
                 `[${domain.domain}]\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\n\n`, 'utf8')) {
-                return callback(new Error('Could not create mail var file:' + safe.error.message));
+                return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
             }
 
             const relay = domain.relay;
@@ -598,7 +597,7 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
 
             if (!safe.fs.appendFileSync(paths.ADDON_CONFIG_DIR + '/mail/smtp_forward.ini',
                 `[${domain.domain}]\nenable_outbound=true\nhost=${host}\nport=${port}\nenable_tls=true\nauth_type=${authType}\nauth_user=${username}\nauth_pass=${password}\n\n`, 'utf8')) {
-                return callback(new Error('Could not create mail var file:' + safe.error.message));
+                return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
             }
         });
 
@@ -627,8 +626,8 @@ function configureMail(mailFqdn, mailDomain, callback) {
         const mailCertFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_cert.pem');
         const mailKeyFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_key.pem');
 
-        if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new Error('Could not create cert file:' + safe.error.message));
-        if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new Error('Could not create key file:' + safe.error.message));
+        if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create cert file:' + safe.error.message));
+        if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create key file:' + safe.error.message));
 
         shell.exec('startMail', 'docker rm -f mail || true', function (error) {
             if (error) return callback(error);
@@ -654,7 +653,6 @@ function configureMail(mailFqdn, mailDomain, callback) {
                             -v "${paths.MAIL_DATA_DIR}:/app/data" \
                             -v "${paths.PLATFORM_DATA_DIR}/addons/mail:/etc/mail" \
                             ${ports} \
-                            -p 127.0.0.1:2020:2020 \
                             --label isCloudronManaged=true \
                             --read-only -v /run -v /tmp ${tag}`;
 
@@ -846,7 +844,7 @@ function upsertDnsRecords(domain, mailFqdn, callback) {
         if (process.env.BOX_ENV === 'test') return callback();
 
         var dkimKey = readDkimPublicKeySync(domain);
-        if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, new Error('Failed to read dkim public key')));
+        if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, 'Failed to read dkim public key'));
 
         // t=s limits the domainkey to this domain and not it's subdomains
         var dkimRecord = { subdomain: `${mailDomain.dkimSelector}._domainkey`, domain: domain, type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
@@ -907,37 +905,21 @@ function onMailFqdnChanged(callback) {
     });
 }
 
-function addDomain(domain, callback) {
+function onDomainAdded(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    const dkimSelector = domain === settings.adminDomain() ? 'cloudron' : ('cloudron-' + settings.adminDomain().replace(/\./g, ''));
-
-    maildb.add(domain, { dkimSelector }, function (error) {
-        if (error) return callback(error);
-
-        async.series([
-            upsertDnsRecords.bind(null, domain, settings.mailFqdn()), // do this first to ensure DKIM keys
-            restartMailIfActivated
-        ], NOOP_CALLBACK); // do these asynchronously
-
-        callback();
-    });
+    async.series([
+        upsertDnsRecords.bind(null, domain, settings.mailFqdn()), // do this first to ensure DKIM keys
+        restartMailIfActivated
+    ], callback);
 }
 
-function removeDomain(domain, callback) {
+function onDomainRemoved(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (domain === settings.adminDomain()) return callback(new BoxError(BoxError.CONFLICT));
-
-    maildb.del(domain, function (error) {
-        if (error) return callback(error);
-
-        restartMail(NOOP_CALLBACK);
-
-        callback();
-    });
+    restartMail(callback);
 }
 
 function clearDomains(callback) {
@@ -1106,18 +1088,25 @@ function addMailbox(name, domain, userId, auditSource, callback) {
     });
 }
 
-function updateMailboxOwner(name, domain, userId, callback) {
+function updateMailboxOwner(name, domain, userId, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     name = name.toLowerCase();
 
-    mailboxdb.updateMailboxOwner(name, domain, userId, function (error) {
+    getMailbox(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        callback(null);
+        mailboxdb.updateMailboxOwner(name, domain, userId, function (error) {
+            if (error) return callback(error);
+
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: result.userId, userId });
+
+            callback(null);
+        });
     });
 }
 
@@ -1196,12 +1185,12 @@ function getLists(domain, callback) {
     });
 }
 
-function getList(domain, listName, callback) {
+function getList(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof listName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    mailboxdb.getList(listName, domain, function (error, result) {
+    mailboxdb.getList(name, domain, function (error, result) {
         if (error) return callback(error);
 
         callback(null, result);
@@ -1227,16 +1216,17 @@ function addList(name, domain, members, auditSource, callback) {
     mailboxdb.addList(name, domain, members, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members });
 
         callback();
     });
 }
 
-function updateList(name, domain, members, callback) {
+function updateList(name, domain, members, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert(Array.isArray(members));
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     name = name.toLowerCase();
@@ -1248,10 +1238,16 @@ function updateList(name, domain, members, callback) {
         if (!validator.isEmail(members[i])) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid email: ' + members[i]));
     }
 
-    mailboxdb.updateList(name, domain, members, function (error) {
+    getList(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        callback(null);
+        mailboxdb.updateList(name, domain, members, function (error) {
+            if (error) return callback(error);
+
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members });
+
+            callback(null);
+        });
     });
 }
 
@@ -1264,7 +1260,7 @@ function removeList(name, domain, auditSource, callback) {
     mailboxdb.del(name, domain, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
 
         callback();
     });
@@ -1315,4 +1311,4 @@ function resolveList(listName, listDomain, callback) {
             });
         });
     });
-}
+}

src/mail_templates/welcome_user.ejs

@@ -5,7 +5,7 @@ Dear <%= user.displayName || user.username || user.email %>,
 Welcome to <%= cloudronName %>!
 
 Follow the link to get started.
-<%- setupLink %>
+<%- inviteLink %>
 
 <% if (invitor && invitor.email) { %>
 You are receiving this email because you were invited by <%= invitor.email %>.
@@ -25,7 +25,7 @@ Powered by https://cloudron.io
 <h2>Welcome to <%= cloudronName %>!</h2>
 
 <p>
-    <a href="<%= setupLink %>">Get started</a>.
+    <a href="<%= inviteLink %>">Get started</a>.
 </p>
 
 <br/>

src/mailboxdb.js

@@ -12,6 +12,8 @@ exports = module.exports = {
     listMailboxes: listMailboxes,
     getLists: getLists,
 
+    listAllMailboxes: listAllMailboxes,
+
     get: get,
     getMailbox: getMailbox,
     getList: getList,
@@ -214,6 +216,21 @@ function listMailboxes(domain, page, perPage, callback) {
         });
 }
 
+function listAllMailboxes(page, perPage, callback) {
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? ORDER BY name LIMIT ${(page-1)*perPage},${perPage}`,
+        [ exports.TYPE_MAILBOX ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
 function getLists(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');

src/maildb.js

@@ -1,8 +1,6 @@
 'use strict';
 
 exports = module.exports = {
-    add: add,
-    del: del,
     get: get,
     list: list,
     update: update,
@@ -34,20 +32,6 @@ function postProcess(data) {
     return data;
 }
 
-function add(domain, data, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('INSERT INTO mail (domain, dkimSelector) VALUES (?, ?)', [ domain, data.dkimSelector || 'cloudron' ], function (error) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'mail domain already exists'));
-        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND), 'no such domain');
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -58,20 +42,6 @@ function clear(callback) {
     });
 }
 
-function del(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    // deletes aliases as well
-    database.query('DELETE FROM mail WHERE domain=?', [ domain ], function (error, result) {
-        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') return callback(new BoxError(BoxError.CONFLICT));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mail domain not found'));
-
-        callback(null);
-    });
-}
-
 function get(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');

src/mailer.js

@@ -3,7 +3,7 @@
 exports = module.exports = {
     userAdded: userAdded,
     userRemoved: userRemoved,
-    adminChanged: adminChanged,
+    roleChanged: roleChanged,
     passwordReset: passwordReset,
     appUpdatesAvailable: appUpdatesAvailable,
 
@@ -26,7 +26,6 @@ exports = module.exports = {
 
 var assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    custom = require('./custom.js'),
     debug = require('debug')('box:mailer'),
     ejs = require('ejs'),
     mail = require('./mail.js'),
@@ -47,15 +46,16 @@ function getMailConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     settings.getCloudronName(function (error, cloudronName) {
-        // this is not fatal
-        if (error) {
-            debug(error);
-            cloudronName = 'Cloudron';
-        }
+        if (error) debug('Error getting cloudron name: ', error);
 
-        callback(null, {
-            cloudronName: cloudronName,
-            notificationFrom: `"${cloudronName}" <no-reply@${settings.adminDomain()}>`
+        settings.getSupportConfig(function (error, supportConfig) {
+            if (error) debug('Error getting support config: ', error);
+
+            callback(null, {
+                cloudronName: cloudronName || '',
+                notificationFrom: `"${cloudronName}" <no-reply@${settings.adminDomain()}>`,
+                supportEmail: supportConfig.email
+            });
         });
     });
 }
@@ -125,9 +125,10 @@ function mailUserEvent(mailTo, user, event) {
     });
 }
 
-function sendInvite(user, invitor) {
+function sendInvite(user, invitor, inviteLink) {
     assert.strictEqual(typeof user, 'object');
-    assert(typeof invitor === 'object');
+    assert.strictEqual(typeof invitor, 'object');
+    assert.strictEqual(typeof inviteLink, 'string');
 
     debug('Sending invite mail');
 
@@ -137,7 +138,7 @@ function sendInvite(user, invitor) {
         var templateData = {
             user: user,
             webadminUrl: settings.adminOrigin(),
-            setupLink: `${settings.adminOrigin()}/api/v1/session/account/setup.html?reset_token=${user.resetToken}&email=${encodeURIComponent(user.email)}`,
+            inviteLink: inviteLink,
             invitor: invitor,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
@@ -203,14 +204,13 @@ function userRemoved(mailTo, user) {
     mailUserEvent(mailTo, user, 'was removed');
 }
 
-function adminChanged(mailTo, user, isAdmin) {
+function roleChanged(mailTo, user) {
     assert.strictEqual(typeof mailTo, 'string');
     assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof isAdmin, 'boolean');
 
-    debug('Sending mail for adminChanged');
+    debug('Sending mail for roleChanged');
 
-    mailUserEvent(mailTo, user, isAdmin ? 'is now an admin' : 'is no more an admin');
+    mailUserEvent(mailTo, user, `now has the role '${user.role}'`);
 }
 
 function passwordReset(user) {
@@ -223,7 +223,7 @@ function passwordReset(user) {
 
         var templateData = {
             user: user,
-            resetLink: `${settings.adminOrigin()}/api/v1/session/password/reset.html?reset_token=${user.resetToken}&email=${encodeURIComponent(user.email)}`,
+            resetLink: `${settings.adminOrigin()}/login.html?resetToken=${user.resetToken}`,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
         };
@@ -279,7 +279,7 @@ function appDied(mailTo, app) {
             from: mailConfig.notificationFrom,
             to: mailTo,
             subject: util.format('[%s] App %s is down', mailConfig.cloudronName, app.fqdn),
-            text: render('app_down.ejs', { title: app.manifest.title, appFqdn: app.fqdn, supportEmail: custom.spec().support.email, format: 'text' })
+            text: render('app_down.ejs', { title: app.manifest.title, appFqdn: app.fqdn, supportEmail: mailConfig.supportEmail, format: 'text' })
         };
 
         sendMail(mailOptions);

src/middleware/index.js

@@ -1,15 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    cookieParser: require('cookie-parser'),
     cors: require('./cors'),
-    csrf: require('csurf'),
     json: require('body-parser').json,
     morgan: require('morgan'),
     proxy: require('proxy-middleware'),
     lastMile: require('connect-lastmile'),
     multipart: require('./multipart.js'),
-    session: require('express-session'),
     timeout: require('connect-timeout'),
     urlencoded: require('body-parser').urlencoded
 };

src/nginxconfig.ejs

@@ -58,17 +58,16 @@ server {
     ssl_certificate      <%= certFilePath %>;
     ssl_certificate_key  <%= keyFilePath %>;
     ssl_session_timeout  5m;
-    ssl_session_cache shared:SSL:50m;
+    ssl_session_cache shared:MozSSL:10m;  # about 40000 sessions
+    ssl_session_tickets off;
 
-    # https://bettercrypto.org/static/applied-crypto-hardening.pdf
-    # https://mozilla.github.io/server-side-tls/ssl-config-generator/
-    # https://cipherli.st/
     # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
-    ssl_prefer_server_ciphers on;
-    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
+    # https://github.com/ssllabs/research/wiki/SSL-and-TLS-Deployment-Best-Practices#25-use-forward-secrecy
+    # ciphers according to https://ssl-config.mozilla.org/#server=nginx&version=1.14.0&config=intermediate&openssl=1.1.1&guideline=5.4
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+    ssl_prefer_server_ciphers off;
 
-    # ciphers according to https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=nginx-1.10.3&openssl=1.0.2g&hsts=yes&profile=modern
-    ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
     ssl_dhparam /home/yellowtent/boxdata/dhparams.pem;
     add_header Strict-Transport-Security "max-age=15768000";
 
@@ -108,7 +107,9 @@ server {
 <% } -%>
 
     proxy_http_version 1.1;
+    # intercept errors (>= 400) and use the error_page handler
     proxy_intercept_errors on;
+    # nginx will return 504 on connect/timeout errors
     proxy_read_timeout       3500;
     proxy_connect_timeout    3250;
 
@@ -125,7 +126,15 @@ server {
 
     # only serve up the status page if we get proxy gateway errors
     root <%= sourceDir %>/dashboard/dist;
-    error_page 502 503 504 /appstatus.html;
+    # some apps use 503 to indicate updating or maintenance
+    error_page 502 504 /app_error_page;
+    location /app_error_page {
+        root /home/yellowtent/boxdata;
+        # the first argument looks for file under the root
+        try_files /custom_pages/$request_uri /custom_pages/app_not_responding.html /appstatus.html;
+        # internal means this is for internal routing and cannot be accessed as URL from browser
+        internal;
+    }
     location /appstatus.html {
         internal;
     }

src/notifications.js

@@ -25,7 +25,6 @@ let assert = require('assert'),
     auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     changelog = require('./changelog.js'),
-    custom = require('./custom.js'),
     debug = require('debug')('box:notifications'),
     eventlog = require('./eventlog.js'),
     mailer = require('./mailer.js'),
@@ -100,7 +99,8 @@ function actionForAllAdmins(skippingUserIds, iterator, callback) {
     assert.strictEqual(typeof iterator, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    users.getAllAdmins(function (error, result) {
+    users.getAdmins(function (error, result) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback();
         if (error) return callback(error);
 
         // filter out users we want to skip (like the user who did the action or the user the action was performed on)
@@ -134,14 +134,14 @@ function userRemoved(performedBy, eventId, user, callback) {
     }, callback);
 }
 
-function adminChanged(performedBy, eventId, user, callback) {
+function roleChanged(performedBy, eventId, user, callback) {
     assert.strictEqual(typeof performedBy, 'string');
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     actionForAllAdmins([ performedBy, user.id ], function (admin, done) {
-        mailer.adminChanged(admin.email, user, user.admin);
-        add(admin.id, eventId, `User '${user.displayName} ' ${user.admin ? 'is now an admin' : 'is no more an admin'}`, `User '${user.username || user.email || user.fallbackEmail}' ${user.admin ? 'is now an admin' : 'is no more an admin'}.`, done);
+        mailer.roleChanged(admin.email, user);
+        add(admin.id, eventId, `User '${user.displayName}'s role changed`, `User '${user.username || user.email || user.fallbackEmail}' now has the role ${user.role}.`, done);
     }, callback);
 }
 
@@ -167,9 +167,6 @@ function oomEvent(eventId, app, addon, containerId, event, callback) {
         message = 'The container has been restarted automatically. Consider increasing the [memory limit](https://docs.docker.com/v17.09/edge/engine/reference/commandline/update/#update-a-containers-kernel-memory-constraints)';
     }
 
-    if (custom.spec().alerts.email) mailer.oomEvent(custom.spec().alerts.email, program, event);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, done) {
         mailer.oomEvent(admin.email, program, event);
 
@@ -182,9 +179,6 @@ function appUp(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.appUp(custom.spec().alerts.email, app);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, done) {
         mailer.appUp(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is back online`, `The application ${app.manifest.title} installed at ${app.fqdn} is back online.`, done);
@@ -196,9 +190,6 @@ function appDied(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.appDied(custom.spec().alerts.email, app);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, callback) {
         mailer.appDied(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is down`, `The application ${app.manifest.title} installed at ${app.fqdn} is not responding.`, callback);
@@ -246,9 +237,6 @@ function boxUpdateError(eventId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.boxUpdateError(custom.spec().alerts.email, errorMessage);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, done) {
         mailer.boxUpdateError(admin.email, errorMessage);
         add(admin.id, eventId, 'Cloudron update failed', `Failed to update Cloudron: ${errorMessage}. Update will be retried in 4 hours`, done);
@@ -261,9 +249,6 @@ function certificateRenewalError(eventId, vhost, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.certificateRenewalError(custom.spec().alerts.email, vhost, errorMessage);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, callback) {
         mailer.certificateRenewalError(admin.email, vhost, errorMessage);
         add(admin.id, eventId, `Certificate renewal of ${vhost} failed`, `Failed to new certs of ${vhost}: ${errorMessage}. Renewal will be retried in 12 hours`, callback);
@@ -276,9 +261,6 @@ function backupFailed(eventId, taskId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.backupFailed(custom.spec().alerts.email, errorMessage, `${settings.adminOrigin()}/logs.html?taskId=${taskId}`);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, callback) {
         mailer.backupFailed(admin.email, errorMessage, `${settings.adminOrigin()}/logs.html?taskId=${taskId}`);
         add(admin.id, eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}). Will be retried in 4 hours`, callback);
@@ -343,8 +325,8 @@ function onEvent(id, action, source, data, callback) {
         return userRemoved(source.userId, id, data.user, callback);
 
     case eventlog.ACTION_USER_UPDATE:
-        if (!data.adminStatusChanged) return callback();
-        return adminChanged(source.userId, id, data.user, callback);
+        if (!data.roleChanged) return callback();
+        return roleChanged(source.userId, id, data.user, callback);
 
     case eventlog.ACTION_APP_OOM:
         return oomEvent(id, data.app, data.addon, data.containerId, data.event, callback);

src/oauth2views/account_setup.ejs

@@ -1,82 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<script>
-
-'use strict';
-
-// very basic angular app
-var app = angular.module('Application', []);
-app.controller('Controller', ['$scope', function ($scope) {
-    $scope.username = '<%= (user && user.username) ? user.username : '' %>';
-    $scope.displayName = '<%= (user && user.displayName) ? user.displayName : '' %>';
-}]);
-
-</script>
-
-<div class="layout-content">
-
-  <center>
-    <br/>
-    <h4>Hello <%= (user && user.email) ? user.email : '' %>, welcome to <%= cloudronName %>!</h4>
-    <h2>Setup your account and password</h2>
-  </center>
-
-  <div class="container" ng-app="Application" ng-controller="Controller">
-    <div class="row">
-      <div class="col-md-6 col-md-offset-3">
-        <form action="/api/v1/session/account/setup" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
-          <input type="password" style="display: none;">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <input type="hidden" name="email" value="<%= email %>"/>
-          <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
-
-          <center><p class="has-error"><%= error %></p></center>
-
-<% if (user && user.username) { %>
-          <div class="form-group"">
-            <label class="control-label">Username</label>
-            <input type="text" class="form-control" ng-model="username" name="username" readonly required>
-          </div>
-<% } else { %>
-          <div class="form-group" ng-class="{ 'has-error': (setupForm.username.$dirty && setupForm.username.$invalid) }">
-            <label class="control-label">Username</label>
-            <div class="control-label" ng-show="setupForm.username.$dirty && setupForm.username.$invalid">
-              <small ng-show="setupForm.username.$error.minlength">The username is too short</small>
-              <small ng-show="setupForm.username.$error.maxlength">The username is too long</small>
-              <small ng-show="setupForm.username.$dirty && setupForm.username.$invalid">Not a valid username</small>
-            </div>
-            <input type="text" class="form-control" ng-model="username" name="username" required autofocus>
-          </div>
-<% } %>
-
-          <div class="form-group">
-            <label class="control-label">Full Name</label>
-            <input type="displayName" class="form-control" ng-model="displayName" name="displayName" required>
-          </div>
-
-          <div class="form-group" ng-class="{ 'has-error': (setupForm.password.$dirty && setupForm.password.$invalid) }">
-            <label class="control-label">New Password</label>
-            <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
-              <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be atleast 8 characters</small>
-            </div>
-            <input type="password" class="form-control" ng-model="password" name="password" ng-pattern="/^.{8,}$/" required>
-          </div>
-
-          <div class="form-group" ng-class="{ 'has-error': (setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)) }">
-            <label class="control-label">Repeat Password</label>
-            <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
-              <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
-            </div>
-            <input type="password" class="form-control" ng-model="passwordRepeat" name="passwordRepeat" required>
-          </div>
-
-          <center><input class="btn btn-primary btn-outline" type="submit" value="Setup" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/></center>
-        </form>
-      </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/callback.ejs

@@ -1,45 +0,0 @@
-<!-- callback tester -->
-
-<script>
-
-'use strict';
-
-var code = null;
-var redirectURI = null;
-var accessToken = null;
-var tokenType = null;
-var state = null;
-
-var args = window.location.search.slice(1).split('&');
-args.forEach(function (arg) {
-    var tmp = arg.split('=');
-    if (tmp[0] === 'redirectURI') {
-        redirectURI = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'code') {
-        code = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'state') {
-        state = window.decodeURIComponent(tmp[1]);
-    }
-});
-
-args = window.location.hash.slice(1).split('&');
-args.forEach(function (arg) {
-    var tmp = arg.split('=');
-    if (tmp[0] === 'access_token') {
-        accessToken = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'token_type') {
-        tokenType = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'state') {
-        state = window.decodeURIComponent(tmp[1]);
-    }
-});
-
-if (code && redirectURI) {
-    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'code=' + code + (state ? '&state=' + state : '');
-} else if (redirectURI && accessToken) {
-    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'token=' + accessToken + (state ? '&state=' + state : '');
-} else {
-    window.location.href = '/api/v1/session/login';
-}
-
-</script>

src/oauth2views/error.ejs

@@ -1,27 +0,0 @@
-<% include header %>
-
-<!-- error tester -->
-
-<div class="layout-content">
-
-  <div class="container" style="margin-top: 50px;">
-    <div class="row">
-      <div class="col-md-2"></div>
-      <div class="col-md-8">
-        <div class="alert alert-danger">
-          <%- message %>
-        </div>
-      </div>
-      <div class="col-md-2"></div>
-    </div>
-    <div class="row">
-      <div class="col-md-2"></div>
-      <div class="col-md-8 text-center">
-        <a href="<%- adminOrigin %>">Back</a>
-      </div>
-      <div class="col-md-2"></div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/footer.ejs

@@ -1,9 +0,0 @@
-
-    <footer class="text-center">
-      <span class="text-muted">&copy; 2016-19 <a href="https://cloudron.io" target="_blank">Cloudron</a></span>
-    </footer>
-
-	</div>
-
-</body>
-</html>

src/oauth2views/header.ejs

@@ -1,41 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, height=device-height" />
-    <meta http-equiv="Content-Security-Policy" content="default-src 'unsafe-inline' 'unsafe-eval' 'self'; img-src 'self'" />
-
-    <title> <%= title %> </title>
-
-    <link href="/api/v1/cloudron/avatar?<%= Math.random() %>" rel="icon" type="image/png">
-
-    <!-- Theme CSS -->
-    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
-
-    <!-- Custom Fonts -->
-    <link href="<%= adminOrigin %>/3rdparty/fontawesome/css/all.min.css" rel="stylesheet" rel="stylesheet" type="text/css">
-
-    <!-- jQuery-->
-    <script src="<%= adminOrigin %>/3rdparty/js/jquery.min.js"></script>
-
-    <!-- Bootstrap Core JavaScript -->
-    <script src="<%= adminOrigin %>/3rdparty/js/bootstrap.min.js"></script>
-
-    <!-- Angularjs scripts -->
-    <script src="<%= adminOrigin %>/3rdparty/js/angular.min.js"></script>
-    <script src="<%= adminOrigin %>/3rdparty/js/angular-loader.min.js"></script>
-
-</head>
-
-<body>
-
-  <div class="layout-root">
-
-    <nav class="navbar navbar-default navbar-static-top shadow" role="navigation" style="margin-bottom: 0">
-      <div class="container-fluid">
-        <div class="navbar-header">
-          <a href="/" class="navbar-brand navbar-brand-icon"><img src="/api/v1/cloudron/avatar?<%= Math.random() %>" width="40" height="40"/></a>
-          <a href="/" class="navbar-brand"><%= cloudronName %></a>
-        </div>
-      </div>
-    </nav>

src/oauth2views/login.ejs

@@ -1,58 +0,0 @@
-<% include header %>
-
-<!-- login tester -->
-
-<div class="layout-content">
-  <div class="card" style="padding: 20px; margin-top: 50px; max-width: 620px;">
-    <div class="row">
-      <div class="col-md-12" style="text-align: center;">
-        <img width="128" height="128" src="<%= applicationLogo %>?<%= Math.random() %>"/>
-        <h1><small>Login to</small> <%= applicationName %></h1>
-        <br/>
-      </div>
-    </div>
-    <br/>
-<% if (error) { -%>
-    <div class="row">
-      <div class="col-md-12">
-        <h4 class="has-error"><%= error %></h4>
-      </div>
-    </div>
-<% } -%>
-    <div class="row">
-      <div class="col-md-12">
-        <form id="loginForm" action="" method="post">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <div class="form-group">
-            <label class="control-label" for="inputUsername">Username</label>
-            <input type="text" class="form-control" id="inputUsername" name="username" value="<%= username %>" autofocus required>
-          </div>
-          <div class="form-group">
-            <label class="control-label" for="inputPassword">Password</label>
-            <input type="password" class="form-control" name="password" id="inputPassword" value="<%= password %>" required>
-          </div>
-          <div class="form-group">
-            <label class="control-label" for="inputPassword">2FA Token (if enabled)</label>
-            <input type="text" class="form-control" name="totpToken" id="inputTotpToken" value="">
-          </div>
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Sign in"/>
-        </form>
-        <a href="/api/v1/session/password/resetRequest.html">Reset password</a>
-      </div>
-    </div>
-  </div>
-</div>
-
-<script>
-
-(function () {
-    'use strict';
-
-    var search = window.location.search.slice(1).split('&').map(function (item) { return item.split('='); }).reduce(function (o, k) { o[k[0]] = k[1]; return o; }, {});
-
-    document.getElementById('loginForm').action = '/api/v1/session/login?returnTo=' + search.returnTo;
-})();
-
-</script>
-
-<% include footer %>

src/oauth2views/password_reset.ejs

@@ -1,53 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<script>
-
-'use strict';
-
-// very basic angular app
-var app = angular.module('Application', []);
-app.controller('Controller', [function () {}]);
-
-</script>
-
-<div class="layout-content">
-
-  <center>
-    <h2>Hello <%= user.username %>, set a new password</h2>
-  </center>
-
-  <br/>
-
-  <div class="container" ng-app="Application" ng-controller="Controller">
-    <div class="row">
-      <div class="col-md-6 col-md-offset-3">
-        <form action="/api/v1/session/password/reset" method="post" name="resetForm" autocomplete="off" role="form" novalidate>
-          <input type="password" style="display: none;">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <input type="hidden" name="email" value="<%= email %>"/>
-          <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
-
-          <div class="form-group" ng-class="{ 'has-error': resetForm.password.$dirty && resetForm.password.$invalid }">
-            <label class="control-label" for="inputPassword">New Password</label>
-            <div class="control-label" ng-show="resetForm.password.$dirty && resetForm.password.$invalid">
-              <small ng-show="resetForm.password.$dirty && resetForm.password.$invalid">Password must be atleast 8 characters</small>
-            </div>
-            <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^.{8,30}$/" autofocus required>
-          </div>
-          <div class="form-group" ng-class="{ 'has-error': resetForm.passwordRepeat.$dirty && (password !== passwordRepeat) }">
-            <label class="control-label" for="inputPasswordRepeat">Repeat Password</label>
-            <div class="control-label" ng-show="resetForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
-              <small ng-show="resetForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
-            </div>
-            <input type="password" class="form-control" id="inputPasswordRepeat" ng-model="passwordRepeat" name="passwordRepeat" required>
-          </div>
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Set New Password" ng-disabled="resetForm.$invalid || password !== passwordRepeat"/>
-        </form>
-      </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/password_reset_request.ejs

@@ -1,30 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<div class="layout-content">
-
-  <center>
-    <h2>Reset password</h2>
-  </center>
-
-  <br/>
-
-  <div class="container">
-    <div class="row">
-      <div class="col-md-6 col-md-offset-3">
-        <form action="/api/v1/session/password/resetRequest" method="post" autocomplete="off">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <div class="form-group">
-            <label class="control-label" for="inputIdentifier">Username</label>
-            <input type="text" class="form-control" id="inputIdentifier" name="identifier" autofocus required>
-          </div>
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Reset"/>
-        </form>
-        <a href="/api/v1/session/login">Login</a>
-      </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/password_reset_sent.ejs

@@ -1,25 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<div class="layout-content">
-
-  <center>
-    <h2>Password reset successful</h2>
-  </center>
-
-  <br/>
-
-  <div class="container">
-    <div class="row">
-        <div class="col-md-6 col-md-offset-3 text-center">
-          <p>An email was sent to you with a link to set a new password.</p>
-          <br/>
-          <br/>
-          If you have not received any email, simply <a href="/api/v1/session/password/resetRequest.html">try again</a>.
-        </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/paths.js

@@ -22,9 +22,7 @@ exports = module.exports = {
 
     PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'),
     APPS_DATA_DIR: path.join(baseDir(), 'appsdata'),
-    BOX_DATA_DIR: path.join(baseDir(), 'boxdata'),
-
-    CUSTOM_FILE: path.join(baseDir(), 'boxdata/custom.yml'),
+    BOX_DATA_DIR: path.join(baseDir(), 'boxdata'),  // box data dir is part of box backup
 
     ACME_CHALLENGES_DIR: path.join(baseDir(), 'platformdata/acme'),
     ADDON_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons'),
@@ -37,23 +35,25 @@ exports = module.exports = {
     UPDATE_DIR: path.join(baseDir(), 'platformdata/update'),
     SNAPSHOT_INFO_FILE: path.join(baseDir(), 'platformdata/backup/snapshot-info.json'),
     DYNDNS_INFO_FILE: path.join(baseDir(), 'platformdata/dyndns-info.json'),
+    FEATURES_INFO_FILE: path.join(baseDir(), 'platformdata/features-info.json'),
     VERSION_FILE: path.join(baseDir(), 'platformdata/VERSION'),
 
-    SESSION_SECRET_FILE: path.join(baseDir(), 'boxdata/session.secret'),
-    SESSION_DIR: path.join(baseDir(), 'platformdata/sessions'),
-
     // this is not part of appdata because an icon may be set before install
     APP_ICONS_DIR: path.join(baseDir(), 'boxdata/appicons'),
+    PROFILE_ICONS_DIR: path.join(baseDir(), 'boxdata/profileicons'),
     MAIL_DATA_DIR: path.join(baseDir(), 'boxdata/mail'),
     ACME_ACCOUNT_KEY_FILE: path.join(baseDir(), 'boxdata/acme/acme.key'),
     APP_CERTS_DIR: path.join(baseDir(), 'boxdata/certs'),
     CLOUDRON_AVATAR_FILE: path.join(baseDir(), 'boxdata/avatar.png'),
     UPDATE_CHECKER_FILE: path.join(baseDir(), 'boxdata/updatechecker.json'),
+    ADDON_TURN_SECRET_FILE: path.join(baseDir(), 'boxdata/addon-turn-secret'),
 
     LOG_DIR: path.join(baseDir(), 'platformdata/logs'),
     TASKS_LOG_DIR: path.join(baseDir(), 'platformdata/logs/tasks'),
     CRASH_LOG_DIR: path.join(baseDir(), 'platformdata/logs/crash'),
 
+    GHOST_USER_FILE: path.join(baseDir(), 'platformdata/cloudron_ghost.json'),
+
     // this pattern is for the cloudron logs API route to work
     BACKUP_LOG_FILE: path.join(baseDir(), 'platformdata/logs/backup/app.log'),
     UPDATER_LOG_FILE: path.join(baseDir(), 'platformdata/logs/updater/app.log')

src/platform.js

@@ -133,11 +133,10 @@ function pruneInfraImages(callback) {
 function stopContainers(existingInfra, callback) {
     // always stop addons to restart them on any infra change, regardless of minor or major update
     if (existingInfra.version !== infra.version) {
-        // TODO: only nuke containers with isCloudronManaged=true
         debug('stopping all containers for infra upgrade');
         async.series([
-            shell.exec.bind(null, 'stopContainers', 'docker ps -qa --filter \'network=cloudron\' | xargs --no-run-if-empty docker stop'),
-            shell.exec.bind(null, 'stopContainers', 'docker ps -qa --filter \'network=cloudron\' | xargs --no-run-if-empty docker rm -f')
+            shell.exec.bind(null, 'stopContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker stop'),
+            shell.exec.bind(null, 'stopContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker rm -f')
         ], callback);
     } else {
         assert(typeof infra.images, 'object');
@@ -150,8 +149,8 @@ function stopContainers(existingInfra, callback) {
         let filterArg = changedAddons.map(function (c) { return `--filter 'name=${c}'`; }).join(' '); // name=c matches *c*. required for redis-{appid}
         // ignore error if container not found (and fail later) so that this code works across restarts
         async.series([
-            shell.exec.bind(null, 'stopContainers', `docker ps -qa ${filterArg} --filter 'network=cloudron' | xargs --no-run-if-empty docker stop || true`),
-            shell.exec.bind(null, 'stopContainers', `docker ps -qa ${filterArg} --filter 'network=cloudron' | xargs --no-run-if-empty docker rm -f || true`)
+            shell.exec.bind(null, 'stopContainers', `docker ps -qa ${filterArg} --filter 'label=isCloudronManaged' | xargs --no-run-if-empty docker stop || true`),
+            shell.exec.bind(null, 'stopContainers', `docker ps -qa ${filterArg} --filter 'label=isCloudronManaged' | xargs --no-run-if-empty docker rm -f || true`)
         ], callback);
     }
 }

src/provision.js

@@ -15,7 +15,6 @@ var appstore = require('./appstore.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    clients = require('./clients.js'),
     cloudron = require('./cloudron.js'),
     debug = require('debug')('box:provision'),
     domains = require('./domains.js'),
@@ -27,9 +26,9 @@ var appstore = require('./appstore.js'),
     semver = require('semver'),
     settings = require('./settings.js'),
     sysinfo = require('./sysinfo.js'),
-    superagent = require('superagent'),
     users = require('./users.js'),
     tld = require('tldjs'),
+    tokens = require('./tokens.js'),
     _ = require('underscore');
 
 const NOOP_CALLBACK = function (error) { if (error) debug(error); };
@@ -122,7 +121,8 @@ function setup(dnsConfig, sysinfoConfig, auditSource, callback) {
                 provider: dnsConfig.provider,
                 config: dnsConfig.config,
                 fallbackCertificate: dnsConfig.fallbackCertificate || null,
-                tlsConfig: dnsConfig.tlsConfig || { provider: 'letsencrypt-prod' }
+                tlsConfig: dnsConfig.tlsConfig || { provider: 'letsencrypt-prod' },
+                dkimSelector: 'cloudron'
             };
 
             domains.add(domain, data, auditSource, function (error) {
@@ -138,7 +138,6 @@ function setup(dnsConfig, sysinfoConfig, auditSource, callback) {
                         settings.setSysinfoConfig.bind(null, sysinfoConfig),
                         domains.prepareDashboardDomain.bind(null, domain, auditSource, (progress) => setProgress('setup', progress.message, NOOP_CALLBACK)),
                         cloudron.setDashboardDomain.bind(null, domain, auditSource),
-                        mail.addDomain.bind(null, domain), // this relies on settings.mailFqdn() and settings.adminDomain()
                         setProgress.bind(null, 'setup', 'Done'),
                         eventlog.add.bind(null, eventlog.ACTION_PROVISION, auditSource, { })
                     ], function (error) {
@@ -151,31 +150,6 @@ function setup(dnsConfig, sysinfoConfig, auditSource, callback) {
     });
 }
 
-function setTimeZone(ip, callback) {
-    assert.strictEqual(typeof ip, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('setTimeZone ip:%s', ip);
-
-    superagent.get('https://geolocation.cloudron.io/json').query({ ip: ip }).timeout(10 * 1000).end(function (error, result) {
-        if ((error && !error.response) || result.statusCode !== 200) {
-            debug('Failed to get geo location: %s', error.message);
-            return callback(null);
-        }
-
-        var timezone = safe.query(result.body, 'location.time_zone');
-
-        if (!timezone || typeof timezone !== 'string') {
-            debug('No timezone in geoip response : %j', result.body);
-            return callback(null);
-        }
-
-        debug('Setting timezone to ', timezone);
-
-        settings.setTimeZone(timezone, callback);
-    });
-}
-
 function activate(username, password, email, displayName, ip, auditSource, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
@@ -187,13 +161,11 @@ function activate(username, password, email, displayName, ip, auditSource, callb
 
     debug('activating user:%s email:%s', username, email);
 
-    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
-
     users.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
         if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(new BoxError(BoxError.CONFLICT, 'Already activated'));
         if (error) return callback(error);
 
-        clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+        tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
             if (error) return callback(error);
 
             eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
@@ -218,7 +190,7 @@ function restore(backupConfig, backupId, version, sysinfoConfig, auditSource, ca
     assert.strictEqual(typeof callback, 'function');
 
     if (!semver.valid(version)) return callback(new BoxError(BoxError.BAD_FIELD, 'version is not a valid semver', { field: 'version' }));
-    if (semver.major(constants.VERSION) !== semver.major(version) || semver.minor(constants.VERSION) !== semver.minor(version)) return callback(new BoxError(BoxError.BAD_STATE, `Run cloudron-setup with --version ${version} to restore from this backup`));
+    if (constants.VERSION !== version) return callback(new BoxError(BoxError.BAD_STATE, `Run cloudron-setup with --version ${version} to restore from this backup`));
 
     if (gProvisionStatus.setup.active || gProvisionStatus.restore.active) return callback(new BoxError(BoxError.BAD_STATE, 'Already setting up or restoring'));
 
@@ -249,7 +221,7 @@ function restore(backupConfig, backupId, version, sysinfoConfig, auditSource, ca
                     backups.restore.bind(null, backupConfig, backupId, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)),
                     settings.setSysinfoConfig.bind(null, sysinfoConfig),
                     cloudron.setupDashboard.bind(null, auditSource, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)),
-                    settings.setBackupConfig.bind(null, backupConfig), // update with the latest backupConfig
+                    settings.setBackupCredentials.bind(null, backupConfig), // update just the credentials and not the policy and flags
                     eventlog.add.bind(null, eventlog.ACTION_RESTORE, auditSource, { backupId }),
                 ], function (error) {
                     gProvisionStatus.restore.active = false;
@@ -268,14 +240,16 @@ function getStatus(callback) {
     users.isActivated(function (error, activated) {
         if (error) return callback(error);
 
-        settings.getCloudronName(function (error, cloudronName) {
+        settings.getAll(function (error, allSettings) {
             if (error) return callback(error);
 
             callback(null, _.extend({
                 version: constants.VERSION,
                 apiServerOrigin: settings.apiServerOrigin(), // used by CaaS tool
+                webServerOrigin: settings.webServerOrigin(), // used by CaaS tool
                 provider: settings.provider(),
-                cloudronName: cloudronName,
+                cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
+                footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
                 adminFqdn: settings.adminDomain() ? settings.adminFqdn() : null,
                 activated: activated,
             }, gProvisionStatus));

src/reverseproxy.js

@@ -79,7 +79,7 @@ function getCertApi(domainObject, callback) {
     // we simply update the account with the latest email we have each time when getting letsencrypt certs
     // https://github.com/ietf-wg-acme/acme/issues/30
     users.getOwner(function (error, owner) {
-        options.email = error ? 'support@cloudron.io' : (owner.fallbackEmail || owner.email); // can error if not activated yet
+        options.email = error ? 'support@cloudron.io' : owner.email; // can error if not activated yet
 
         callback(null, api, options);
     });
@@ -146,19 +146,19 @@ function validateCertificate(location, domainObject, certificate) {
     // -checkhost checks for SAN or CN exclusively. SAN takes precedence and if present, ignores the CN.
     const fqdn = domains.fqdn(location, domainObject);
 
-    var result = safe.child_process.execSync(`openssl x509 -noout -checkhost "${fqdn}"`, { encoding: 'utf8', input: cert });
+    let result = safe.child_process.execSync(`openssl x509 -noout -checkhost "${fqdn}"`, { encoding: 'utf8', input: cert });
     if (result === null) return new BoxError(BoxError.BAD_FIELD, 'Unable to get certificate subject:' + safe.error.message, { field: 'cert' });
 
     if (result.indexOf('does match certificate') === -1) return new BoxError(BoxError.BAD_FIELD, `Certificate is not valid for this domain. Expecting ${fqdn}`, { field: 'cert' });
 
-    // http://httpd.apache.org/docs/2.0/ssl/ssl_faq.html#verify
-    var certModulus = safe.child_process.execSync('openssl x509 -noout -modulus', { encoding: 'utf8', input: cert });
-    if (certModulus === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get cert modulus: ${safe.error.message}`, { field: 'cert' });
+    // check if public key in the cert and private key matches. pkey below works for RSA and ECDSA keys
+    const pubKeyFromCert = safe.child_process.execSync('openssl x509 -noout -pubkey', { encoding: 'utf8', input: cert });
+    if (pubKeyFromCert === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from cert: ${safe.error.message}`, { field: 'cert' });
 
-    var keyModulus = safe.child_process.execSync('openssl rsa -noout -modulus', { encoding: 'utf8', input: key });
-    if (keyModulus === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get key modulus: ${safe.error.message}`, { field: 'cert' });
+    const pubKeyFromKey = safe.child_process.execSync('openssl pkey -pubout', { encoding: 'utf8', input: key });
+    if (pubKeyFromKey === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from private key: ${safe.error.message}`, { field: 'cert' });
 
-    if (certModulus !== keyModulus) return new BoxError(BoxError.BAD_FIELD, 'Key does not match the certificate.', { field: 'cert' });
+    if (pubKeyFromCert !== pubKeyFromKey) return new BoxError(BoxError.BAD_FIELD, 'Public key does not match the certificate.', { field: 'cert' });
 
     // check expiration
     result = safe.child_process.execSync('openssl x509 -checkend 0', { encoding: 'utf8', input: cert });
@@ -171,7 +171,7 @@ function reload(callback) {
     if (process.env.BOX_ENV === 'test') return callback();
 
     shell.sudo('reload', [ RELOAD_NGINX_CMD ], {}, function (error) {
-        if (error) return callback(new BoxError(BoxError.NGINX_ERROR, error));
+        if (error) return callback(new BoxError(BoxError.NGINX_ERROR, `Error reloading nginx: ${error.message}`));
 
         callback();
     });
@@ -346,7 +346,7 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
                 debug('ensureCertificate: getting certificate for %s with options %j', vhost, apiOptions);
 
                 api.getCertificate(vhost, domain, apiOptions, function (error, certFilePath, keyFilePath) {
-                    debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath}`);
+                    debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath || 'null'}`);
 
                     eventlog.add(currentBundle ? eventlog.ACTION_CERTIFICATE_RENEWAL : eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: vhost, errorMessage: error ? error.message : '' });
 
@@ -582,6 +582,8 @@ function renewCerts(options, auditSource, progressCallback, callback) {
 
         // add app main
         allApps.forEach(function (app) {
+            if (app.runState === apps.RSTATE_STOPPED) return; // do not renew certs of stopped apps
+
             appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'main', app: app, nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf') });
 
             app.alternateDomains.forEach(function (alternateDomain) {

src/routes/accesscontrol.js

@@ -1,138 +1,134 @@
 'use strict';
 
 exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize,
+    passwordAuth: passwordAuth,
+    tokenAuth: tokenAuth,
 
-    scope: scope,
+    authorize: authorize,
     websocketAuth: websocketAuth
 };
 
 var accesscontrol = require('../accesscontrol.js'),
     assert = require('assert'),
-    BasicStrategy = require('passport-http').BasicStrategy,
-    BearerStrategy = require('passport-http-bearer').Strategy,
     BoxError = require('../boxerror.js'),
-    clients = require('../clients.js'),
-    ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    externalLdap = require('../externalldap.js'),
     HttpError = require('connect-lastmile').HttpError,
-    LocalStrategy = require('passport-local').Strategy,
-    passport = require('passport'),
-    users = require('../users.js');
+    users = require('../users.js'),
+    speakeasy = require('speakeasy');
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function passwordAuth(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
 
-    // serialize user into session
-    passport.serializeUser(function (user, callback) {
-        callback(null, user.id);
-    });
+    if (!req.body.username || typeof req.body.username !== 'string') return next(new HttpError(400, 'A username must be non-empty string'));
+    if (!req.body.password || typeof req.body.password !== 'string') return next(new HttpError(400, 'A password must be non-empty string'));
 
-    // deserialize user from session
-    passport.deserializeUser(function(userId, callback) {
-        users.get(userId, function (error, result) {
-            if (error) return callback(error);
+    const username = req.body.username;
+    const password = req.body.password;
 
-            callback(null, result);
-        });
-    });
+    function check2FA(user) {
+        assert.strictEqual(typeof user, 'object');
 
-    // used when username/password is sent in request body. used in CLI tool login route
-    passport.use(new LocalStrategy(function (username, password, callback) {
-        if (username.indexOf('@') === -1) {
-            users.verifyWithUsername(username, password, function (error, result) {
-                if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === BoxError.INVALID_CREDENTIALS) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, result);
-            });
-        } else {
-            users.verifyWithEmail(username, password, function (error, result) {
-                if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === BoxError.INVALID_CREDENTIALS) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, result);
-            });
+        if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
+            if (!req.body.totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
+
+            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
+            if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
         }
-    }));
-
-    // Used to authenticate a OAuth2 client which uses clientId and clientSecret in the Authorization header
-    passport.use(new BasicStrategy(function (clientId, clientSecret, callback) {
-        clients.get(clientId, function (error, client) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-            if (error) return callback(error);
-            if (client.clientSecret !== clientSecret) return callback(null, false);
-            callback(null, client);
-        });
-    }));
-
-    // Used to authenticate a OAuth2 client which uses clientId and clientSecret in the request body (client_id, client_secret)
-    passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clients.get(clientId, function(error, client) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-            if (error) { return callback(error); }
-            if (client.clientSecret !== clientSecret) { return callback(null, false); }
-            callback(null, client);
-        });
-    }));
-
-    // used for "Authorization: Bearer token" or access_token query param authentication
-    passport.use(new BearerStrategy(function (token, callback) {
-        accesscontrol.validateToken(token, callback);
-    }));
-
-    callback(null);
-}
-
-function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    callback(null);
-}
-
-//  The scope middleware provides an auth middleware for routes.
-//
-//  It is used for API routes, which are authenticated using accesstokens.
-//  Those accesstokens carry OAuth scopes and the middleware takes the required
-//  scope as an argument and will verify the accesstoken against it.
-//
-//  See server.js:
-//    var profileScope = routes.oauth2.scope('profile');
-//
-function scope(requiredScope) {
-    assert.strictEqual(typeof requiredScope, 'string');
-
-    var requiredScopes = requiredScope.split(',');
-
-    return [
-        passport.authenticate(['bearer'], { session: false }),
-
-        function (req, res, next) {
-            assert(req.authInfo && typeof req.authInfo === 'object');
-
-            var error = accesscontrol.hasScopes(req.authInfo.authorizedScopes, requiredScopes);
-            if (error) return next(new HttpError(403, error.message));
-
-            next();
-        }
-    ];
-}
-
-function websocketAuth(requiredScopes, req, res, next) {
-    assert(Array.isArray(requiredScopes));
-
-    if (typeof req.query.access_token !== 'string') return next(new HttpError(401, 'Unauthorized'));
-
-    accesscontrol.validateToken(req.query.access_token, function (error, user, info) {
-        if (error) return next(new HttpError(500, error.message));
-        if (!user) return next(new HttpError(401, 'Unauthorized'));
 
         req.user = user;
 
-        var e = accesscontrol.hasScopes(info.authorizedScopes, requiredScopes);
-        if (e) return next(new HttpError(403, e.message));
+        next();
+    }
+
+    function createAndVerifyUserIfNotExist(identifier, password) {
+        assert.strictEqual(typeof identifier, 'string');
+        assert.strictEqual(typeof password, 'string');
+
+        externalLdap.createAndVerifyUserIfNotExist(identifier.toLowerCase(), password, function (error, result) {
+            if (error && error.reason === BoxError.BAD_STATE) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.CONFLICT) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.NOT_FOUND) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+            if (error) return next(new HttpError(500, error));
+
+            check2FA(result);
+        });
+    }
+
+    if (username.indexOf('@') === -1) {
+        users.verifyWithUsername(username, password, users.AP_WEBADMIN, function (error, result) {
+            if (error && error.reason === BoxError.NOT_FOUND) return createAndVerifyUserIfNotExist(username, password);
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+            if (error) return next(new HttpError(500, error));
+            if (!result) return next(new HttpError(401, 'Unauthorized'));
+
+            check2FA(result);
+        });
+    } else {
+        users.verifyWithEmail(username, password, users.AP_WEBADMIN, function (error, result) {
+            if (error && error.reason === BoxError.NOT_FOUND) return createAndVerifyUserIfNotExist(username, password);
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+            if (error) return next(new HttpError(500, error));
+            if (!result) return next(new HttpError(401, 'Unauthorized'));
+
+            check2FA(result);
+        });
+    }
+}
+
+function tokenAuth(req, res, next) {
+    var token;
+
+    // this determines the priority
+    if (req.body && req.body.access_token) token = req.body.access_token;
+    if (req.query && req.query.access_token) token = req.query.access_token;
+    if (req.headers && req.headers.authorization) {
+        var parts = req.headers.authorization.split(' ');
+        if (parts.length == 2) {
+            var scheme = parts[0];
+            var credentials = parts[1];
+
+            if (/^Bearer$/i.test(scheme)) token = credentials;
+        }
+    }
+
+    if (!token) return next(new HttpError(401, 'Unauthorized'));
+
+    accesscontrol.verifyToken(token, function (error, user) {
+        if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+        if (error) return next(new HttpError(500, error.message));
+
+        req.user = user;
+
+        next();
+    });
+}
+
+function authorize(requiredRole) {
+    assert.strictEqual(typeof requiredRole, 'string');
+
+    return function (req, res, next) {
+        assert.strictEqual(typeof req.user, 'object');
+
+        if (users.compareRoles(req.user.role, requiredRole) < 0) return next(new HttpError(403, `role '${requiredRole}' is required but user has only '${req.user.role}'`));
+
+        next();
+    };
+}
+
+function websocketAuth(requiredRole, req, res, next) {
+    assert.strictEqual(typeof requiredRole, 'string');
+
+    if (typeof req.query.access_token !== 'string') return next(new HttpError(401, 'Unauthorized'));
+
+    accesscontrol.verifyToken(req.query.access_token, function (error, user) {
+        if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+        if (error) return next(new HttpError(500, error.message));
+
+        req.user = user;
+
+        if (users.compareRoles(req.user.role, requiredRole) < 0) return next(new HttpError(403, `role '${requiredRole}' is required but user has only '${user.role}'`));
 
         next();
     });

src/routes/apppasswords.js

@@ -0,0 +1,61 @@
+'use strict';
+
+exports = module.exports = {
+    list: list,
+    get: get,
+    del: del,
+    add: add
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    users = require('../users.js');
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    users.getAppPassword(req.params.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function add(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+    if (typeof req.body.identifier !== 'string') return next(new HttpError(400, 'identifier must be string'));
+
+    users.addAppPassword(req.user.id, req.body.identifier, req.body.name, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(201, result));
+    });
+}
+
+function list(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    users.getAppPasswords(req.user.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { appPasswords: result }));
+    });
+}
+
+function del(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    // TODO: verify userId owns the id ?
+    users.delAppPassword(req.params.id, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(204, {}));
+    });
+}

src/routes/apps.js

@@ -4,22 +4,23 @@ exports = module.exports = {
     getApp: getApp,
     getApps: getApps,
     getAppIcon: getAppIcon,
-    installApp: installApp,
-    uninstallApp: uninstallApp,
-    restoreApp: restoreApp,
+    install: install,
+    uninstall: uninstall,
+    restore: restore,
     importApp: importApp,
-    backupApp: backupApp,
-    updateApp: updateApp,
+    backup: backup,
+    update: update,
     getLogs: getLogs,
     getLogStream: getLogStream,
     listBackups: listBackups,
-    repairApp: repairApp,
+    repair: repair,
 
     setAccessRestriction: setAccessRestriction,
     setLabel: setLabel,
     setTags: setTags,
     setIcon: setIcon,
     setMemoryLimit: setMemoryLimit,
+    setCpuShares: setCpuShares,
     setAutomaticBackup: setAutomaticBackup,
     setAutomaticUpdate: setAutomaticUpdate,
     setReverseProxyConfig: setReverseProxyConfig,
@@ -30,15 +31,18 @@ exports = module.exports = {
     setLocation: setLocation,
     setDataDir: setDataDir,
 
-    stopApp: stopApp,
-    startApp: startApp,
+    stop: stop,
+    start: start,
+    restart: restart,
     exec: exec,
     execWebSocket: execWebSocket,
 
-    cloneApp: cloneApp,
+    clone: clone,
 
     uploadFile: uploadFile,
-    downloadFile: downloadFile
+    downloadFile: downloadFile,
+
+    load: load
 };
 
 var apps = require('../apps.js'),
@@ -49,19 +53,28 @@ var apps = require('../apps.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     safe = require('safetydance'),
+    users = require('../users.js'),
     util = require('util'),
     WebSocket = require('ws');
 
-function getApp(req, res, next) {
+function load(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
-    apps.get(req.params.id, function (error, app) {
+    apps.get(req.params.id, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
-        next(new HttpSuccess(200, apps.removeInternalFields(app)));
+        req.resource = result;
+
+        next();
     });
 }
 
+function getApp(req, res, next) {
+    assert.strictEqual(typeof req.resource, 'object');
+
+    next(new HttpSuccess(200, apps.removeInternalFields(req.resource)));
+}
+
 function getApps(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
@@ -75,19 +88,19 @@ function getApps(req, res, next) {
 }
 
 function getAppIcon(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
-    apps.getIconPath(req.params.id, { original: req.query.original }, function (error, iconPath) {
+    apps.getIconPath(req.resource, { original: req.query.original }, function (error, iconPath) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.sendFile(iconPath);
     });
 }
 
-function installApp(req, res, next) {
+function install(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    var data = req.body;
+    const data = req.body;
 
     // atleast one
     if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
@@ -131,22 +144,28 @@ function installApp(req, res, next) {
 
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
 
-    debug('Installing app :%j', data);
-
-    apps.install(data, req.user, auditSource.fromRequest(req), function (error, result) {
+    apps.downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
         if (error) return next(BoxError.toHttpError(error));
 
-        next(new HttpSuccess(202, { id: result.id, taskId: result.taskId }));
+        if (safe.query(manifest, 'addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is required to install app with docker addon'));
+
+        data.appStoreId = appStoreId;
+        data.manifest = manifest;
+        apps.install(data, auditSource.fromRequest(req), function (error, result) {
+            if (error) return next(BoxError.toHttpError(error));
+
+            next(new HttpSuccess(202, { id: result.id, taskId: result.taskId }));
+        });
     });
 }
 
 function setAccessRestriction(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (typeof req.body.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction must be an object'));
 
-    apps.setAccessRestriction(req.params.id, req.body.accessRestriction, auditSource.fromRequest(req), function (error) {
+    apps.setAccessRestriction(req.resource, req.body.accessRestriction, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -155,11 +174,11 @@ function setAccessRestriction(req, res, next) {
 
 function setLabel(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (typeof req.body.label !== 'string') return next(new HttpError(400, 'label must be a string'));
 
-    apps.setLabel(req.params.id, req.body.label, auditSource.fromRequest(req), function (error) {
+    apps.setLabel(req.resource, req.body.label, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -168,12 +187,12 @@ function setLabel(req, res, next) {
 
 function setTags(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (!Array.isArray(req.body.tags)) return next(new HttpError(400, 'tags must be an array'));
     if (req.body.tags.some((t) => typeof t !== 'string')) return next(new HttpError(400, 'tags array must contain strings'));
 
-    apps.setTags(req.params.id, req.body.tags, auditSource.fromRequest(req), function (error) {
+    apps.setTags(req.resource, req.body.tags, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -182,11 +201,11 @@ function setTags(req, res, next) {
 
 function setIcon(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (req.body.icon !== null && typeof req.body.icon !== 'string') return next(new HttpError(400, 'icon is null or a base-64 image string'));
 
-    apps.setIcon(req.params.id, req.body.icon, auditSource.fromRequest(req), function (error) {
+    apps.setIcon(req.resource, req.body.icon, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -195,11 +214,24 @@ function setIcon(req, res, next) {
 
 function setMemoryLimit(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (typeof req.body.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
 
-    apps.setMemoryLimit(req.params.id, req.body.memoryLimit, auditSource.fromRequest(req), function (error, result) {
+    apps.setMemoryLimit(req.resource, req.body.memoryLimit, auditSource.fromRequest(req), function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, { taskId: result.taskId }));
+    });
+}
+
+function setCpuShares(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.resource, 'object');
+
+    if (typeof req.body.cpuShares !== 'number') return next(new HttpError(400, 'cpuShares is not a number'));
+
+    apps.setCpuShares(req.resource, req.body.cpuShares, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -208,11 +240,11 @@ function setMemoryLimit(req, res, next) {
 
 function setAutomaticBackup(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (typeof req.body.enable !== 'boolean') return next(new HttpError(400, 'enable must be a boolean'));
 
-    apps.setAutomaticBackup(req.params.id, req.body.enable, auditSource.fromRequest(req), function (error) {
+    apps.setAutomaticBackup(req.resource, req.body.enable, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -221,11 +253,11 @@ function setAutomaticBackup(req, res, next) {
 
 function setAutomaticUpdate(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (typeof req.body.enable !== 'boolean') return next(new HttpError(400, 'enable must be a boolean'));
 
-    apps.setAutomaticUpdate(req.params.id, req.body.enable, auditSource.fromRequest(req), function (error) {
+    apps.setAutomaticUpdate(req.resource, req.body.enable, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -234,13 +266,13 @@ function setAutomaticUpdate(req, res, next) {
 
 function setReverseProxyConfig(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (req.body.robotsTxt !== null && typeof req.body.robotsTxt !== 'string') return next(new HttpError(400, 'robotsTxt is not a string'));
 
     if (req.body.csp !== null && typeof req.body.csp !== 'string') return next(new HttpError(400, 'csp is not a string'));
 
-    apps.setReverseProxyConfig(req.params.id, req.body, auditSource.fromRequest(req), function (error) {
+    apps.setReverseProxyConfig(req.resource, req.body, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -249,14 +281,14 @@ function setReverseProxyConfig(req, res, next) {
 
 function setCertificate(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (req.body.key !== null && typeof req.body.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
     if (req.body.cert !== null && typeof req.body.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (req.body.cert && !req.body.key) return next(new HttpError(400, 'key must be provided'));
     if (!req.body.cert && req.body.key) return next(new HttpError(400, 'cert must be provided'));
 
-    apps.setCertificate(req.params.id, req.body, auditSource.fromRequest(req), function (error) {
+    apps.setCertificate(req.resource, req.body, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -265,12 +297,12 @@ function setCertificate(req, res, next) {
 
 function setEnvironment(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (!req.body.env || typeof req.body.env !== 'object') return next(new HttpError(400, 'env must be an object'));
     if (Object.keys(req.body.env).some((key) => typeof req.body.env[key] !== 'string')) return next(new HttpError(400, 'env must contain values as strings'));
 
-    apps.setEnvironment(req.params.id, req.body.env, auditSource.fromRequest(req), function (error, result) {
+    apps.setEnvironment(req.resource, req.body.env, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -279,11 +311,11 @@ function setEnvironment(req, res, next) {
 
 function setDebugMode(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (req.body.debugMode !== null && typeof req.body.debugMode !== 'object') return next(new HttpError(400, 'debugMode must be an object'));
 
-    apps.setDebugMode(req.params.id, req.body.debugMode, auditSource.fromRequest(req), function (error, result) {
+    apps.setDebugMode(req.resource, req.body.debugMode, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -292,12 +324,12 @@ function setDebugMode(req, res, next) {
 
 function setMailbox(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (req.body.mailboxName !== null && typeof req.body.mailboxName !== 'string') return next(new HttpError(400, 'mailboxName must be a string'));
     if (typeof req.body.mailboxDomain !== 'string') return next(new HttpError(400, 'mailboxDomain must be a string'));
 
-    apps.setMailbox(req.params.id, req.body.mailboxName, req.body.mailboxDomain, auditSource.fromRequest(req), function (error, result) {
+    apps.setMailbox(req.resource, req.body.mailboxName, req.body.mailboxDomain, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -306,7 +338,7 @@ function setMailbox(req, res, next) {
 
 function setLocation(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (typeof req.body.location !== 'string') return next(new HttpError(400, 'location must be string')); // location may be an empty string
     if (!req.body.domain) return next(new HttpError(400, 'domain is required'));
@@ -321,7 +353,7 @@ function setLocation(req, res, next) {
 
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
 
-    apps.setLocation(req.params.id, req.body, auditSource.fromRequest(req), function (error, result) {
+    apps.setLocation(req.resource, req.body, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -330,57 +362,49 @@ function setLocation(req, res, next) {
 
 function setDataDir(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (req.body.dataDir !== null && typeof req.body.dataDir !== 'string') return next(new HttpError(400, 'dataDir must be a string'));
 
-    apps.setDataDir(req.params.id, req.body.dataDir, auditSource.fromRequest(req), function (error, result) {
+    apps.setDataDir(req.resource, req.body.dataDir, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function repairApp(req, res, next) {
+function repair(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    debug('Repair app id:%s', req.params.id);
+    assert.strictEqual(typeof req.resource, 'object');
 
     const data = req.body;
 
-    if (data.backupId && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
-    if (data.backupFormat && typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string or null'));
+    if ('manifest' in data) {
+        if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
 
-    if (data.location && typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (data.domain && typeof data.domain !== 'string') return next(new HttpError(400, 'domain is required'));
-
-    if ('alternateDomains' in data) {
-        if (!Array.isArray(data.alternateDomains)) return next(new HttpError(400, 'alternateDomains must be an array'));
-        if (data.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
+        if (safe.query(data.manifest, 'addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is required to repair app with docker addon'));
     }
 
-    if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
+    if ('dockerImage' in data) {
+        if (!data.dockerImage || typeof data.dockerImage !== 'string') return next(new HttpError(400, 'dockerImage must be a string'));
+    }
 
-    apps.repair(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
+    apps.repair(req.resource, data, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function restoreApp(req, res, next) {
+function restore(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     var data = req.body;
 
-    debug('Restore app id:%s', req.params.id);
+    if (!data.backupId || typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be non-empty string'));
 
-    if (!('backupId' in req.body)) return next(new HttpError(400, 'backupId is required'));
-    if (data.backupId !== null && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
-
-    apps.restore(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
+    apps.restore(req.resource, data.backupId, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -389,30 +413,41 @@ function restoreApp(req, res, next) {
 
 function importApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     var data = req.body;
 
-    debug('Importing app id:%s', req.params.id);
+    if ('backupId' in data) { // if not provided, we import in-place
+        if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string'));
+        if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
 
-    if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string'));
-    if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
+        if ('backupConfig' in data && typeof data.backupConfig !== 'object') return next(new HttpError(400, 'backupConfig must be an object'));
 
-    apps.importApp(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
+        const backupConfig = req.body.backupConfig;
+
+        if (req.body.backupConfig) {
+            if (typeof backupConfig.provider !== 'string') return next(new HttpError(400, 'provider is required'));
+            if ('key' in backupConfig && typeof backupConfig.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+            if ('acceptSelfSignedCerts' in backupConfig && typeof backupConfig.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
+
+            // testing backup config can take sometime
+            req.clearTimeout();
+        }
+    }
+
+    apps.importApp(req.resource, data, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function cloneApp(req, res, next) {
+function clone(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     var data = req.body;
 
-    debug('Clone app id:%s', req.params.id);
-
     if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be a string'));
     if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
     if (typeof data.domain !== 'string') return next(new HttpError(400, 'domain is required'));
@@ -420,64 +455,66 @@ function cloneApp(req, res, next) {
 
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
 
-    apps.clone(req.params.id, data, req.user, auditSource.fromRequest(req), function (error, result) {
+    apps.clone(req.resource, data, req.user, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(201, { id: result.id, taskId: result.taskId }));
     });
 }
 
-function backupApp(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
+function backup(req, res, next) {
+    assert.strictEqual(typeof req.resource, 'object');
 
-    debug('Backup app id:%s', req.params.id);
-
-    apps.backup(req.params.id, function (error, result) {
+    apps.backup(req.resource, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function uninstallApp(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
+function uninstall(req, res, next) {
+    assert.strictEqual(typeof req.resource, 'object');
 
-    debug('Uninstalling app id:%s', req.params.id);
-
-    apps.uninstall(req.params.id, auditSource.fromRequest(req), function (error, result) {
+    apps.uninstall(req.resource, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function startApp(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
+function start(req, res, next) {
+    assert.strictEqual(typeof req.resource, 'object');
 
-    debug('Start app id:%s', req.params.id);
-
-    apps.start(req.params.id, function (error, result) {
+    apps.start(req.resource, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function stopApp(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
+function stop(req, res, next) {
+    assert.strictEqual(typeof req.resource, 'object');
 
-    debug('Stop app id:%s', req.params.id);
-
-    apps.stop(req.params.id, function (error, result) {
+    apps.stop(req.resource, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function updateApp(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
+function restart(req, res, next) {
+    assert.strictEqual(typeof req.resource, 'object');
+
+    apps.restart(req.resource, auditSource.fromRequest(req), function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, { taskId: result.taskId }));
+    });
+}
+
+function update(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.resource, 'object');
 
     var data = req.body;
 
@@ -489,20 +526,24 @@ function updateApp(req, res, next) {
     if ('skipBackup' in data && typeof data.skipBackup !== 'boolean') return next(new HttpError(400, 'skipBackup must be a boolean'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
-    debug('Update app id:%s to manifest:%j', req.params.id, data.manifest);
-
-    apps.update(req.params.id, req.body, auditSource.fromRequest(req), function (error, result) {
+    apps.downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
         if (error) return next(BoxError.toHttpError(error));
 
-        next(new HttpSuccess(202, { taskId: result.taskId }));
+        if (safe.query(manifest, 'addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is required to update app with docker addon'));
+
+        data.appStoreId = appStoreId;
+        data.manifest = manifest;
+        apps.update(req.resource, data, auditSource.fromRequest(req), function (error, result) {
+            if (error) return next(BoxError.toHttpError(error));
+
+            next(new HttpSuccess(202, { taskId: result.taskId }));
+        });
     });
 }
 
 // this route is for streaming logs
 function getLogStream(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    debug('Getting logstream of app id:%s', req.params.id);
+    assert.strictEqual(typeof req.resource, 'object');
 
     var lines = 'lines' in req.query ? parseInt(req.query.lines, 10) : 10; // we ignore last-event-id
     if (isNaN(lines)) return next(new HttpError(400, 'lines must be a valid number'));
@@ -517,7 +558,7 @@ function getLogStream(req, res, next) {
         format: 'json'
     };
 
-    apps.getLogs(req.params.id, options, function (error, logStream) {
+    apps.getLogs(req.resource, options, function (error, logStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.writeHead(200, {
@@ -539,20 +580,18 @@ function getLogStream(req, res, next) {
 }
 
 function getLogs(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     var lines = 'lines' in req.query ? parseInt(req.query.lines, 10) : 10;
     if (isNaN(lines)) return next(new HttpError(400, 'lines must be a number'));
 
-    debug('Getting logs of app id:%s', req.params.id);
-
     var options = {
         lines: lines,
         follow: false,
         format: req.query.format || 'json'
     };
 
-    apps.getLogs(req.params.id, options, function (error, logStream) {
+    apps.getLogs(req.resource, options, function (error, logStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.writeHead(200, {
@@ -588,9 +627,7 @@ function demuxStream(stream, stdin) {
 }
 
 function exec(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    debug('Execing into app id:%s and cmd:%s', req.params.id, req.query.cmd);
+    assert.strictEqual(typeof req.resource, 'object');
 
     var cmd = null;
     if (req.query.cmd) {
@@ -604,9 +641,11 @@ function exec(req, res, next) {
     var rows = req.query.rows ? parseInt(req.query.rows, 10) : null;
     if (isNaN(rows)) return next(new HttpError(400, 'rows must be a number'));
 
-    var tty = req.query.tty === 'true' ? true : false;
+    var tty = req.query.tty === 'true';
 
-    apps.exec(req.params.id, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) {
+    if (safe.query(req.resource, 'manifest.addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is requied to exec app with docker addon'));
+
+    apps.exec(req.resource, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         if (req.headers['upgrade'] !== 'tcp') return next(new HttpError(404, 'exec requires TCP upgrade'));
@@ -628,9 +667,7 @@ function exec(req, res, next) {
 }
 
 function execWebSocket(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    debug('Execing websocket into app id:%s and cmd:%s', req.params.id, req.query.cmd);
+    assert.strictEqual(typeof req.resource, 'object');
 
     var cmd = null;
     if (req.query.cmd) {
@@ -646,11 +683,9 @@ function execWebSocket(req, res, next) {
 
     var tty = req.query.tty === 'true' ? true : false;
 
-    apps.exec(req.params.id, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) {
+    apps.exec(req.resource, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) {
         if (error) return next(BoxError.toHttpError(error));
 
-        debug('Connected to terminal');
-
         req.clearTimeout();
 
         res.handleUpgrade(function (ws) {
@@ -678,7 +713,7 @@ function execWebSocket(req, res, next) {
 }
 
 function listBackups(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.resource, 'object');
 
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
     if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
@@ -686,7 +721,7 @@ function listBackups(req, res, next) {
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
     if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
 
-    apps.listBackups(page, perPage, req.params.id, function (error, result) {
+    apps.listBackups(req.resource, page, perPage, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { backups: result }));
@@ -694,30 +729,24 @@ function listBackups(req, res, next) {
 }
 
 function uploadFile(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    debug('uploadFile: %s %j -> %s', req.params.id, req.files, req.query.file);
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (typeof req.query.file !== 'string' || !req.query.file) return next(new HttpError(400, 'file query argument must be provided'));
     if (!req.files.file) return next(new HttpError(400, 'file must be provided as multipart'));
 
-    apps.uploadFile(req.params.id, req.files.file.path, req.query.file, function (error) {
+    apps.uploadFile(req.resource, req.files.file.path, req.query.file, function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
-        debug('uploadFile: done');
-
         next(new HttpSuccess(202, {}));
     });
 }
 
 function downloadFile(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    debug('downloadFile: ', req.params.id, req.query.file);
+    assert.strictEqual(typeof req.resource, 'object');
 
     if (typeof req.query.file !== 'string' || !req.query.file) return next(new HttpError(400, 'file query argument must be provided'));
 
-    apps.downloadFile(req.params.id, req.query.file, function (error, stream, info) {
+    apps.downloadFile(req.resource, req.query.file, function (error, stream, info) {
         if (error) return next(BoxError.toHttpError(error));
 
         var headers = {

src/routes/appstore.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     getApp: getApp,
     getAppVersion: getAppVersion,
 
+    createUserToken: createUserToken,
     registerCloudron: registerCloudron,
     getSubscription: getSubscription
 };
@@ -12,35 +13,20 @@ exports = module.exports = {
 var appstore = require('../appstore.js'),
     assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    custom = require('../custom.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
-function isAppAllowed(appstoreId) {
-    if (custom.spec().appstore.blacklist.includes(appstoreId)) return false;
-
-    if (!custom.spec().appstore.whitelist) return true;
-    if (!custom.spec().appstore.whitelist[appstoreId]) return false;
-
-    return true;
-}
-
 function getApps(req, res, next) {
     appstore.getApps(function (error, apps) {
         if (error) return next(BoxError.toHttpError(error));
 
-        let filteredApps = apps.filter((app) => !custom.spec().appstore.blacklist.includes(app.id));
-        if (custom.spec().appstore.whitelist) filteredApps = filteredApps.filter((app) => app.id in custom.spec().appstore.whitelist);
-
-        next(new HttpSuccess(200, { apps: filteredApps }));
+        next(new HttpSuccess(200, { apps }));
     });
 }
 
 function getApp(req, res, next) {
     assert.strictEqual(typeof req.params.appstoreId, 'string');
 
-    if (!isAppAllowed(req.params.appstoreId)) return next(new HttpError(405, 'feature disabled by admin'));
-
     appstore.getApp(req.params.appstoreId, function (error, app) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -52,8 +38,6 @@ function getAppVersion(req, res, next) {
     assert.strictEqual(typeof req.params.appstoreId, 'string');
     assert.strictEqual(typeof req.params.versionId, 'string');
 
-    if (!isAppAllowed(req.params.appstoreId)) return next(new HttpError(405, 'feature disabled by admin'));
-
     appstore.getAppVersion(req.params.appstoreId, req.params.versionId, function (error, manifest) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -61,6 +45,14 @@ function getAppVersion(req, res, next) {
     });
 }
 
+function createUserToken(req, res, next) {
+    appstore.getUserToken(function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(201, { accessToken: result }));
+    });
+}
+
 function registerCloudron(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
@@ -68,6 +60,7 @@ function registerCloudron(req, res, next) {
     if (typeof req.body.password !== 'string' || !req.body.password) return next(new HttpError(400, 'password must be string'));
     if ('totpToken' in req.body && typeof req.body.totpToken !== 'string') return next(new HttpError(400, 'totpToken must be string'));
     if (typeof req.body.signup !== 'boolean') return next(new HttpError(400, 'signup must be a boolean'));
+    if ('purpose' in req.body && typeof req.body.purpose !== 'string') return next(new HttpError(400, 'purpose must be string'));
 
     appstore.registerWithLoginCredentials(req.body, function (error) {
         if (error) return next(BoxError.toHttpError(error));
@@ -82,6 +75,6 @@ function getSubscription(req, res, next) {
     appstore.getSubscription(function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
-        next(new HttpSuccess(200, result)); // { email, cloudronId, plan, cancel_at, status }
+        next(new HttpSuccess(200, result)); // { email, cloudronId, cloudronCreatedAt, plan, current_period_end, canceled_at, cancel_at, status, features }
     });
 }

src/routes/branding.js

@@ -0,0 +1,140 @@
+'use strict';
+
+exports = module.exports = {
+    get,
+    set,
+
+    getCloudronAvatar
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    safe = require('safetydance'),
+    settings = require('../settings.js'),
+    _ = require('underscore');
+
+function getFooter(req, res, next) {
+    settings.getFooter(function (error, footer) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { footer }));
+    });
+}
+
+function setFooter(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.footer !== 'string') return next(new HttpError(400, 'footer is required'));
+
+    settings.setFooter(req.body.footer, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, {}));
+    });
+}
+
+function setCloudronName(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name is required'));
+
+    settings.setCloudronName(req.body.name, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
+function getCloudronName(req, res, next) {
+    settings.getCloudronName(function (error, name) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { name: name }));
+    });
+}
+
+function setAppstoreListingConfig(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    const listingConfig = _.pick(req.body, 'whitelist', 'blacklist');
+    if (Object.keys(listingConfig).length === 0) return next(new HttpError(400, 'blacklist or whitelist is required'));
+
+    if ('whitelist' in listingConfig) {
+        if (listingConfig.whitelist !== null && !Array.isArray(listingConfig.whitelist)) return next(new HttpError(400, 'whitelist is null or an array of strings'));
+
+        if (listingConfig.whitelist && !listingConfig.whitelist.every(id => typeof id === 'string')) return next(new HttpError(400, 'whitelist must be array of strings'));
+    }
+
+    if ('blacklist' in listingConfig) {
+        if (!Array.isArray(listingConfig.blacklist)) return next(new HttpError(400, 'blacklist an array of strings'));
+
+        if (!listingConfig.blacklist.every(id => typeof id === 'string')) return next(new HttpError(400, 'blacklist must be array of strings'));
+    }
+
+    settings.setAppstoreListingConfig(listingConfig, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
+function getAppstoreListingConfig(req, res, next) {
+    settings.getAppstoreListingConfig(function (error, listingConfig) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, listingConfig));
+    });
+}
+
+function setCloudronAvatar(req, res, next) {
+    assert.strictEqual(typeof req.files, 'object');
+
+    if (!req.files.avatar) return next(new HttpError(400, 'avatar must be provided'));
+    var avatar = safe.fs.readFileSync(req.files.avatar.path);
+
+    settings.setCloudronAvatar(avatar, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
+function getCloudronAvatar(req, res, next) {
+    settings.getCloudronAvatar(function (error, avatar) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        // avoid caching the avatar on the client to see avatar changes immediately
+        res.set('Cache-Control', 'no-cache');
+
+        res.set('Content-Type', 'image/png');
+        res.status(200).send(avatar);
+    });
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.params.setting, 'string');
+
+    switch (req.params.setting) {
+    case settings.APPSTORE_LISTING_CONFIG_KEY: return getAppstoreListingConfig(req, res, next);
+    case settings.CLOUDRON_AVATAR_KEY: return getCloudronAvatar(req, res, next);
+    case settings.CLOUDRON_NAME_KEY: return getCloudronName(req, res, next);
+    case settings.FOOTER_KEY: return getFooter(req, res, next);
+
+    default: return next(new HttpError(404, 'No such setting'));
+    }
+}
+
+function set(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    switch (req.params.setting) {
+    case settings.APPSTORE_LISTING_CONFIG_KEY: return setAppstoreListingConfig(req, res, next);
+    case settings.CLOUDRON_AVATAR_KEY: return setCloudronAvatar(req, res, next);
+    case settings.CLOUDRON_NAME_KEY: return setCloudronName(req, res, next);
+    case settings.FOOTER_KEY: return setFooter(req, res, next);
+
+    default: return next(new HttpError(404, 'No such branding'));
+    }
+}

src/routes/clients.js

@@ -1,124 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    add: add,
-    get: get,
-    del: del,
-    getAll: getAll,
-    addToken: addToken,
-    getTokens: getTokens,
-    delTokens: delTokens,
-    delToken: delToken
-};
-
-var assert = require('assert'),
-    BoxError = require('../boxerror.js'),
-    clients = require('../clients.js'),
-    constants = require('../constants.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    validUrl = require('valid-url');
-
-function add(req, res, next) {
-    var data = req.body;
-
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (typeof data.appId !== 'string' || !data.appId) return next(new HttpError(400, 'appId is required'));
-    if (typeof data.redirectURI !== 'string' || !data.redirectURI) return next(new HttpError(400, 'redirectURI is required'));
-    if (typeof data.scope !== 'string' || !data.scope) return next(new HttpError(400, 'scope is required'));
-    if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));
-
-    clients.add(data.appId, clients.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(201, result));
-    });
-}
-
-function get(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-
-    clients.get(req.params.clientId, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, result));
-    });
-}
-
-function del(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-
-    clients.get(req.params.clientId, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        // we do not allow to use the REST API to delete addon clients
-        if (result.type !== clients.TYPE_EXTERNAL) return next(new HttpError(405, 'Deleting app addon clients is not allowed.'));
-
-        clients.del(req.params.clientId, function (error, result) {
-            if (error) return next(BoxError.toHttpError(error));
-
-            next(new HttpSuccess(204, result));
-        });
-    });
-}
-
-function getAll(req, res, next) {
-    clients.getAll(function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, { clients: result }));
-    });
-}
-
-function addToken(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-    assert.strictEqual(typeof req.body, 'object');
-
-    var data = req.body;
-    var expiresAt = data.expiresAt ? parseInt(data.expiresAt, 10) : Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
-    if (isNaN(expiresAt) || expiresAt <= Date.now()) return next(new HttpError(400, 'expiresAt must be a timestamp in the future'));
-    if ('name' in req.body && typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be a string'));
-
-    clients.addTokenByUserId(req.params.clientId, req.user.id, expiresAt, { name: req.body.name || '' }, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(201, { token: result }));
-    });
-}
-
-function getTokens(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-
-    clients.getTokensByUserId(req.params.clientId, req.user.id, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        result = result.map(clients.removeTokenPrivateFields);
-
-        next(new HttpSuccess(200, { tokens: result }));
-    });
-}
-
-function delTokens(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-
-    clients.delTokensByUserId(req.params.clientId, req.user.id, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(204));
-    });
-}
-
-function delToken(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.params.tokenId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-
-    clients.delToken(req.params.clientId, req.params.tokenId, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(204));
-    });
-}

src/routes/cloudron.js

@@ -1,10 +1,16 @@
 'use strict';
 
 exports = module.exports = {
+    login: login,
+    logout: logout,
+    passwordResetRequest: passwordResetRequest,
+    passwordReset: passwordReset,
+    setupAccount: setupAccount,
     reboot: reboot,
     isRebootRequired: isRebootRequired,
     getConfig: getConfig,
     getDisks: getDisks,
+    getMemory: getMemory,
     getUpdateInfo: getUpdateInfo,
     update: update,
     checkForUpdates: checkForUpdates,
@@ -22,15 +28,135 @@ let assert = require('assert'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
     cloudron = require('../cloudron.js'),
-    custom = require('../custom.js'),
-    disks = require('../disks.js'),
+    constants = require('../constants.js'),
+    eventlog = require('../eventlog.js'),
     externalLdap = require('../externalldap.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     sysinfo = require('../sysinfo.js'),
+    system = require('../system.js'),
+    tokendb = require('../tokendb.js'),
+    tokens = require('../tokens.js'),
     updater = require('../updater.js'),
+    users = require('../users.js'),
     updateChecker = require('../updatechecker.js');
 
+function login(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    if ('type' in req.body && typeof req.body.type !== 'string') return next(new HttpError(400, 'type must be a string'));
+
+    const type = req.body.type || tokens.ID_WEBADMIN;
+    const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    const auditSource = { authType: 'basic', ip: ip };
+
+    const error = tokens.validateTokenType(type);
+    if (error) return next(new HttpError(400, error.message));
+
+    tokens.add(type, req.user.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function logout(req, res) {
+    var token;
+
+    // this determines the priority
+    if (req.body && req.body.access_token) token = req.body.access_token;
+    if (req.query && req.query.access_token) token = req.query.access_token;
+    if (req.headers && req.headers.authorization) {
+        var parts = req.headers.authorization.split(' ');
+        if (parts.length == 2) {
+            var scheme = parts[0];
+            var credentials = parts[1];
+
+            if (/^Bearer$/i.test(scheme)) token = credentials;
+        }
+    }
+
+    if (!token) return res.redirect('/login.html');
+
+    tokendb.delByAccessToken(token, function () { res.redirect('/login.html'); });
+}
+
+function passwordResetRequest(req, res, next) {
+    if (!req.body.identifier || typeof req.body.identifier !== 'string') return next(new HttpError(401, 'A identifier must be non-empty string'));
+
+    users.resetPasswordByIdentifier(req.body.identifier, function (error) {
+        if (error && error.reason !== BoxError.NOT_FOUND) console.error(error);
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
+function passwordReset(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
+    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
+
+    users.getByResetToken(req.body.resetToken, function (error, userObject) {
+        if (error) return next(new HttpError(401, 'Invalid resetToken'));
+
+        if (Date.now() - userObject.resetTokenCreationTime > 24 * 60 * 60 * 1000) return next(new HttpError(401, 'Token expired'));
+        if (!userObject.username) return next(new HttpError(409, 'No username set'));
+
+        // setPassword clears the resetToken
+        users.setPassword(userObject, req.body.password, function (error) {
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+            if (error) return next(new HttpError(500, error));
+
+            tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+                if (error) return next(new HttpError(500, error));
+
+                next(new HttpSuccess(202, { accessToken: result.accessToken }));
+            });
+        });
+    });
+}
+
+function setupAccount(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.email || typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be a non-empty string'));
+    if (!req.body.resetToken || typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'resetToken must be a non-empty string'));
+    if (!req.body.password || typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be a non-empty string'));
+    if (!req.body.username || typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be a non-empty string'));
+    if (!req.body.displayName || typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be a non-empty string'));
+
+    users.getByResetToken(req.body.resetToken, function (error, userObject) {
+        if (error) return next(new HttpError(401, 'Invalid Reset Token'));
+
+        if (Date.now() - userObject.resetTokenCreationTime > 24 * 60 * 60 * 1000) return next(new HttpError(401, 'Token expired'));
+
+        users.update(userObject, { username: req.body.username, displayName: req.body.displayName }, auditSource.fromRequest(req), function (error) {
+            if (error && error.reason === BoxError.ALREADY_EXISTS) return next(new HttpError(409, 'Username already used'));
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+            if (error && error.reason === BoxError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
+            if (error) return next(new HttpError(500, error));
+
+            userObject.username = req.body.username;
+            userObject.displayName = req.body.displayName;
+
+            // setPassword clears the resetToken
+            users.setPassword(userObject, req.body.password, function (error) {
+                if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+                if (error) return next(new HttpError(500, error));
+
+                tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+                    if (error) return next(new HttpError(500, error));
+
+                    next(new HttpSuccess(201, { accessToken: result.accessToken }));
+                });
+            });
+        });
+    });
+}
+
 function reboot(req, res, next) {
     // Finish the request, to let the appstore know we triggered the reboot
     next(new HttpSuccess(202, {}));
@@ -55,7 +181,15 @@ function getConfig(req, res, next) {
 }
 
 function getDisks(req, res, next) {
-    disks.getDisks(function (error, result) {
+    system.getDisks(function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function getMemory(req, res, next) {
+    system.getMemory(function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, result));
@@ -156,8 +290,6 @@ function getLogStream(req, res, next) {
 function setDashboardAndMailDomain(req, res, next) {
     if (!req.body.domain || typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
 
-    if (!custom.spec().domains.changeDashboardDomain) return next(new HttpError(405, 'feature disabled by admin'));
-
     cloudron.setDashboardAndMailDomain(req.body.domain, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -168,8 +300,6 @@ function setDashboardAndMailDomain(req, res, next) {
 function prepareDashboardDomain(req, res, next) {
     if (!req.body.domain || typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
 
-    if (!custom.spec().domains.changeDashboardDomain) return next(new HttpError(405, 'feature disabled by admin'));
-
     cloudron.prepareDashboardDomain(req.body.domain, auditSource.fromRequest(req), function (error, taskId) {
         if (error) return next(BoxError.toHttpError(error));
 

src/routes/developer.js

@@ -1,35 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    login: login
-};
-
-var clients = require('../clients.js'),
-    passport = require('passport'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    speakeasy = require('speakeasy');
-
-function login(req, res, next) {
-    passport.authenticate('local', function (error, user) {
-        if (error) return next(new HttpError(500, error));
-        if (!user) return next(new HttpError(401, 'Invalid credentials'));
-
-        var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-
-        if (!user.ghost && user.twoFactorAuthenticationEnabled) {
-            if (!req.body.totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
-
-            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
-            if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
-        }
-
-        const auditSource = { authType: 'cli', ip: ip };
-        clients.issueDeveloperToken(user, auditSource, function (error, result) {
-            if (error) return next(new HttpError(500, error));
-
-            next(new HttpSuccess(200, result));
-        });
-    })(req, res, next);
-}
-

src/routes/domains.js

@@ -8,8 +8,6 @@ exports = module.exports = {
     del: del,
 
     checkDnsRecords: checkDnsRecords,
-
-    verifyDomainLock: verifyDomainLock
 };
 
 var assert = require('assert'),
@@ -19,18 +17,6 @@ var assert = require('assert'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
-function verifyDomainLock(req, res, next) {
-    assert.strictEqual(typeof req.params.domain, 'string');
-
-    domains.get(req.params.domain, function (error, domain) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        if (domain.locked) return next(new HttpError(423, 'This domain is locked'));
-
-        next();
-    });
-}
-
 function add(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
@@ -69,7 +55,7 @@ function add(req, res, next) {
     domains.add(req.body.domain, data, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
-        next(new HttpSuccess(201, { domain: req.body.domain, config: req.body.config }));
+        next(new HttpSuccess(201, {}));
     });
 }
 
@@ -159,4 +145,4 @@ function checkDnsRecords(req, res, next) {
 
         next(new HttpSuccess(200, { needsOverwrite: result.needsOverwrite }));
     });
-}
+}

src/routes/index.js

@@ -2,18 +2,18 @@
 
 exports = module.exports = {
     accesscontrol: require('./accesscontrol.js'),
+    appPasswords: require('./apppasswords.js'),
     apps: require('./apps.js'),
     appstore: require('./appstore.js'),
     backups: require('./backups.js'),
-    clients: require('./clients.js'),
+    branding: require('./branding.js'),
     cloudron: require('./cloudron.js'),
-    developer: require('./developer.js'),
     domains: require('./domains.js'),
     eventlog: require('./eventlog.js'),
     graphs: require('./graphs.js'),
     groups: require('./groups.js'),
-    oauth2: require('./oauth2.js'),
     mail: require('./mail.js'),
+    mailserver: require('./mailserver.js'),
     notifications: require('./notifications.js'),
     profile: require('./profile.js'),
     provision: require('./provision.js'),
@@ -21,5 +21,6 @@ exports = module.exports = {
     settings: require('./settings.js'),
     support: require('./support.js'),
     tasks: require('./tasks.js'),
+    tokens: require('./tokens.js'),
     users: require('./users.js')
 };

src/routes/mail.js

@@ -2,9 +2,6 @@
 
 exports = module.exports = {
     getDomain: getDomain,
-    addDomain: addDomain,
-    getDomainStats: getDomainStats,
-    removeDomain: removeDomain,
 
     setDnsRecords: setDnsRecords,
 
@@ -31,7 +28,7 @@ exports = module.exports = {
     getList: getList,
     addList: addList,
     updateList: updateList,
-    removeList: removeList
+    removeList: removeList,
 };
 
 var assert = require('assert'),
@@ -39,11 +36,7 @@ var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     mail = require('../mail.js'),
     HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    middleware = require('../middleware/index.js'),
-    url = require('url');
-
-var mailProxy = middleware.proxy(url.parse('http://127.0.0.1:2020'));
+    HttpSuccess = require('connect-lastmile').HttpSuccess;
 
 function getDomain(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
@@ -55,31 +48,6 @@ function getDomain(req, res, next) {
     });
 }
 
-function addDomain(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
-
-    mail.addDomain(req.body.domain, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(201, { domain: req.body.domain }));
-    });
-}
-
-function getDomainStats(req, res, next) {
-    assert.strictEqual(typeof req.params.domain, 'string');
-
-    var parsedUrl = url.parse(req.url, true /* parseQueryString */);
-    delete parsedUrl.query['access_token'];
-    delete req.headers['authorization'];
-    delete req.headers['cookies'];
-
-    req.url = url.format({ pathname: req.params.domain, query: parsedUrl.query });
-
-    mailProxy(req, res, next);
-}
-
 function setDnsRecords(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.domain, 'string');
@@ -95,16 +63,6 @@ function setDnsRecords(req, res, next) {
     });
 }
 
-function removeDomain(req, res, next) {
-    assert.strictEqual(typeof req.params.domain, 'string');
-
-    mail.removeDomain(req.params.domain, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(204));
-    });
-}
-
 function getStatus(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
@@ -197,10 +155,10 @@ function listMailboxes(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
-    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a positive number'));
 
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
-    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a positive number'));
 
     mail.listMailboxes(req.params.domain, page, perPage, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -239,7 +197,7 @@ function updateMailbox(req, res, next) {
 
     if (typeof req.body.userId !== 'string') return next(new HttpError(400, 'userId must be a string'));
 
-    mail.updateMailboxOwner(req.params.name, req.params.domain, req.body.userId, function (error) {
+    mail.updateMailboxOwner(req.params.name, req.params.domain, req.body.userId, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));
@@ -261,10 +219,10 @@ function listAliases(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
-    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a positive number'));
 
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
-    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a positive number'));
 
     mail.listAliases(req.params.domain, page, perPage, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -316,7 +274,7 @@ function getList(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.params.name, 'string');
 
-    mail.getList(req.params.domain, req.params.name, function (error, result) {
+    mail.getList(req.params.name, req.params.domain, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { list: result }));
@@ -353,7 +311,7 @@ function updateList(req, res, next) {
         if (typeof req.body.members[i] !== 'string') return next(new HttpError(400, 'member must be a string'));
     }
 
-    mail.updateList(req.params.name, req.params.domain, req.body.members, function (error) {
+    mail.updateList(req.params.name, req.params.domain, req.body.members, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));

src/routes/mailserver.js

@@ -0,0 +1,43 @@
+'use strict';
+
+exports = module.exports = {
+    proxy
+};
+
+var addons = require('../addons.js'),
+    assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    middleware = require('../middleware/index.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    url = require('url');
+
+function proxy(req, res, next) {
+    assert.strictEqual(typeof req.params.pathname, 'string');
+
+    let parsedUrl = url.parse(req.url, true /* parseQueryString */);
+
+    // do not proxy protected values
+    delete parsedUrl.query['access_token'];
+    delete req.headers['authorization'];
+    delete req.headers['cookies'];
+
+    addons.getServiceDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        parsedUrl.query['access_token'] = addonDetails.token;
+        req.url = url.format({ pathname: req.params.pathname, query: parsedUrl.query });
+
+        const proxyOptions = url.parse(`https://${addonDetails.ip}:3000`);
+        proxyOptions.rejectUnauthorized = false;
+        const mailserverProxy = middleware.proxy(proxyOptions);
+
+        mailserverProxy(req, res, function (error) {
+            if (!error) return next();
+
+            if (error.code === 'ECONNREFUSED') return next(new HttpError(424, 'Unable to connect to mail server'));
+            if (error.code === 'ECONNRESET') return next(new HttpError(424, 'Unable to query mail server'));
+
+            next(new HttpError(500, error));
+        });
+    });
+}

src/routes/oauth2.js

@@ -1,563 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize,
-    loginForm: loginForm,
-    login: login,
-    logout: logout,
-    sessionCallback: sessionCallback,
-    passwordResetRequestSite: passwordResetRequestSite,
-    passwordResetRequest: passwordResetRequest,
-    passwordSentSite: passwordSentSite,
-    passwordResetSite: passwordResetSite,
-    passwordReset: passwordReset,
-    accountSetupSite: accountSetupSite,
-    accountSetup: accountSetup,
-    authorization: authorization,
-    token: token,
-    csrf: csrf
-};
-
-var apps = require('../apps.js'),
-    assert = require('assert'),
-    async = require('async'),
-    authcodedb = require('../authcodedb.js'),
-    BoxError = require('../boxerror.js'),
-    clients = require('../clients'),
-    constants = require('../constants.js'),
-    debug = require('debug')('box:routes/oauth2'),
-    eventlog = require('../eventlog.js'),
-    hat = require('../hat.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    middleware = require('../middleware/index.js'),
-    oauth2orize = require('oauth2orize'),
-    passport = require('passport'),
-    querystring = require('querystring'),
-    session = require('connect-ensure-login'),
-    settings = require('../settings.js'),
-    speakeasy = require('speakeasy'),
-    url = require('url'),
-    users = require('../users.js'),
-    util = require('util'),
-    _ = require('underscore');
-
-// appId is optional here
-function auditSource(req, appId) {
-    var tmp = {
-        authType: 'oauth',
-        ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress || null
-    };
-
-    if (appId) tmp.appId = appId;
-
-    return tmp;
-}
-
-var gServer = null;
-
-function initialize() {
-    assert.strictEqual(gServer, null);
-
-    gServer = oauth2orize.createServer();
-
-    gServer.serializeClient(function (client, callback) {
-        return callback(null, client.id);
-    });
-
-    gServer.deserializeClient(function (id, callback) {
-        clients.get(id, callback);
-    });
-
-    // grant authorization code that can be exchanged for access tokens. this is used by external oauth clients
-    gServer.grant(oauth2orize.grant.code({ scopeSeparator: ',' }, function (client, redirectURI, user, ares, callback) {
-        debug('grant code:', client.id, redirectURI, user.id, ares);
-
-        var code = hat(256);
-        var expiresAt = Date.now() + 60 * 60000; // 1 hour
-
-        authcodedb.add(code, client.id, user.id, expiresAt, function (error) {
-            if (error) return callback(error);
-
-            debug('grant code: new auth code for client %s code %s', client.id, code);
-
-            callback(null, code);
-        });
-    }));
-
-    // exchange authorization codes for access tokens. this is used by external oauth clients
-    gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI, callback) {
-        authcodedb.get(code, function (error, authCode) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-            if (error) return callback(error);
-            if (client.id !== authCode.clientId) return callback(null, false);
-
-            authcodedb.del(code, function (error) {
-                if(error) return callback(error);
-
-                clients.addTokenByUserId(client.id, authCode.userId, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                    if (error) return callback(error);
-
-                    debug('exchange: new access token for client %s user %s token %s', client.id, authCode.userId, result.accessToken.slice(0, 6)); // partial token for security
-
-                    callback(null, result.accessToken);
-                });
-            });
-        });
-    }));
-
-    // implicit token grant that skips issuing auth codes. this is used by our webadmin
-    gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client, user, ares, callback) {
-        clients.addTokenByUserId(client.id, user.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-            if (error) return callback(error);
-
-            debug('grant token: new access token for client %s user %s token %s', client.id, user.id, result.accessToken.slice(0, 6)); // partial token for security
-
-            callback(null, result.accessToken);
-        });
-    }));
-
-    // overwrite the session.ensureLoggedIn to not use res.redirect() due to a chrome bug not sending cookies on redirects
-    session.ensureLoggedIn = function (redirectTo) {
-        assert.strictEqual(typeof redirectTo, 'string');
-
-        return function (req, res, next) {
-            if (!req.isAuthenticated || !req.isAuthenticated()) {
-                if (req.session) {
-                    req.session.returnTo = req.originalUrl || req.url;
-                }
-
-                res.status(200).send(util.format('<script>window.location.href = "%s";</script>', redirectTo));
-            } else {
-                next();
-            }
-        };
-    };
-}
-
-function uninitialize() {
-    gServer = null;
-}
-
-function renderTemplate(res, template, data) {
-    assert.strictEqual(typeof res, 'object');
-    assert.strictEqual(typeof template, 'string');
-    assert.strictEqual(typeof data, 'object');
-
-    settings.getCloudronName(function (error, cloudronName) {
-        if (error) {
-            console.error(error);
-            cloudronName = 'Cloudron';
-        }
-
-        // amend template properties, for example used in the header
-        data.title = data.title || 'Cloudron';
-        data.adminOrigin = settings.adminOrigin();
-        data.cloudronName = cloudronName;
-
-        res.render(template, data);
-    });
-}
-
-function sendErrorPageOrRedirect(req, res, message) {
-    assert.strictEqual(typeof req, 'object');
-    assert.strictEqual(typeof res, 'object');
-    assert.strictEqual(typeof message, 'string');
-
-    debug('sendErrorPageOrRedirect: returnTo %s.', req.query.returnTo, message);
-
-    if (typeof req.query.returnTo !== 'string') {
-        renderTemplate(res, 'error', {
-            message: message,
-            title: 'Cloudron Error'
-        });
-    } else {
-        var u = url.parse(req.query.returnTo);
-        if (!u.protocol || !u.host) {
-            return renderTemplate(res, 'error', {
-                message: 'Invalid request. returnTo query is not a valid URI. ' + message,
-                title: 'Cloudron Error'
-            });
-        }
-
-        res.redirect(util.format('%s//%s', u.protocol, u.host));
-    }
-}
-
-// use this instead of sendErrorPageOrRedirect(), in case we have a returnTo provided in the query, to avoid login loops
-// This usually happens when the OAuth client ID is wrong
-function sendError(req, res, message) {
-    assert.strictEqual(typeof req, 'object');
-    assert.strictEqual(typeof res, 'object');
-    assert.strictEqual(typeof message, 'string');
-
-    renderTemplate(res, 'error', {
-        message: message,
-        title: 'Cloudron Error'
-    });
-}
-
-// -> GET /api/v1/session/login
-function loginForm(req, res) {
-    if (typeof req.session.returnTo !== 'string') return sendErrorPageOrRedirect(req, res, 'Invalid login request. No returnTo provided.');
-
-    var u = url.parse(req.session.returnTo, true);
-    if (!u.query.client_id) return sendErrorPageOrRedirect(req, res, 'Invalid login request. No client_id provided.');
-
-    function render(applicationName, applicationLogo) {
-        var error = req.query.error || null;
-
-        renderTemplate(res, 'login', {
-            csrf: req.csrfToken(),
-            applicationName: applicationName,
-            applicationLogo: applicationLogo,
-            error: error,
-            username: settings.isDemo() ? constants.DEMO_USERNAME : '',
-            password: settings.isDemo() ? 'cloudron' : '',
-            title: applicationName + ' Login'
-        });
-    }
-
-    function renderBuiltIn() {
-        settings.getCloudronName(function (error, cloudronName) {
-            if (error) {
-                console.error(error);
-                cloudronName = 'Cloudron';
-            }
-
-            render(cloudronName, '/api/v1/cloudron/avatar');
-        });
-    }
-
-    clients.get(u.query.client_id, function (error, result) {
-        if (error) return sendError(req, res, 'Unknown OAuth client');
-
-        switch (result.type) {
-        case clients.TYPE_BUILT_IN: return renderBuiltIn();
-        case clients.TYPE_EXTERNAL: return render(result.appId, '/api/v1/cloudron/avatar');
-        default: break;
-        }
-
-        apps.get(result.appId, function (error, result) {
-            if (error) return sendErrorPageOrRedirect(req, res, 'Unknown Application for those OAuth credentials');
-
-            var applicationName = result.fqdn;
-            render(applicationName, '/api/v1/apps/' + result.id + '/icon');
-        });
-    });
-}
-
-// -> POST /api/v1/session/login
-function login(req, res) {
-    var returnTo = req.session.returnTo || req.query.returnTo;
-
-    var failureQuery = querystring.stringify({ error: 'Invalid username or password', returnTo: returnTo });
-    passport.authenticate('local', {
-        failureRedirect: '/api/v1/session/login?' + failureQuery
-    })(req, res, function (error) {
-        if (error) return res.redirect('/api/v1/session/login?' + failureQuery); // on some exception in the handlers
-
-        if (!req.user.ghost && req.user.twoFactorAuthenticationEnabled) {
-            if (!req.body.totpToken) {
-                let failureQuery = querystring.stringify({ error: 'A 2FA token is required', returnTo: returnTo });
-                return res.redirect('/api/v1/session/login?' + failureQuery);
-            }
-
-            let verified = speakeasy.totp.verify({ secret: req.user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
-            if (!verified) {
-                let failureQuery = querystring.stringify({ error: 'The 2FA token is invalid', returnTo: returnTo });
-                return res.redirect('/api/v1/session/login?' + failureQuery);
-            }
-        }
-
-        res.redirect(returnTo);
-    });
-}
-
-// -> GET /api/v1/session/logout
-function logout(req, res) {
-    function done() {
-        req.logout();
-
-        if (req.query && req.query.redirect) res.redirect(req.query.redirect);
-        else res.redirect('/');
-    }
-
-    if (!req.query.all) return done();
-
-    // find and destroy all login sessions by this user - this got rather complex quickly
-    req.sessionStore.list(function (error, result) {
-        if (error) {
-            console.error('Error listing sessions', error);
-            return done();
-        }
-
-        // WARNING fix this if we change the storage backend - Great stuff!
-        var sessionIds = result.map(function(s) { return s.replace('.json', ''); });
-
-        async.each(sessionIds, function (id, callback) {
-            req.sessionStore.get(id, function (error, result) {
-                if (error) {
-                    console.error(`Error getting session ${id}`, error);
-                    return callback();
-                }
-
-                // ignore empty or non passport sessions
-                if (!result || !result.passport || !result.passport.user) return callback();
-
-                // not this user
-                if (result.passport.user !== req.user.id) return callback();
-
-                req.sessionStore.destroy(id, function (error) {
-                    if (error) console.error(`Unable to destroy session ${id}`, error);
-                    callback();
-                });
-            });
-        }, done);
-    });
-}
-
-// Form to enter email address to send a password reset request mail
-// -> GET /api/v1/session/password/resetRequest.html
-function passwordResetRequestSite(req, res) {
-    var data = {
-        csrf: req.csrfToken(),
-        title: 'Password Reset'
-    };
-
-    renderTemplate(res, 'password_reset_request', data);
-}
-
-// This route is used for above form submission
-// -> POST /api/v1/session/password/resetRequest
-function passwordResetRequest(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.identifier !== 'string') return next(new HttpError(400, 'Missing identifier')); // email or username
-
-    debug('passwordResetRequest: email or username %s.', req.body.identifier);
-
-    users.resetPasswordByIdentifier(req.body.identifier, function (error) {
-        if (error && error.reason !== BoxError.NOT_FOUND) {
-            console.error(error);
-            return sendErrorPageOrRedirect(req, res, 'User not found');
-        }
-
-        res.redirect('/api/v1/session/password/sent.html');
-    });
-}
-
-// -> GET /api/v1/session/password/sent.html
-function passwordSentSite(req, res) {
-    renderTemplate(res, 'password_reset_sent', { title: 'Cloudron Password Reset' });
-}
-
-function renderAccountSetupSite(res, req, userObject, error) {
-    renderTemplate(res, 'account_setup', {
-        user: userObject,
-        error: error,
-        csrf: req.csrfToken(),
-        resetToken: req.query.reset_token || req.body.resetToken,
-        email: req.query.email || req.body.email,
-        title: 'Account Setup'
-    });
-}
-
-// -> GET /api/v1/session/account/setup.html
-function accountSetupSite(req, res) {
-    if (!req.query.reset_token) return sendError(req, res, 'Missing Reset Token');
-    if (!req.query.email) return sendError(req, res, 'Missing Email');
-
-    users.getByResetToken(req.query.email, req.query.reset_token, function (error, userObject) {
-        if (error) return sendError(req, res, 'Invalid Email or Reset Token');
-
-        renderAccountSetupSite(res, req, userObject, '');
-    });
-}
-
-// -> POST /api/v1/session/account/setup
-function accountSetup(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.email !== 'string') return next(new HttpError(400, 'Missing email'));
-    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
-    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
-    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'Missing username'));
-    if (typeof req.body.displayName !== 'string') return next(new HttpError(400, 'Missing displayName'));
-
-    debug(`acountSetup: for email ${req.body.email} with token ${req.body.resetToken}`);
-
-    users.getByResetToken(req.body.email, req.body.resetToken, function (error, userObject) {
-        if (error) return sendError(req, res, 'Invalid Reset Token');
-
-        var data = _.pick(req.body, 'username', 'displayName');
-        users.update(userObject.id, data, auditSource(req), function (error) {
-            if (error && error.reason === BoxError.ALREADY_EXISTS) return renderAccountSetupSite(res, req, userObject, 'Username already exists');
-            if (error && error.reason === BoxError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
-            if (error && error.reason === BoxError.NOT_FOUND) return renderAccountSetupSite(res, req, userObject, 'No such user');
-            if (error) return next(new HttpError(500, error));
-
-            userObject.username = req.body.username;
-            userObject.displayName = req.body.displayName;
-
-            // setPassword clears the resetToken
-            users.setPassword(userObject.id, req.body.password, function (error) {
-                if (error && error.reason === BoxError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
-                if (error) return next(new HttpError(500, error));
-
-                clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                    if (error) return next(new HttpError(500, error));
-
-                    res.redirect(`${settings.adminOrigin()}?accessToken=${result.accessToken}&expiresAt=${result.expires}`);
-                });
-            });
-        });
-    });
-}
-
-// -> GET /api/v1/session/password/reset.html
-function passwordResetSite(req, res, next) {
-    if (!req.query.email) return next(new HttpError(400, 'Missing email'));
-    if (!req.query.reset_token) return next(new HttpError(400, 'Missing reset_token'));
-
-    users.getByResetToken(req.query.email, req.query.reset_token, function (error, user) {
-        if (error) return next(new HttpError(401, 'Invalid email or reset token'));
-
-        renderTemplate(res, 'password_reset', {
-            user: user,
-            csrf: req.csrfToken(),
-            resetToken: req.query.reset_token,
-            email: req.query.email,
-            title: 'Password Reset'
-        });
-    });
-}
-
-// -> POST /api/v1/session/password/reset
-function passwordReset(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.email !== 'string') return next(new HttpError(400, 'Missing email'));
-    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
-    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
-
-    debug(`passwordReset: for ${req.body.email} with token ${req.body.resetToken}`);
-
-    users.getByResetToken(req.body.email, req.body.resetToken, function (error, userObject) {
-        if (error) return next(new HttpError(401, 'Invalid email or resetToken'));
-
-        if (!userObject.username) return next(new HttpError(401, 'No username set'));
-
-        // setPassword clears the resetToken
-        users.setPassword(userObject.id, req.body.password, function (error) {
-            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(406, error.message));
-            if (error) return next(new HttpError(500, error));
-
-            clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                if (error) return next(new HttpError(500, error));
-
-                res.redirect(`${settings.adminOrigin()}?accessToken=${result.accessToken}&expiresAt=${result.expires}`);
-            });
-        });
-    });
-}
-
-
-// The callback page takes the redirectURI and the authCode and redirects the browser accordingly
-//
-// -> GET /api/v1/session/callback
-function sessionCallback() {
-    return [
-        session.ensureLoggedIn('/api/v1/session/login'),
-        function (req, res) {
-            renderTemplate(res, 'callback', { callbackServer: req.query.redirectURI });
-        }
-    ];
-}
-
-// The authorization endpoint is the entry point for an OAuth login.
-//
-// Each app would start OAuth by redirecting the user to:
-//
-//    /api/v1/oauth/dialog/authorize?response_type=code&client_id=<clientId>&redirect_uri=<callbackURL>&scope=<ignored>
-//
-//  - First, this will ensure the user is logged in.
-//  - Then it will redirect the browser to the given <callbackURL> containing the authcode in the query
-//
-// -> GET /api/v1/oauth/dialog/authorize
-function authorization() {
-    return [
-        function (req, res, next) {
-            if (!req.query.redirect_uri) return sendErrorPageOrRedirect(req, res, 'Invalid request. redirect_uri query param is not set.');
-            if (!req.query.client_id) return sendErrorPageOrRedirect(req, res, 'Invalid request. client_id query param is not set.');
-            if (!req.query.response_type) return sendErrorPageOrRedirect(req, res, 'Invalid request. response_type query param is not set.');
-            if (req.query.response_type !== 'code' && req.query.response_type !== 'token') return sendErrorPageOrRedirect(req, res, 'Invalid request. Only token and code response types are supported.');
-
-            session.ensureLoggedIn('/api/v1/session/login?returnTo=' + req.query.redirect_uri)(req, res, next);
-        },
-        gServer.authorization({}, function (clientId, redirectURI, callback) {
-            debug('authorization: client %s with callback to %s.', clientId, redirectURI);
-
-            clients.get(clientId, function (error, client) {
-                if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-                if (error) return callback(error);
-
-                // ignore the origin passed into form the client, but use the one from the clientdb
-                var redirectPath = url.parse(redirectURI).path;
-                var redirectOrigin = client.redirectURI;
-
-                callback(null, client, '/api/v1/session/callback?redirectURI=' + encodeURIComponent(url.resolve(redirectOrigin, redirectPath)));
-            });
-        }),
-        function (req, res, next) {
-            // Handle our different types of oauth clients
-            var type = req.oauth2.client.type;
-
-            if (type === clients.TYPE_EXTERNAL || type === clients.TYPE_BUILT_IN) {
-                eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, req.oauth2.client.appId), { userId: req.oauth2.user.id, user: users.removePrivateFields(req.oauth2.user) });
-                return next();
-            }
-
-            apps.get(req.oauth2.client.appId, function (error, appObject) {
-                if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
-
-                apps.hasAccessTo(appObject, req.oauth2.user, function (error, access) {
-                    if (error) return sendError(req, res, 'Internal error');
-                    if (!access) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
-
-                    eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, appObject.id), { userId: req.oauth2.user.id, user: users.removePrivateFields(req.oauth2.user) });
-
-                    next();
-                });
-            });
-        },
-        gServer.decision({ loadTransaction: false })
-    ];
-}
-
-//  The token endpoint allows an OAuth client to exchange an authcode with an accesstoken.
-//
-//  Authcodes are obtained using the authorization endpoint. The route is authenticated by
-//  providing a Basic auth with clientID as username and clientSecret as password.
-//  An authcode is only good for one such exchange to an accesstoken.
-//
-// -> POST /api/v1/oauth/token
-function token() {
-    return [
-        passport.authenticate(['basic', 'oauth2-client-password'], { session: false }),
-        gServer.token(), // will call the token grant callback registered in initialize()
-        gServer.errorHandler()
-    ];
-}
-
-// Cross-site request forgery protection middleware for login form
-function csrf() {
-    return [
-        middleware.csrf(),
-        function (err, req, res, next) {
-            if (err.code !== 'EBADCSRFTOKEN') return next(err);
-
-            sendErrorPageOrRedirect(req, res, 'Form expired');
-        }
-    ];
-}