Fix spamd logs

this lets us put images in app_not_responding.html

CHANGES

@@ -1730,3 +1730,64 @@
 * Fix registry detection in private images
 * Make mailbox domain configurable for apps
 
+[4.3.4]
+* Do not error if fallback certs went missing
+* Add 'New Apps' section to Appstore view
+* Fix issue where graphs of some apps were not appearing
+
+[4.4.0]
+* Show swap in graphs
+* Make avatars customizable
+* Hide access tokens from logs
+* Add missing '@' sign for email address in app mailbox
+* Add app fqdn to backup progress message
+* import: add option to import app in-place
+* import: add option to import app from arbitrary backup config
+* Show download progress for rsync backups
+* Fix various repair workflows
+* acme2: Implement post-as-get
+
+[4.4.1]
+* ami: fix AWS provider validation
+
+[4.4.2]
+* Fix crash when reporting that DKIM is not setup correctly
+* Stopped apps cannot be updated or auto-updated
+* eventlog: track support ticket creation and remote support status
+
+[4.4.3]
+* Add restart button in recovery section
+* Fix issue where memory usage was not computed correctly
+* cloudflare: support API tokens
+
+[4.4.4]
+* Fix bug where restart button in terminal was not working
+* Add search field in apps view
+* Make app view tags and domain filter persistent
+* Add timezone UI
+
+[4.4.5]
+* Fix user listing regression in group edit dialog
+* Do not show error page for 503
+* Add mail list and mail box update events
+* Certs of stopped apps are not renewed anymore
+* Fix broken memory sliders in the services UI
+* Set CPU Shares
+* Update nodejs to 12.14.1
+* Update MySQL addon packet size to 64M
+
+[5.0.0]
+* Show backup disk usage in graphs
+* Add per-user app passwords
+* Make app not responding page customizable
+* Make footer customizable
+* Add UI to import backups
+* Display timestamps in browser timezone in the UI
+* Mail eventlog and usage
+* Add user roles - owner, admin, user manager and user
+* Setup logrotate configs for collectd since upstream does not set it up
+* mail: Add X-Envelope-To and X-Envelope-From headers for incoming mails
+* linode: add object storage backend
+* restore: carefully replace backup config
+* spam: add default corpus and global db
+

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2019 Cloudron UG
+Copyright (c) 2020 Cloudron UG
 
 With regard to the Cloudron Software:
 

baseimage/initializeBaseUbuntuImage.sh

@@ -62,10 +62,10 @@ apt-get -o Dpkg::Options::="--force-confold" install -y sudo
 cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
 echo "==> Installing node.js"
-mkdir -p /usr/local/node-10.15.1
-curl -sL https://nodejs.org/dist/v10.15.1/node-v10.15.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.15.1
-ln -sf /usr/local/node-10.15.1/bin/node /usr/bin/node
-ln -sf /usr/local/node-10.15.1/bin/npm /usr/bin/npm
+mkdir -p /usr/local/node-10.18.1
+curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
+ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
+ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
 apt-get install -y python   # Install python which is required for npm rebuild
 [[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
@@ -123,6 +123,13 @@ timedatectl set-ntp 1
 # mysql follows the system timezone
 timedatectl set-timezone UTC
 
+echo "==> Adding sshd configuration warning"
+sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://cloudron.io/documentation/security/#securing-ssh-access' -i /etc/ssh/sshd_config
+
+# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
+echo "==> Disabling motd news"
+sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
+
 # Disable bind for good measure (on online.net, kimsufi servers these are pre-installed and conflicts with unbound)
 systemctl stop bind9 || true
 systemctl disable bind9 || true

migrations/20190911193842-mailboxes-make-members-fully-qualified.js

@@ -9,7 +9,7 @@ exports.up = function(db, callback) {
             if (!mailbox.membersJson) return iteratorDone();
 
             let members = JSON.parse(mailbox.membersJson);
-            members = members.map((m) => m.indexOf('@') === -1 ? `${m}@${mailbox.domain}` : m); // only because we don't do things in a xction
+            members = members.map((m) => m && m.indexOf('@') === -1 ? `${m}@${mailbox.domain}` : m); // only because we don't do things in a xction
 
             db.runSql('UPDATE mailboxes SET membersJson=? WHERE name=? AND domain=?', [ JSON.stringify(members), mailbox.name, mailbox.domain ], iteratorDone);
         }, callback);

migrations/20200101235723-domains-set-cloudflare-tokenType.js

@@ -0,0 +1,22 @@
+'use strict';
+
+let async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('SELECT * FROM domains', function (error, domains) {
+        if (error) return callback(error);
+
+        async.eachSeries(domains, function (domain, iteratorCallback) {
+            if (domain.provider !== 'cloudflare') return iteratorCallback();
+
+            let config = JSON.parse(domain.configJson);
+            config.tokenType = 'GlobalApiKey';
+
+            db.runSql('UPDATE domains SET configJson = ? WHERE domain = ?', [ JSON.stringify(config), domain.domain ], iteratorCallback);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20200129052822-apps-add-cpuShares.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN cpuShares INTEGER DEFAULT 512', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN cpuShares', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200131232251-appPasswords-create-table.js

@@ -0,0 +1,26 @@
+'use strict';
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE appPasswords(' +
+                'id VARCHAR(128) NOT NULL UNIQUE,' +
+				'name VARCHAR(128) NOT NULL,' +
+                'userId VARCHAR(128) NOT NULL,' +
+                'identifier VARCHAR(128) NOT NULL,' +
+                'hashedPassword VARCHAR(1024) NOT NULL,' +
+				'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
+                'FOREIGN KEY(userId) REFERENCES users(id),' +
+                'UNIQUE (name, userId),' +
+				'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE appPasswords', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200205172146-remove-authcodedb.js

@@ -0,0 +1,22 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DROP TABLE authcodes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    var cmd = `CREATE TABLE IF NOT EXISTS authcodes(
+        authCode VARCHAR(128) NOT NULL UNIQUE,
+        userId VARCHAR(128) NOT NULL,
+        clientId VARCHAR(128) NOT NULL,
+        expiresAt BIGINT NOT NULL,
+        PRIMARY KEY(authCode)) CHARACTER SET utf8 COLLATE utf8_bin`;
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200206154808-remove-clients.js

@@ -0,0 +1,24 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DROP TABLE clients', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    var cmd = `CREATE TABLE IF NOT EXISTS clients(
+      id VARCHAR(128) NOT NULL UNIQUE,
+      appId VARCHAR(128) NOT NULL,
+      type VARCHAR(16) NOT NULL,
+      clientSecret VARCHAR(512) NOT NULL,
+      redirectURI VARCHAR(512) NOT NULL,
+      scope VARCHAR(512) NOT NULL,
+      PRIMARY KEY(id)) CHARACTER SET utf8 COLLATE utf8_bin`;
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200214051201-domains-drop-locked.js

@@ -0,0 +1,17 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE domains DROP COLUMN locked', function (error) {
+        if (error) return callback(error);
+
+        callback();
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE domains ADD COLUMN locked BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20200221211850-users-add-role.js

@@ -0,0 +1,40 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        db.runSql.bind(db, 'ALTER TABLE users ADD COLUMN role VARCHAR(32)'),
+        function migrateAdminFlag(done) {
+            db.all('SELECT * FROM users ORDER BY createdAt', function (error, results) {
+                if (error) return done(error);
+                let ownerFound = false;
+
+                async.eachSeries(results, function (user, next) {
+                    let role;
+                    if (!ownerFound && user.admin) {
+                        role = 'owner';
+                        ownerFound = true;
+                        console.log(`Designating ${user.username} ${user.email} ${user.id} as the owner of this cloudron`);
+                    } else {
+                        role = user.admin ? 'admin' : 'user';
+                    }
+                    db.runSql('UPDATE users SET role=? WHERE id=?', [ role, user.id ], next);
+                }, done);
+            });
+        },
+        db.runSql.bind(db, 'ALTER TABLE users DROP COLUMN admin'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY role VARCHAR(32) NOT NULL'),
+        db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN role', function (error) {
+        if (error) console.error(error);
+
+        callback(error);
+    });
+};
+

migrations/schema.sql

@@ -26,8 +26,8 @@ CREATE TABLE IF NOT EXISTS users(
     fallbackEmail VARCHAR(512) DEFAULT "",
     twoFactorAuthenticationSecret VARCHAR(128) DEFAULT "",
     twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
-    admin BOOLEAN DEFAULT false,
     source VARCHAR(128) DEFAULT "",
+    role VARCHAR(32),
 
     PRIMARY KEY(id));
 
@@ -52,15 +52,6 @@ CREATE TABLE IF NOT EXISTS tokens(
     expires BIGINT NOT NULL, // FIXME: make this a timestamp
     PRIMARY KEY(accessToken));
 
-CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(128) NOT NULL UNIQUE, // prefixed with cid- to identify token easily in auth routes
-    appId VARCHAR(128) NOT NULL,     // name of the client (for external apps) or id of app (for built-in apps)
-    type VARCHAR(16) NOT NULL,
-    clientSecret VARCHAR(512) NOT NULL,
-    redirectURI VARCHAR(512) NOT NULL,
-    scope VARCHAR(512) NOT NULL,
-    PRIMARY KEY(id));
-
 CREATE TABLE IF NOT EXISTS apps(
     id VARCHAR(128) NOT NULL UNIQUE,
     appStoreId VARCHAR(128) NOT NULL, // empty for custom apps
@@ -78,6 +69,7 @@ CREATE TABLE IF NOT EXISTS apps(
     updateTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,   // when the last app update was done
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, // when this db record was updated (useful for UI caching)
     memoryLimit BIGINT DEFAULT 0,
+    cpuShares INTEGER DEFAULT 512,
     xFrameOptions VARCHAR(512),
     sso BOOLEAN DEFAULT 1, // whether user chose to enable SSO
     debugModeJson TEXT, // options for development mode
@@ -104,13 +96,6 @@ CREATE TABLE IF NOT EXISTS appPortBindings(
     FOREIGN KEY(appId) REFERENCES apps(id),
     PRIMARY KEY(hostPort));
 
-CREATE TABLE IF NOT EXISTS authcodes(
-    authCode VARCHAR(128) NOT NULL UNIQUE,
-    userId VARCHAR(128) NOT NULL,
-    clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL, // ## FIXME: make this a timestamp
-    PRIMARY KEY(authCode));
-
 CREATE TABLE IF NOT EXISTS settings(
     name VARCHAR(128) NOT NULL UNIQUE,
     value TEXT,
@@ -157,7 +142,6 @@ CREATE TABLE IF NOT EXISTS domains(
     provider VARCHAR(16) NOT NULL,
     configJson TEXT, /* JSON containing the dns backend provider config */
     tlsConfigJson TEXT, /* JSON containing the tls provider config */
-    locked BOOLEAN,
 
     PRIMARY KEY (domain))
 
@@ -231,4 +215,16 @@ CREATE TABLE IF NOT EXISTS notifications(
     PRIMARY KEY (id)
 );
 
+CREATE TABLE IF NOT EXISTS appPasswords(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    name VARCHAR(128) NOT NULL,
+    userId VARCHAR(128) NOT NULL,
+    identifier VARCHAR(128) NOT NULL, // resourceId: app id or mail or webadmin
+    hashedPassword VARCHAR(1024) NOT NULL,
+    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY(userId) REFERENCES users(id),
+    UNIQUE (name, userId),
+    PRIMARY KEY (id)
+);
+
 CHARACTER SET utf8 COLLATE utf8_bin;

package.json

@@ -17,71 +17,59 @@
     "@google-cloud/dns": "^1.1.0",
     "@google-cloud/storage": "^2.5.0",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^2.6.2",
-    "aws-sdk": "^2.476.0",
+    "async": "^2.6.3",
+    "aws-sdk": "^2.610.0",
     "body-parser": "^1.19.0",
     "cloudron-manifestformat": "^4.0.0",
     "connect": "^3.7.0",
-    "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "^1.2.1",
+    "connect-lastmile": "^1.2.2",
     "connect-timeout": "^1.9.0",
-    "cookie-parser": "^1.4.4",
-    "cookie-session": "^1.3.3",
-    "cron": "^1.7.1",
-    "csurf": "^1.10.0",
+    "cookie-session": "^1.4.0",
+    "cron": "^1.8.2",
     "db-migrate": "^0.11.6",
     "db-migrate-mysql": "^1.1.10",
     "debug": "^4.1.1",
     "dockerode": "^2.5.8",
     "ejs": "^2.6.1",
-    "ejs-cli": "^2.0.1",
+    "ejs-cli": "^2.1.1",
     "express": "^4.17.1",
-    "express-session": "^1.16.2",
     "js-yaml": "^3.13.1",
     "json": "^9.0.6",
     "ldapjs": "^1.0.2",
-    "lodash": "^4.17.11",
+    "lodash": "^4.17.15",
     "lodash.chunk": "^4.2.0",
     "mime": "^2.4.4",
-    "moment-timezone": "^0.5.25",
+    "moment-timezone": "^0.5.27",
     "morgan": "^1.9.1",
     "multiparty": "^4.2.1",
-    "mysql": "^2.17.1",
-    "nodemailer": "^6.2.1",
+    "mysql": "^2.18.1",
+    "nodemailer": "^6.4.2",
     "nodemailer-smtp-transport": "^2.7.4",
-    "oauth2orize": "^1.11.0",
     "once": "^1.4.0",
     "parse-links": "^0.1.0",
-    "passport": "^0.4.0",
-    "passport-http": "^0.3.0",
-    "passport-http-bearer": "^1.0.1",
-    "passport-local": "^1.0.0",
-    "passport-oauth2-client-password": "^0.1.2",
     "pretty-bytes": "^5.3.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
-    "qrcode": "^1.3.3",
-    "readdirp": "^3.0.2",
+    "qrcode": "^1.4.4",
+    "readdirp": "^3.3.0",
     "request": "^2.88.0",
     "rimraf": "^2.6.3",
     "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^0.7.1",
+    "safetydance": "^1.0.0",
     "semver": "^6.1.1",
-    "session-file-store": "^1.3.1",
-    "showdown": "^1.9.0",
+    "showdown": "^1.9.1",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^5.0.9",
+    "superagent": "^5.2.1",
     "supererror": "^0.7.2",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.1.0",
     "tldjs": "^2.3.1",
-    "underscore": "^1.9.1",
-    "uuid": "^3.3.2",
-    "valid-url": "^1.0.9",
+    "underscore": "^1.9.2",
+    "uuid": "^3.4.0",
     "validator": "^11.0.0",
-    "ws": "^7.0.0",
-    "xml2js": "^0.4.19"
+    "ws": "^7.2.1",
+    "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",

runTests

@@ -22,7 +22,7 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
 mkdir -p platformdata/addons/mail platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
 
 # put cert

scripts/cloudron-setup

@@ -92,13 +92,14 @@ fi
 echo "Running cloudron-setup with args : $@" > "${LOG_FILE}"
 
 # validate arguments in the absence of data
-readonly AVAILABLE_PROVIDERS="azure, caas, cloudscale, contabo, digitalocean, ec2, exoscale, galaxygate, gce, hetzner, interox, lightsail, linode, netcup, ovh, rosehosting, scaleway, skysilk, time4vps, upcloud, vultr or generic"
+readonly AVAILABLE_PROVIDERS="azure, caas, cloudscale, contabo, digitalocean, ec2, exoscale, gce, hetzner, interox, lightsail, linode, netcup, ovh, rosehosting, scaleway, skysilk, time4vps, upcloud, vultr or generic"
 if [[ -z "${provider}" ]]; then
     echo "--provider is required ($AVAILABLE_PROVIDERS)"
     exit 1
 elif [[  \
             "${provider}" != "ami" && \
             "${provider}" != "azure" && \
+            "${provider}" != "azure-image" && \
             "${provider}" != "caas" && \
             "${provider}" != "cloudscale" && \
             "${provider}" != "contabo" && \
@@ -106,13 +107,13 @@ elif [[  \
             "${provider}" != "digitalocean-mp" && \
             "${provider}" != "ec2" && \
             "${provider}" != "exoscale" && \
-            "${provider}" != "galaxygate" && \
             "${provider}" != "gce" && \
             "${provider}" != "hetzner" && \
             "${provider}" != "interox" && \
             "${provider}" != "interox-image" && \
             "${provider}" != "lightsail" && \
             "${provider}" != "linode" && \
+            "${provider}" != "linode-oneclick" && \
             "${provider}" != "linode-stackscript" && \
             "${provider}" != "netcup" && \
             "${provider}" != "netcup-image" && \
@@ -202,21 +203,11 @@ if [[ "${initBaseImage}" == "true" ]]; then
     echo ""
 fi
 
-# NOTE: this install script only supports 3.x and above
+# NOTE: this install script only supports 4.2 and above
 echo "=> Installing version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
-# this file is used >= 4.2
 echo "${provider}" > /etc/cloudron/PROVIDER
 
-# this file is unused <= 4.2 and exists to make legacy installations work. the start script will remove this file anyway
-cat > "/etc/cloudron/cloudron.conf" <<CONF_END
-{
-    "apiServerOrigin": "${apiServerOrigin}",
-    "webServerOrigin": "${webServerOrigin}",
-    "provider": "${provider}"
-}
-CONF_END
-
 [[ -n "${license}" ]] && echo -n "$license" > /etc/cloudron/LICENSE
 
 if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
@@ -224,7 +215,6 @@ if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
     exit 1
 fi
 
-# only needed for >= 4.2
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('api_server_origin', '${apiServerOrigin}');" 2>/dev/null
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('web_server_origin', '${webServerOrigin}');" 2>/dev/null
 

scripts/cloudron-support

@@ -34,7 +34,7 @@ while true; do
     --help) echo -e "${HELP_MESSAGE}"; exit 0;;
     --enable-ssh) enableSSH="true"; shift;;
     --admin-login)
-        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE admin=1 LIMIT 1" 2>/dev/null)
+        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
         printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > /tmp/cloudron_ghost.json
         echo "Login as ${admin_username} / ${admin_password} . Remove /tmp/cloudron_ghost.json when done."
@@ -80,13 +80,13 @@ echo -e $LINE"Filesystem stats"$LINE >> $OUT
 df -h &>> $OUT
 
 echo -e $LINE"Appsdata stats"$LINE >> $OUT
-du -hcsL /home/yellowtent/appsdata/* &>> $OUT
+du -hcsL /home/yellowtent/appsdata/* &>> $OUT || true
 
 echo -e $LINE"Boxdata stats"$LINE >> $OUT
 du -hcsL /home/yellowtent/boxdata/* &>> $OUT
 
 echo -e $LINE"Backup stats (possibly misleading)"$LINE >> $OUT
-du -hcsL /var/backups/* &>> $OUT
+du -hcsL /var/backups/* &>> $OUT || true
 
 echo -e $LINE"System daemon status"$LINE >> $OUT
 systemctl status --lines=100 cloudron.target box mysql unbound cloudron-syslog nginx collectd docker &>> $OUT

scripts/createReleaseTarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v10.15.1" ]]; then
-    echo "This script requires node 10.15.1"
+if [[ "$(node --version)" != "v10.18.1" ]]; then
+    echo "This script requires node 10.18.1"
     exit 1
 fi
 

scripts/installer.sh

@@ -57,12 +57,12 @@ if [[ $(docker version --format {{.Client.Version}}) != "18.09.2" ]]; then
 fi
 
 echo "==> installer: updating node"
-if [[ "$(node --version)" != "v10.15.1" ]]; then
-    mkdir -p /usr/local/node-10.15.1
-    $curl -sL https://nodejs.org/dist/v10.15.1/node-v10.15.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.15.1
-    ln -sf /usr/local/node-10.15.1/bin/node /usr/bin/node
-    ln -sf /usr/local/node-10.15.1/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-8.11.2 /usr/local/node-8.9.3
+if [[ "$(node --version)" != "v10.18.1" ]]; then
+    mkdir -p /usr/local/node-10.18.1
+    $curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
+    ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
+    ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
+    rm -rf /usr/local/node-10.15.1
 fi
 
 # this is here (and not in updater.js) because rebuild requires the above node

setup/start.sh

@@ -47,10 +47,12 @@ mkdir -p "${PLATFORM_DATA_DIR}/backup"
 mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
          "${PLATFORM_DATA_DIR}/logs/updater" \
          "${PLATFORM_DATA_DIR}/logs/tasks" \
-         "${PLATFORM_DATA_DIR}/logs/crash"
+         "${PLATFORM_DATA_DIR}/logs/crash" \
+         "${PLATFORM_DATA_DIR}/logs/collectd"
 mkdir -p "${PLATFORM_DATA_DIR}/update"
 
 mkdir -p "${BOX_DATA_DIR}/appicons"
+mkdir -p "${BOX_DATA_DIR}/profileicons"
 mkdir -p "${BOX_DATA_DIR}/certs"
 mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
 mkdir -p "${BOX_DATA_DIR}/mail/dkim"
@@ -113,7 +115,7 @@ rm -f /etc/sudoers.d/${USER}
 cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER}
 
 echo "==> Configuring collectd"
-rm -rf /etc/collectd
+rm -rf /etc/collectd /var/log/collectd.log
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
 systemctl restart collectd

setup/start/collectd/collectd.conf

@@ -57,7 +57,7 @@ LoadPlugin logfile
 
 <Plugin logfile>
    LogLevel "info"
-   File "/var/log/collectd.log"
+   File "/home/yellowtent/platformdata/logs/collectd/collectd.log"
    Timestamp true
    PrintSeverity false
 </Plugin>

setup/start/custom.yml

@@ -37,4 +37,4 @@
 #   notifyCloudronAdmins: false
 #
 # footer:
-#   body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
+#   body: '&copy; 2020 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'

setup/start/logrotate/platform

@@ -9,6 +9,7 @@
 /home/yellowtent/platformdata/logs/sftp/*.log
 /home/yellowtent/platformdata/logs/redis-*/*.log
 /home/yellowtent/platformdata/logs/crash/*.log
+/home/yellowtent/platformdata/logs/collectd/*.log
 /home/yellowtent/platformdata/logs/updater/*.log {
     # only keep one rotated file, we currently do not send that over the api
     rotate 1

src/accesscontrol.js

@@ -1,140 +1,29 @@
 'use strict';
 
 exports = module.exports = {
-    SCOPE_APPS_READ: 'apps:read',
-    SCOPE_APPS_MANAGE: 'apps:manage',
-    SCOPE_APPSTORE: 'appstore',
-    SCOPE_CLIENTS: 'clients',
-    SCOPE_CLOUDRON: 'cloudron',
-    SCOPE_DOMAINS_READ: 'domains:read',
-    SCOPE_DOMAINS_MANAGE: 'domains:manage',
-    SCOPE_MAIL: 'mail',
-    SCOPE_PROFILE: 'profile',
-    SCOPE_SETTINGS: 'settings',
-    SCOPE_SUBSCRIPTION: 'subscription',
-    SCOPE_USERS_READ: 'users:read',
-    SCOPE_USERS_MANAGE: 'users:manage',
-    VALID_SCOPES: [ 'apps', 'appstore', 'clients', 'cloudron', 'domains', 'mail', 'profile', 'settings', 'subscription', 'users' ], // keep this sorted
-
-    SCOPE_ANY: '*',
-
-    validateScopeString: validateScopeString,
-    hasScopes: hasScopes,
-    canonicalScopeString: canonicalScopeString,
-    intersectScopes: intersectScopes,
-    validateToken: validateToken,
-    scopesForUser: scopesForUser
+    verifyToken: verifyToken
 };
 
 var assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    debug = require('debug')('box:accesscontrol'),
     tokendb = require('./tokendb.js'),
-    users = require('./users.js'),
-    _ = require('underscore');
+    users = require('./users.js');
 
-// returns scopes that does not have wildcards and is sorted
-function canonicalScopeString(scope) {
-    if (scope === exports.SCOPE_ANY) return exports.VALID_SCOPES.join(',');
-
-    return scope.split(',').sort().join(',');
-}
-
-function intersectScopes(allowedScopes, wantedScopes) {
-    assert(Array.isArray(allowedScopes), 'Expecting sorted array');
-    assert(Array.isArray(wantedScopes), 'Expecting sorted array');
-
-    if (_.isEqual(allowedScopes, wantedScopes)) return allowedScopes; // quick path
-
-    let wantedScopesMap = new Map();
-    let results = [];
-
-    // make a map of scope -> [ subscopes ]
-    for (let w of wantedScopes) {
-        let parts = w.split(':');
-        let subscopes = wantedScopesMap.get(parts[0]) || new Set();
-        subscopes.add(parts[1] || '*');
-        wantedScopesMap.set(parts[0], subscopes);
-    }
-
-    for (let a of allowedScopes) {
-        let parts = a.split(':');
-        let as = parts[1] || '*';
-
-        let subscopes = wantedScopesMap.get(parts[0]);
-        if (!subscopes) continue;
-
-        if (subscopes.has('*') || subscopes.has(as)) {
-            results.push(a);
-        } else if (as === '*') {
-            results = results.concat(Array.from(subscopes).map(function (ss) { return `${a}:${ss}`; }));
-        }
-    }
-
-    return results;
-}
-
-function validateScopeString(scope) {
-    assert.strictEqual(typeof scope, 'string');
-
-    if (scope === '') return new BoxError(BoxError.BAD_FIELD, 'Empty scope not allowed', { field: 'scope' });
-
-    // NOTE: this function intentionally does not allow '*'. This is only allowed in the db to allow
-    // us not write a migration script every time we add a new scope
-    var allValid = scope.split(',').every(function (s) { return exports.VALID_SCOPES.indexOf(s.split(':')[0]) !== -1; });
-    if (!allValid) return new BoxError(BoxError.BAD_FIELD, 'Invalid scope. Available scopes are ' + exports.VALID_SCOPES.join(', '), { field: 'scope' });
-    return null;
-}
-
-// tests if all requiredScopes are attached to the request
-function hasScopes(authorizedScopes, requiredScopes) {
-    assert(Array.isArray(authorizedScopes), 'Expecting array');
-    assert(Array.isArray(requiredScopes), 'Expecting array');
-
-    if (authorizedScopes.indexOf(exports.SCOPE_ANY) !== -1) return null;
-
-    for (var i = 0; i < requiredScopes.length; ++i) {
-        const scopeParts = requiredScopes[i].split(':');
-
-        // this allows apps:write if the token has a higher apps scope
-        if (authorizedScopes.indexOf(requiredScopes[i]) === -1 && authorizedScopes.indexOf(scopeParts[0]) === -1) {
-            debug('scope: missing scope "%s".', requiredScopes[i]);
-            return new BoxError(BoxError.NOT_FOUND, 'Missing required scope "' + requiredScopes[i] + '"');
-        }
-    }
-
-    return null;
-}
-
-function scopesForUser(user, callback) {
-    assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (user.admin) return callback(null, exports.VALID_SCOPES);
-
-    callback(null, [ 'profile', 'apps:read' ]);
-}
-
-function validateToken(accessToken, callback) {
+function verifyToken(accessToken, callback) {
     assert.strictEqual(typeof accessToken, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     tokendb.getByAccessToken(accessToken, function (error, token) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
-        if (error) return callback(error); // this triggers 'internal error' in passport
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+        if (error) return callback(error);
 
         users.get(token.identifier, function (error, user) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
+            if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
             if (error) return callback(error);
 
-            if (!user.active) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
+            if (!user.active) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
 
-            scopesForUser(user, function (error, userScopes) {
-                if (error) return callback(error);
-
-                const authorizedScopes = intersectScopes(userScopes, token.scope.split(','));
-                callback(null, user, { authorizedScopes }); // ends up in req.authInfo
-            });
+            callback(null, user);
         });
     });
 }

src/addons.js

@@ -20,6 +20,8 @@ exports = module.exports = {
     getMountsSync: getMountsSync,
     getContainerNamesSync: getContainerNamesSync,
 
+    getServiceDetails: getServiceDetails,
+
     // exported for testing
     _setupOauth: setupOauth,
     _teardownOauth: teardownOauth,
@@ -29,18 +31,15 @@ exports = module.exports = {
     SERVICE_STATUS_STOPPED: 'stopped'
 };
 
-var accesscontrol = require('./accesscontrol.js'),
-    appdb = require('./appdb.js'),
+var appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
-    clients = require('./clients.js'),
     constants = require('./constants.js'),
     crypto = require('crypto'),
     debug = require('debug')('box:addons'),
     docker = require('./docker.js'),
-    dockerConnection = docker.connection,
     fs = require('fs'),
     graphs = require('./graphs.js'),
     hat = require('./hat.js'),
@@ -232,33 +231,10 @@ function dumpPath(addon, appId) {
     }
 }
 
-function restartContainer(serviceName, callback) {
-    assert.strictEqual(typeof serviceName, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    assert(KNOWN_SERVICES[serviceName], `Unknown service ${serviceName}`);
-
-    docker.stopContainer(serviceName, function (error) {
-        if (error) return callback(error);
-
-        docker.startContainer(serviceName, function (error) {
-            if (error && error.reason === BoxError.NOT_FOUND) {
-                callback(null); // callback early since rebuilding takes long
-                return rebuildService(serviceName, function (error) { if (error) console.error(`Unable to rebuild service ${serviceName}`, error); });
-            }
-            if (error) return callback(error);
-
-            callback(null);
-        });
-    });
-}
-
 function rebuildService(serviceName, callback) {
     assert.strictEqual(typeof serviceName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    assert(KNOWN_SERVICES[serviceName], `Unknown service ${serviceName}`);
-
     // this attempts to recreate the service docker container if they don't exist but platform infra version is unchanged
     // passing an infra version of 'none' will not attempt to purge existing data, not sure if this is good or bad
     if (serviceName === 'mongodb') return startMongodb({ version: 'none' }, callback);
@@ -271,6 +247,24 @@ function rebuildService(serviceName, callback) {
     callback();
 }
 
+function restartContainer(serviceName, callback) {
+    assert.strictEqual(typeof serviceName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.stopContainer(serviceName, function (error) {
+        if (error) return callback(error);
+
+        docker.startContainer(serviceName, function (error) {
+            if (error && error.reason === BoxError.NOT_FOUND) {
+                callback(null); // callback early since rebuilding takes long
+                return rebuildService(serviceName, function (error) { if (error) console.error(`Unable to rebuild service ${serviceName}`, error); });
+            }
+
+            callback(error);
+        });
+    });
+}
+
 function getServiceDetails(containerName, tokenEnvName, callback) {
     assert.strictEqual(typeof containerName, 'string');
     assert.strictEqual(typeof tokenEnvName, 'string');
@@ -280,15 +274,15 @@ function getServiceDetails(containerName, tokenEnvName, callback) {
         if (error) return callback(error);
 
         const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
-        if (!ip) return callback(new BoxError(BoxError.INACTIVE, `Error getting ${containerName} container ip`));
+        if (!ip) return callback(new BoxError(BoxError.INACTIVE, `Error getting IP of ${containerName} service`));
 
         // extract the cloudron token for auth
         const env = safe.query(result, 'Config.Env', null);
-        if (!env) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} env`));
+        if (!env) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error inspecting environment of ${containerName} service`));
         const tmp = env.find(function (e) { return e.indexOf(tokenEnvName) === 0; });
-        if (!tmp) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} cloudron token env var`));
+        if (!tmp) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`));
         const token = tmp.slice(tokenEnvName.length + 1); // +1 for the = sign
-        if (!token)  return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting ${containerName} cloudron token`));
+        if (!token)  return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`));
 
         callback(null, { ip: ip, token: token, state: result.State });
     });
@@ -339,6 +333,9 @@ function getService(serviceName, callback) {
     var tmp = {
         name: serviceName,
         status: null,
+        memoryUsed: 0,
+        memoryPercent: 0,
+        error: null,
         config: {
             // If a property is not set then we cannot change it through the api, see below
             // memory: 0,
@@ -483,8 +480,8 @@ function waitForService(containerName, tokenEnvName, callback) {
 
         async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
             request.get(`https://${result.ip}:3000/healthcheck?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
-                if (error) return retryCallback(new Error(`Error waiting for ${containerName}: ${error.message}`));
-                if (response.statusCode !== 200 || !response.body.status) return retryCallback(new Error(`Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Network error waiting for ${containerName}: ${error.message}`));
+                if (response.statusCode !== 200 || !response.body.status) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 retryCallback(null);
             });
@@ -502,7 +499,7 @@ function setupAddons(app, addons, callback) {
     debugApp(app, 'setupAddons: Setting up %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         debugApp(app, 'Setting up addon %s with options %j', addon, addons[addon]);
 
@@ -520,7 +517,7 @@ function teardownAddons(app, addons, callback) {
     debugApp(app, 'teardownAddons: Tearing down %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         debugApp(app, 'Tearing down addon %s with options %j', addon, addons[addon]);
 
@@ -540,7 +537,7 @@ function backupAddons(app, addons, callback) {
     debugApp(app, 'backupAddons: Backing up %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         KNOWN_ADDONS[addon].backup(app, addons[addon], iteratorCallback);
     }, callback);
@@ -558,7 +555,7 @@ function clearAddons(app, addons, callback) {
     debugApp(app, 'clearAddons: clearing %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         KNOWN_ADDONS[addon].clear(app, addons[addon], iteratorCallback);
     }, callback);
@@ -576,7 +573,7 @@ function restoreAddons(app, addons, callback) {
     debugApp(app, 'restoreAddons: restoring %j', Object.keys(addons));
 
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
-        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
         KNOWN_ADDONS[addon].restore(app, addons[addon], iteratorCallback);
     }, callback);
@@ -587,7 +584,7 @@ function importAppDatabase(app, addon, callback) {
     assert.strictEqual(typeof addon, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!(addon in KNOWN_ADDONS)) return callback(new Error(`No such addon: ${addon}`));
+    if (!(addon in KNOWN_ADDONS)) return callback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`));
 
     async.series([
         KNOWN_ADDONS[addon].setup.bind(null, app, app.manifest.addons[addon]),
@@ -627,7 +624,6 @@ function updateServiceConfig(platformConfig, callback) {
 
     debug('updateServiceConfig: %j', platformConfig);
 
-    // TODO: this should possibly also rollback memory to default
     async.eachSeries([ 'mysql', 'postgresql', 'mail', 'mongodb', 'graphite' ], function iterator(serviceName, iteratorCallback) {
         const containerConfig = platformConfig[serviceName];
         let memory, memorySwap;
@@ -781,29 +777,11 @@ function setupOauth(app, options, callback) {
 
     if (!app.sso) return callback(null);
 
-    var appId = app.id;
-    var redirectURI = 'https://' + app.fqdn;
-    var scope = accesscontrol.SCOPE_PROFILE;
+    const env = [];
 
-    clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds
-        if (error && error.reason !== BoxError.NOT_FOUND) return callback(error);
+    debugApp(app, 'Setting oauth addon config to %j', env);
 
-        clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) {
-            if (error) return callback(error);
-
-            const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
-            var env = [
-                { name: `${envPrefix}OAUTH_CLIENT_ID`, value: result.id },
-                { name: `${envPrefix}OAUTH_CLIENT_SECRET`, value: result.clientSecret },
-                { name: `${envPrefix}OAUTH_ORIGIN`, value: settings.adminOrigin() }
-            ];
-
-            debugApp(app, 'Setting oauth addon config to %j', env);
-
-            appdb.setAddonConfig(appId, 'oauth', env, callback);
-        });
-    });
+    appdb.setAddonConfig(app.id, 'oauth', env, callback);
 }
 
 function teardownOauth(app, options, callback) {
@@ -813,11 +791,7 @@ function teardownOauth(app, options, callback) {
 
     debugApp(app, 'teardownOauth');
 
-    clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) {
-        if (error && error.reason !== BoxError.NOT_FOUND) debug(error);
-
-        appdb.unsetAddonConfig(app.id, 'oauth', callback);
-    });
+    appdb.unsetAddonConfig(app.id, 'oauth', callback);
 }
 
 function setupEmail(app, options, callback) {
@@ -1052,8 +1026,8 @@ function setupMySql(app, options, callback) {
             if (error) return callback(error);
 
             request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up mysql: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mysql: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
@@ -1090,9 +1064,10 @@ function clearMySql(app, options, callback) {
     getServiceDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mysql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+
             callback();
         });
     });
@@ -1109,9 +1084,9 @@ function teardownMySql(app, options, callback) {
     getServiceDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.delete(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mysql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             appdb.unsetAddonConfig(app.id, 'mysql', callback);
         });
@@ -1130,14 +1105,15 @@ function pipeRequestToFile(url, filename, callback) {
         callback(error);
     });
 
-    writeStream.on('error', done);
+    writeStream.on('error', (error) => done(new BoxError(BoxError.FS_ERROR, `Error writing to ${filename}: ${error.message}`)));
+
     writeStream.on('open', function () {
         // note: do not attach to post callback handler because this will buffer the entire reponse!
         // see https://github.com/request/request/issues/2270
         const req = request.post(url, { rejectUnauthorized: false });
-        req.on('error', done); // network error, dns error, request errored in middle etc
+        req.on('error', (error) => done(new BoxError(BoxError.NETWORK_ERROR, `Request error writing to ${filename}: ${error.message}`))); // network error, dns error, request errored in middle etc
         req.on('response', function (response) {
-            if (response.statusCode !== 200) return done(new Error(`Unexpected response code: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`));
+            if (response.statusCode !== 200) return done(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code when piping ${url}: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`));
 
             response.pipe(writeStream).on('finish', done); // this is hit after data written to disk
         });
@@ -1176,11 +1152,11 @@ function restoreMySql(app, options, callback) {
         if (error) return callback(error);
 
         var input = fs.createReadStream(dumpPath('mysql', app.id));
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mysql: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from mysql addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1265,8 +1241,8 @@ function setupPostgreSql(app, options, callback) {
             if (error) return callback(error);
 
             request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up postgresql: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up postgresql: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
@@ -1298,9 +1274,9 @@ function clearPostgreSql(app, options, callback) {
     getServiceDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing postgresql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1317,9 +1293,9 @@ function teardownPostgreSql(app, options, callback) {
     getServiceDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error tearing down postgresql: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error tearing down postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error tearing down postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             appdb.unsetAddonConfig(app.id, 'postgresql', callback);
         });
@@ -1358,11 +1334,11 @@ function restorePostgreSql(app, options, callback) {
         if (error) return callback(error);
 
         var input = fs.createReadStream(dumpPath('postgresql', app.id));
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring postgresql: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from postgresql addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1441,8 +1417,8 @@ function setupMongoDb(app, options, callback) {
             if (error) return callback(error);
 
             request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) {
-                if (error) return callback(new Error('Error setting up mongodb: ' + error));
-                if (response.statusCode !== 201) return callback(new Error(`Error setting up mongodb. Status code: ${response.statusCode}`));
+                if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mongodb: ${error.message}`));
+                if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
                 const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
 
@@ -1476,9 +1452,9 @@ function clearMongodb(app, options, callback) {
     getServiceDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/databases/${app.id}/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing mongodb: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/databases/${app.id}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback();
         });
@@ -1495,9 +1471,9 @@ function teardownMongoDb(app, options, callback) {
     getServiceDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.delete(`https://${result.ip}:3000/databases/${app.id}?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error tearing down mongodb: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error tearing down mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.delete(`https://${result.ip}:3000/databases/${app.id}?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             appdb.unsetAddonConfig(app.id, 'mongodb', callback);
         });
@@ -1532,11 +1508,11 @@ function restoreMongoDb(app, options, callback) {
         if (error) return callback(error);
 
         const readStream = fs.createReadStream(dumpPath('mongodb', app.id));
-        readStream.on('error', callback);
+        readStream.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mongodb: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/databases/${app.id}/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from mongodb addon ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/databases/${app.id}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1654,9 +1630,9 @@ function clearRedis(app, options, callback) {
     getServiceDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) {
         if (error) return callback(error);
 
-        request.post(`https://${result.ip}:3000/clear?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(new Error('Error clearing redis: ' + error));
-            if (response.statusCode !== 200) return callback(new Error(`Error clearing redis. Status code: ${response.statusCode} message: ${response.body.message}`));
+        request.post(`https://${result.ip}:3000/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing redis: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing redis. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1668,18 +1644,11 @@ function teardownRedis(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = dockerConnection.getContainer('redis-' + app.id);
-
-    var removeOptions = {
-        force: true, // kill container if it's running
-        v: true // removes volumes associated with the container
-    };
-
-    container.remove(removeOptions, function (error) {
-        if (error && error.statusCode !== 404) return callback(new Error('Error removing container:' + error));
+    docker.deleteContainer(`redis-${app.id}`, function (error) {
+        if (error) return callback(error);
 
         shell.sudo('removeVolume', [ RMADDONDIR_CMD, 'redis', app.id ], {}, function (error) {
-            if (error) return callback(new Error('Error removing redis data:' + error));
+            if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`));
 
             rimraf(path.join(paths.LOG_DIR, `redis-${app.id}`), function (error) {
                 if (error) debugApp(app, 'cannot cleanup logs: %s', error);
@@ -1712,6 +1681,8 @@ function restoreRedis(app, options, callback) {
 
     debugApp(app, 'Restoring redis');
 
+    callback = once(callback); // protect from multiple returns with streams
+
     getServiceDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) {
         if (error) return callback(error);
 
@@ -1722,11 +1693,11 @@ function restoreRedis(app, options, callback) {
         } else { // old location of dumps
             input = fs.createReadStream(path.join(paths.APPS_DATA_DIR, app.id, 'redis/dump.rdb'));
         }
-        input.on('error', callback);
+        input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring redis: ${error.message}`)));
 
-        const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { rejectUnauthorized: false }, function (error, response) {
-            if (error) return callback(error);
-            if (response.statusCode !== 200) return callback(new Error(`Unexpected response from redis addon: ${response.statusCode} message: ${response.body.message}`));
+        const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis: ${error.message}`));
+            if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis. Status code: ${response.statusCode} message: ${response.body.message}`));
 
             callback(null);
         });
@@ -1811,7 +1782,7 @@ function statusGraphite(callback) {
         if (error && error.reason === BoxError.NOT_FOUND) return callback(null, { status: exports.SERVICE_STATUS_STOPPED });
         if (error) return callback(error);
 
-        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { timeout: 3000 }, function (error, response) {
+        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { json: true, timeout: 3000 }, function (error, response) {
             if (error) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite: ${error.message}` });
             if (response.statusCode !== 200) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite. Status code: ${response.statusCode} message: ${response.body.message}` });
 

src/appdb.js

@@ -40,7 +40,7 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'subdomains.subdomain AS location', 'subdomains.domain',
-    'apps.accessRestrictionJson', 'apps.memoryLimit',
+    'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup',
     'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
@@ -242,6 +242,7 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
     const accessRestriction = data.accessRestriction || null;
     const accessRestrictionJson = JSON.stringify(accessRestriction);
     const memoryLimit = data.memoryLimit || 0;
+    const cpuShares = data.cpuShares || 512;
     const installationState = data.installationState;
     const runState = data.runState;
     const sso = 'sso' in data ? data.sso : null;
@@ -256,10 +257,10 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
     var queries = [];
 
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, '
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
             + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit,
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
             sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson ]
     });
 
@@ -348,12 +349,13 @@ function del(id, callback) {
         { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
         { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
     ];
 
     database.transaction(queries, function (error, results) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[3].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+        if (results[4].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
 
         callback(null);
     });

src/apphealthmonitor.js

@@ -186,9 +186,10 @@ function processApp(callback) {
         async.each(result, checkAppHealth, function (error) {
             if (error) console.error(error);
 
-            var alive = result
+            const alive = result
                 .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; })
-                .map(function (a) { return (a.location || 'naked_domain') + '|' + (a.manifest.id || 'customapp'); }).join(', ');
+                .map(a => a.fqdn)
+                .join(', ');
 
             debug('apps alive: [%s]', alive);
 

src/apps.js

@@ -19,6 +19,7 @@ exports = module.exports = {
     setIcon: setIcon,
     setTags: setTags,
     setMemoryLimit: setMemoryLimit,
+    setCpuShares: setCpuShares,
     setAutomaticBackup: setAutomaticBackup,
     setAutomaticUpdate: setAutomaticUpdate,
     setReverseProxyConfig: setReverseProxyConfig,
@@ -43,6 +44,7 @@ exports = module.exports = {
 
     start: start,
     stop: stop,
+    restart: restart,
 
     exec: exec,
 
@@ -79,6 +81,7 @@ exports = module.exports = {
     ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app. this is state because it blocks other operations
     ISTATE_PENDING_START: 'pending_start',
     ISTATE_PENDING_STOP: 'pending_stop',
+    ISTATE_PENDING_RESTART: 'pending_restart',
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
@@ -129,6 +132,7 @@ var appdb = require('./appdb.js'),
     tasks = require('./tasks.js'),
     TransformStream = require('stream').Transform,
     updateChecker = require('./updatechecker.js'),
+    users = require('./users.js'),
     util = require('util'),
     uuid = require('uuid'),
     validator = require('validator'),
@@ -158,7 +162,6 @@ function validatePortBindings(portBindings, manifest) {
         993, /* imaps */
         2003, /* graphite (lo) */
         2004, /* graphite (lo) */
-        2020, /* mail server */
         2514, /* cloudron-syslog (lo) */
         constants.PORT, /* app server (lo) */
         constants.SYSADMIN_PORT, /* sysadmin app server (lo) */
@@ -247,6 +250,14 @@ function validateMemoryLimit(manifest, memoryLimit) {
     return null;
 }
 
+function validateCpuShares(cpuShares) {
+    assert.strictEqual(typeof cpuShares, 'number');
+
+    if (cpuShares < 2 || cpuShares > 1024) return new BoxError(BoxError.BAD_FIELD, 'cpuShares has to be between 2 and 1024');
+
+    return null;
+}
+
 function validateDebugMode(debugMode) {
     assert.strictEqual(typeof debugMode, 'object');
 
@@ -316,23 +327,25 @@ function validateEnv(env) {
 function validateDataDir(dataDir) {
     if (dataDir === null) return null;
 
-    if (path.resolve(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, 'dataDir must be an absolute path', { field: 'dataDir' });
+    if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`, { field: 'dataDir' });
+    if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`, { field: 'dataDir' });
+    if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`, { field: 'dataDir' });
 
     // nfs shares will have the directory mounted already
     let stat = safe.fs.lstatSync(dataDir);
     if (stat) {
-        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} is not a directory`, { field: 'dataDir' });
+        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`, { field: 'dataDir' });
         let entries = safe.fs.readdirSync(dataDir);
-        if (!entries) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} could not be listed`, { field: 'dataDir' });
-        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} is not empty`, { field: 'dataDir' });
+        if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`, { field: 'dataDir' });
+        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`, { field: 'dataDir' });
     }
 
     // backup logic relies on paths not overlapping (because it recurses)
-    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} cannot be inside apps data`, { field: 'dataDir' });
+    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`, { field: 'dataDir' });
 
     // if we made it this far, it cannot start with any of these realistically
     const fhs = [ '/bin', '/boot', '/etc', '/lib', '/lib32', '/lib64', '/proc', '/run', '/sbin', '/tmp', '/usr' ];
-    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} cannot be placed inside this location`, { field: 'dataDir' });
+    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`, { field: 'dataDir' });
 
     return null;
 }
@@ -381,7 +394,7 @@ function removeInternalFields(app) {
     return _.pick(app,
         'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId',
         'location', 'domain', 'fqdn', 'mailboxName', 'mailboxDomain',
-        'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit',
+        'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
         'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir');
 }
@@ -443,7 +456,7 @@ function hasAccessTo(app, user, callback) {
     // check user access
     if (app.accessRestriction.users.some(function (e) { return e === user.id; })) return callback(null, true);
 
-    if (user.admin) return callback(null, true); // admins can always access any app
+    if (users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) return callback(null, true); // admins can always access any app
 
     if (!app.accessRestriction.groups) return callback(null, false);
 
@@ -622,8 +635,8 @@ function addTask(appId, installationState, task, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     const { args, values } = task;
-    // by default, a task can only run on installed state. if it's null, it can be run on any state
-    const requiredState = 'requiredState' in task ? task.requiredState : exports.ISTATE_INSTALLED;
+    // TODO: match the SQL logic to match checkAppState. this means checking the error.installationState and installationState. Unfortunately, former is JSON right now
+    const requiredState = null; // 'requiredState' in task ? task.requiredState : exports.ISTATE_INSTALLED;
     const scheduleNow = 'scheduleNow' in task ? task.scheduleNow : true;
     const requireNullTaskId = 'requireNullTaskId' in task ? task.requireNullTaskId : true;
 
@@ -645,10 +658,14 @@ function checkAppState(app, state) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof state, 'string');
 
-    if (app.taskId) return new BoxError(BoxError.BAD_STATE, `Not allowed in this app state : ${app.installationState} / ${app.runState}`);
+    if (app.taskId) return new BoxError(BoxError.BAD_STATE, `Locked by task ${app.taskId} : ${app.installationState} / ${app.runState}`);
 
     if (app.installationState === exports.ISTATE_ERROR) {
-        if (state !== exports.ISTATE_PENDING_UNINSTALL) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
+        // allow task to be called again if that was the errored task
+        if (app.error.installationState === state) return null;
+
+        // allow uninstall from any state
+        if (state !== exports.ISTATE_PENDING_UNINSTALL && state !== exports.ISTATE_PENDING_RESTORE) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
     }
 
     return null;
@@ -927,6 +944,35 @@ function setMemoryLimit(appId, memoryLimit, auditSource, callback) {
     });
 }
 
+function setCpuShares(appId, cpuShares, auditSource, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof cpuShares, 'number');
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(appId, function (error, app) {
+        if (error) return callback(error);
+
+        error = checkAppState(app, exports.ISTATE_PENDING_RESIZE);
+        if (error) return callback(error);
+
+        error = validateCpuShares(cpuShares);
+        if (error) return callback(error);
+
+        const task = {
+            args: {},
+            values: { cpuShares }
+        };
+        addTask(appId, exports.ISTATE_PENDING_RESIZE, task, function (error, result) {
+            if (error) return callback(error);
+
+            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId, app: app, cpuShares, taskId: result.taskId });
+
+            callback(null, { taskId: result.taskId });
+        });
+    });
+}
+
 function setEnvironment(appId, env, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof env, 'object');
@@ -1196,13 +1242,13 @@ function setDataDir(appId, dataDir, auditSource, callback) {
         if (error) return callback(error);
 
         const task = {
-            args: { oldDataDir: app.dataDir },
-            values: { dataDir: dataDir }
+            args: { newDataDir: dataDir },
+            values: { }
         };
         addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, function (error, result) {
             if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId, app: app, dataDir: dataDir, taskId: result.taskId });
+            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId, app: app, newDataDir: dataDir, taskId: result.taskId });
 
             callback(null, { taskId: result.taskId });
         });
@@ -1225,6 +1271,8 @@ function update(appId, data, auditSource, callback) {
         error = checkAppState(app, exports.ISTATE_PENDING_UPDATE);
         if (error) return callback(error);
 
+        if (app.runState === exports.RSTATE_STOPPED) return callback(new BoxError(BoxError.BAD_STATE, 'Stopped apps cannot be updated'));
+
         downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
             if (error) return callback(error);
 
@@ -1344,9 +1392,11 @@ function getLogs(appId, options, callback) {
     });
 }
 
+// does a re-configure when called from most states. for install/clone errors, it re-installs with an optional manifest
+// re-configure can take a dockerImage but not a manifest because re-configure does not clean up addons
 function repair(appId, data, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof data, 'object'); // { manifest }
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -1355,42 +1405,48 @@ function repair(appId, data, auditSource, callback) {
     get(appId, function (error, app) {
         if (error) return callback(error);
 
-        const appError = app.error || {}; // repair can always be called
-        const newState = appError.installationState ? appError.installationState : exports.ISTATE_PENDING_CONFIGURE;
+        let errorState = (app.error && app.error.installationState) || exports.ISTATE_PENDING_CONFIGURE;
 
-        debug(`Repairing app with error: ${JSON.stringify(error)} and state: ${newState}`);
+        const task = {
+            args: {},
+            values: {},
+            requiredState: null
+        };
 
-        let values = _.pick(data, 'location', 'domain', 'alternateDomains');
+        // maybe split this into a separate route like reinstall?
+        if (errorState === exports.ISTATE_PENDING_INSTALL || errorState === exports.ISTATE_PENDING_CLONE) {
+            task.args = { overwriteDns: true };
+            if (data.manifest) {
+                error = manifestFormat.parse(data.manifest);
+                if (error) return callback(new BoxError(BoxError.BAD_FIELD, `manifest error: ${error.message}`));
 
-        const locations = (values.location ? [ { subdomain: values.location, domain: values.domain } ] : []).concat(values.alternateDomains || []);
-        validateLocations(locations, function (error, domainObjectMap) {
+                error = checkManifestConstraints(data.manifest);
+                if (error) return callback(error);
+
+                task.values.manifest = data.manifest;
+                task.args.oldManifest = app.manifest;
+            }
+        } else {
+            errorState = exports.ISTATE_PENDING_CONFIGURE;
+            if (data.dockerImage) {
+                let newManifest = _.extend({}, app.manifest, { dockerImage: data.dockerImage });
+                task.values.manifest = newManifest;
+            }
+        }
+
+        addTask(appId, errorState, task, function (error, result) {
             if (error) return callback(error);
 
-            tasks.get(appError.taskId || '', function (error, task) {
-                let args = !error ? task.args[1] : {}; // pick args for the failed task. the first argument is the app id
+            eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { taskId: result.taskId, app });
 
-                if ('backupId' in data) {
-                    args.restoreConfig = data.backupId ? { backupId: data.backupId, backupFormat: data.backupFormat, oldManifest: app.manifest } : null; // when null, apptask simply reinstalls
-                }
-                args.overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false;
-
-                // create a new task instead of updating the old one, since it helps tracking
-                addTask(appId, newState, { args, values, requiredState: null }, function (error, result) {
-                    if (error && error.reason === BoxError.ALREADY_EXISTS) error = getDuplicateErrorDetails(error.message, locations, domainObjectMap, { /* portBindings */});
-                    if (error) return callback(error);
-
-                    eventlog.add(eventlog.ACTION_APP_REPAIR, auditSource, { taskId: result.taskId, app, newState });
-
-                    callback(null, { taskId: result.taskId });
-                });
-            });
+            callback(null, { taskId: result.taskId });
         });
     });
 }
 
-function restore(appId, data, auditSource, callback) {
+function restore(appId, backupId, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -1403,7 +1459,7 @@ function restore(appId, data, auditSource, callback) {
         if (error) return callback(error);
 
         // for empty or null backupId, use existing manifest to mimic a reinstall
-        var func = data.backupId ? backups.get.bind(null, data.backupId) : function (next) { return next(null, { manifest: app.manifest }); };
+        var func = backupId ? backups.get.bind(null, backupId) : function (next) { return next(null, { manifest: app.manifest }); };
 
         func(function (error, backupInfo) {
             if (error) return callback(error);
@@ -1414,11 +1470,12 @@ function restore(appId, data, auditSource, callback) {
             error = checkManifestConstraints(backupInfo.manifest);
             if (error) return callback(error);
 
-            const restoreConfig = { backupId: data.backupId, backupFormat: backupInfo.format, oldManifest: app.manifest };
+            const restoreConfig = { backupId, backupFormat: backupInfo.format };
 
             const task = {
                 args: {
                     restoreConfig,
+                    oldManifest: app.manifest,
                     overwriteDns: true
                 },
                 values: {
@@ -1447,28 +1504,41 @@ function importApp(appId, data, auditSource, callback) {
     get(appId, function (error, app) {
         if (error) return callback(error);
 
-        error = validateBackupFormat(data.backupFormat);
+        // all fields are optional
+        data.backupId = data.backupId || null;
+        data.backupFormat = data.backupFormat || null;
+        data.backupConfig = data.backupConfig || null;
+        const { backupId, backupFormat, backupConfig } = data;
+
+        error = backupFormat ? validateBackupFormat(backupFormat) : null;
         if (error) return callback(error);
 
         error = checkAppState(app, exports.ISTATE_PENDING_RESTORE);
         if (error) return callback(error);
 
-        // TODO: check if the file exists in the storage backend
-        const restoreConfig = { backupId: data.backupId, backupFormat: data.backupFormat, oldManifest: app.manifest };
+        // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
+        const testBackupConfig = backupConfig ? backups.testProviderConfig.bind(null, backupConfig) : (next) => next();
 
-        const task = {
-            args: {
-                restoreConfig,
-                overwriteDns: true
-            },
-            values: {}
-        };
-        addTask(appId, exports.ISTATE_PENDING_RESTORE, task, function (error, result) {
+        testBackupConfig(function (error) {
             if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId: data.backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId: result.taskId });
+            const restoreConfig = { backupId, backupFormat, backupConfig };
 
-            callback(null, { taskId: result.taskId });
+            const task = {
+                args: {
+                    restoreConfig,
+                    oldManifest: app.manifest,
+                    overwriteDns: true
+                },
+                values: {}
+            };
+            addTask(appId, exports.ISTATE_PENDING_RESTORE, task, function (error, result) {
+                if (error) return callback(error);
+
+                eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId: result.taskId });
+
+                callback(null, { taskId: result.taskId });
+            });
         });
     });
 }
@@ -1526,7 +1596,7 @@ function clone(appId, data, user, auditSource, callback) {
             error = validatePortBindings(portBindings, manifest);
             if (error) return callback(error);
 
-            const mailboxName = mailboxNameForLocation(location, manifest);
+            const mailboxName = app.mailboxName.endsWith('.app') ? mailboxNameForLocation(location, manifest) : app.mailboxName;
             const locations = [{subdomain: location, domain}];
             validateLocations(locations, function (error, domainObjectMap) {
                 if (error) return callback(error);
@@ -1554,9 +1624,9 @@ function clone(appId, data, user, auditSource, callback) {
                     purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' }, function (error) {
                         if (error) return callback(error);
 
-                        const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format, oldManifest: null };
+                        const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format };
                         const task = {
-                            args: { restoreConfig, overwriteDns },
+                            args: { restoreConfig, overwriteDns, oldManifest: null },
                             values: {},
                             requiredState: exports.ISTATE_PENDING_CLONE
                         };
@@ -1657,6 +1727,30 @@ function stop(appId, callback) {
     });
 }
 
+function restart(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will restart app with id:%s', appId);
+
+    get(appId, function (error, app) {
+        if (error) return callback(error);
+
+        error = checkAppState(app, exports.ISTATE_PENDING_RESTART);
+        if (error) return callback(error);
+
+        const task = {
+            args: {},
+            values: { runState: exports.RSTATE_RUNNING }
+        };
+        addTask(appId, exports.ISTATE_PENDING_RESTART, task, function (error, result) {
+            if (error) return callback(error);
+
+            callback(null, { taskId: result.taskId });
+        });
+    });
+}
+
 function checkManifestConstraints(manifest) {
     assert(manifest && typeof manifest === 'object');
 
@@ -1690,8 +1784,6 @@ function exec(appId, options, callback) {
             return callback(new BoxError(BoxError.BAD_STATE, 'App not installed or running'));
         }
 
-        var container = docker.connection.getContainer(app.containerId);
-
         var execOptions = {
             AttachStdin: true,
             AttachStdout: true,
@@ -1704,55 +1796,46 @@ function exec(appId, options, callback) {
             Cmd: cmd
         };
 
-        container.exec(execOptions, function (error, exec) {
+        var startOptions = {
+            Detach: false,
+            Tty: options.tty,
+            // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
+            // we can work with half-close connections (not defined in http). this way, the client
+            // can properly signal that stdin is EOF by closing it's side of the socket. In http,
+            // the whole connection will be dropped when stdin get EOF.
+            // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
+            hijack: true,
+            stream: true,
+            stdin: true,
+            stdout: true,
+            stderr: true
+        };
+
+        docker.execContainer(app.containerId, { execOptions, startOptions, rows: options.rows, columns: options.columns }, function (error, stream) {
             if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running
             if (error) return callback(error);
 
-            var startOptions = {
-                Detach: false,
-                Tty: options.tty,
-                // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
-                // we can work with half-close connections (not defined in http). this way, the client
-                // can properly signal that stdin is EOF by closing it's side of the socket. In http,
-                // the whole connection will be dropped when stdin get EOF.
-                // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
-                hijack: true,
-                stream: true,
-                stdin: true,
-                stdout: true,
-                stderr: true
-            };
-            exec.start(startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
-                if (error) return callback(error);
-
-                if (options.rows && options.columns) {
-                    // there is a race where resizing too early results in a 404 "no such exec"
-                    // https://git.cloudron.io/cloudron/box/issues/549
-                    setTimeout(function () {
-                        exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
-                    }, 2000);
-                }
-
-                return callback(null, stream);
-            });
+            callback(null, stream);
         });
     });
 }
 
 function canAutoupdateApp(app, newManifest) {
-    if (!app.enableAutomaticUpdate) return new Error('Automatic update disabled');
-    if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(newManifest.version))) return new Error('Major version change'); // major changes are blocking
+    if (!app.enableAutomaticUpdate) return false;
+    if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(newManifest.version))) return false; // major changes are blocking
+
+    if (app.runState === exports.RSTATE_STOPPED) return false; // stopped apps won't run migration scripts and shouldn't be updated
 
     const newTcpPorts = newManifest.tcpPorts || { };
     const newUdpPorts = newManifest.udpPorts || { };
     const portBindings = app.portBindings; // this is never null
 
     for (let portName in portBindings) {
-        if (!(portName in newTcpPorts) && !(portName in newUdpPorts)) return new Error(`${portName} was in use but new update removes it`);
+        if (!(portName in newTcpPorts) && !(portName in newUdpPorts)) return false; // portName was in use but new update removes it
     }
 
     // it's fine if one or more (unused) keys got removed
-    return null;
+    return true;
 }
 
 function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is { appId -> { manifest } }
@@ -1769,9 +1852,8 @@ function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is {
                 return iteratorDone();
             }
 
-            error = canAutoupdateApp(app, updateInfo[appId].manifest);
-            if (error) {
-                debug('app %s requires manual update. %s', appId, error.message);
+            if (!canAutoupdateApp(app, updateInfo[appId].manifest)) {
+                debug(`app ${app.fqdn} requires manual update`);
                 return iteratorDone();
             }
 
@@ -1841,17 +1923,19 @@ function restoreInstalledApps(callback) {
 
         async.eachSeries(apps, function (app, iteratorDone) {
             backups.getByAppIdPaged(1, 1, app.id, function (error, results) {
-                let installationState, restoreConfig;
+                let installationState, restoreConfig, oldManifest;
                 if (!error && results.length) {
                     installationState = exports.ISTATE_PENDING_RESTORE;
-                    restoreConfig = { backupId: results[0].id, backupFormat: results[0].format, oldManifest: app.manifest };
+                    restoreConfig = { backupId: results[0].id, backupFormat: results[0].format };
+                    oldManifest = app.manifest;
                 } else {
                     installationState = exports.ISTATE_PENDING_INSTALL;
                     restoreConfig = null;
+                    oldManifest = null;
                 }
 
                 const task = {
-                    args: { restoreConfig, overwriteDns: true },
+                    args: { restoreConfig, overwriteDns: true, oldManifest },
                     values: {},
                     scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                     requireNullTaskId: false // ignore existing stale taskId
@@ -1882,9 +1966,8 @@ function configureInstalledApps(callback) {
         async.eachSeries(apps, function (app, iteratorDone) {
             debug(`configureInstalledApps: marking ${app.fqdn} for reconfigure`);
 
-            const oldConfig = _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'portBindings');
             const task = {
-                args: { oldConfig, overwriteDns: true },
+                args: {},
                 values: {},
                 scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                 requireNullTaskId: false // ignore existing stale taskId
@@ -1994,7 +2077,8 @@ function uploadFile(appId, sourceFilePath, destFilePath, callback) {
 
     const done = once(function (error) {
         safe.fs.unlinkSync(sourceFilePath); // remove file in /tmp
-        callback(error);
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, error.message)); // blame it on filesystem for now
+        callback(null);
     });
 
     // the built-in bash printf understands "%q" but not /usr/bin/printf.

src/appstore.js

@@ -1,16 +1,22 @@
 'use strict';
 
 exports = module.exports = {
+    getFeatures: getFeatures,
+
     getApps: getApps,
     getApp: getApp,
     getAppVersion: getAppVersion,
 
+    trackBeginSetup: trackBeginSetup,
+    trackFinishedSetup: trackFinishedSetup,
+
     registerWithLoginCredentials: registerWithLoginCredentials,
     registerWithLicense: registerWithLicense,
 
     purchaseApp: purchaseApp,
     unpurchaseApp: unpurchaseApp,
 
+    getUserToken: getUserToken,
     getSubscription: getSubscription,
     isFreePlan: isFreePlan,
 
@@ -27,13 +33,13 @@ var apps = require('./apps.js'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    custom = require('./custom.js'),
     debug = require('debug')('box:appstore'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     groups = require('./groups.js'),
     mail = require('./mail.js'),
     os = require('os'),
+    paths = require('./paths.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     settings = require('./settings.js'),
@@ -43,6 +49,38 @@ var apps = require('./apps.js'),
 
 const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
+// These are the default options and will be adjusted once a subscription state is obtained
+// Keep in sync with appstore/routes/cloudrons.js
+let gFeatures = {
+    userMaxCount: null,
+    externalLdap: true,
+    eventLog: true,
+    privateDockerRegistry: true,
+    branding: true,
+    userManager: true,
+    multiAdmin: true,
+    support: true
+};
+
+// attempt to load feature cache in case appstore would be down
+let tmp = safe.JSON.parse(safe.fs.readFileSync(paths.FEATURES_INFO_FILE, 'utf8'));
+if (tmp) gFeatures = tmp;
+
+function getFeatures() {
+    return gFeatures;
+}
+
+function isAppAllowed(appstoreId, listingConfig) {
+    assert.strictEqual(typeof listingConfig, 'object');
+    assert.strictEqual(typeof appstoreId, 'string');
+
+    if (listingConfig.blacklist && listingConfig.blacklist.includes(appstoreId)) return false;
+
+    if (listingConfig.whitelist) return listingConfig.whitelist.includes(appstoreId);
+
+    return true;
+}
+
 function getCloudronToken(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -96,6 +134,23 @@ function registerUser(email, password, callback) {
     });
 }
 
+function getUserToken(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
+
+        const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
+
+        superagent.post(url).send({}).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${result.status}`));
+
+            callback(null, result.body.accessToken);
+        });
+    });
+}
+
 function getSubscription(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -110,7 +165,11 @@ function getSubscription(callback) {
             if (result.statusCode === 502) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`));
             if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`));
 
-            callback(null, result.body); // { email, subscription }
+            // update the features cache
+            gFeatures = result.body.features;
+            safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
+
+            callback(null, result.body);
         });
     });
 }
@@ -375,6 +434,35 @@ function registerCloudron(data, callback) {
     });
 }
 
+// This works without a Cloudron token as this Cloudron was not yet registered
+let gBeginSetupAlreadyTracked = false;
+function trackBeginSetup(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
+    // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
+    if (gBeginSetupAlreadyTracked) return;
+    gBeginSetupAlreadyTracked = true;
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
+
+    superagent.post(url).send({ provider }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
+// This works without a Cloudron token as this Cloudron was not yet registered
+function trackFinishedSetup(domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;
+
+    superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
 function registerWithLicense(license, domain, callback) {
     assert.strictEqual(typeof license, 'string');
     assert.strictEqual(typeof domain, 'string');
@@ -383,7 +471,10 @@ function registerWithLicense(license, domain, callback) {
     getCloudronToken(function (error, token) {
         if (token) return callback(new BoxError(BoxError.CONFLICT));
 
-        registerCloudron({ license, domain }, callback);
+        const provider = settings.provider();
+        const version = constants.VERSION;
+
+        registerCloudron({ license, domain, provider, version }, callback);
     });
 }
 
@@ -406,19 +497,20 @@ function registerWithLoginCredentials(options, callback) {
             login(options.email, options.password, options.totpToken || '', function (error, result) {
                 if (error) return callback(error);
 
-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken }, callback);
+                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, provider: settings.provider(), version: constants.VERSION, purpose: options.purpose || '' }, callback);
             });
         });
     });
 }
 
-function createTicket(info, callback) {
+function createTicket(info, auditSource, callback) {
     assert.strictEqual(typeof info, 'object');
     assert.strictEqual(typeof info.email, 'string');
     assert.strictEqual(typeof info.displayName, 'string');
     assert.strictEqual(typeof info.type, 'string');
     assert.strictEqual(typeof info.subject, 'string');
     assert.strictEqual(typeof info.description, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     function collectAppInfoIfNeeded(callback) {
@@ -435,7 +527,7 @@ function createTicket(info, callback) {
 
             let url = settings.apiServerOrigin() + '/api/v1/ticket';
 
-            info.supportEmail = custom.spec().support.email; // destination address for tickets
+            info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
 
             superagent.post(url).query({ accessToken: token }).send(info).timeout(10 * 1000).end(function (error, result) {
                 if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
@@ -443,7 +535,9 @@ function createTicket(info, callback) {
                 if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
                 if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-                callback(null);
+                eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+
+                callback(null, { message: `An email for sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
             });
         });
     });
@@ -466,7 +560,13 @@ function getApps(callback) {
                 if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
                 if (!result.body.apps) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-                callback(null, result.body.apps);
+                settings.getAppstoreListingConfig(function (error, listingConfig) {
+                    if (error) return callback(error);
+
+                    const filteredApps = result.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
+
+                    callback(null, filteredApps);
+                });
             });
         });
     });
@@ -477,20 +577,26 @@ function getAppVersion(appId, version, callback) {
     assert.strictEqual(typeof version, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
+    settings.getAppstoreListingConfig(function (error, listingConfig) {
         if (error) return callback(error);
 
-        let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
-        if (version !== 'latest') url += `/versions/${version}`;
+        if (!isAppAllowed(appId, listingConfig)) return callback(new BoxError(BoxError.FEATURE_DISABLED));
 
-        superagent.get(url).query({ accessToken: token }).timeout(10 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
+        getCloudronToken(function (error, token) {
+            if (error) return callback(error);
 
-            callback(null, result.body);
+            let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
+            if (version !== 'latest') url += `/versions/${version}`;
+
+            superagent.get(url).query({ accessToken: token }).timeout(10 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
+
+                callback(null, result.body);
+            });
         });
     });
 }

src/apptask.js

@@ -27,6 +27,7 @@ var addons = require('./addons.js'),
     auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
+    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:apptask'),
     df = require('@sindresorhus/df'),
@@ -51,8 +52,7 @@ var addons = require('./addons.js'),
     util = require('util'),
     _ = require('underscore');
 
-const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
-    CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh'),
+const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/app.ejs', { encoding: 'utf8' }),
     MV_VOLUME_CMD = path.join(__dirname, 'scripts/mvvolume.sh'),
     LOGROTATE_CONFIG_EJS = fs.readFileSync(__dirname + '/logrotate.ejs', { encoding: 'utf8' }),
     CONFIGURE_LOGROTATE_CMD = path.join(__dirname, 'scripts/configurelogrotate.sh');
@@ -64,11 +64,13 @@ function debugApp(app) {
 }
 
 function makeTaskError(error, app) {
-    let boxError = error instanceof BoxError ? error : new BoxError(BoxError.UNKNOWN_ERROR, error.message); // until we port everything to BoxError
+    assert.strictEqual(typeof error, 'object');
+    assert.strictEqual(typeof app, 'object');
+
     // track a few variables which helps 'repair' restart the task (see also scheduleTask in apps.js)
-    boxError.details.taskId = app.taskId;
-    boxError.details.installationState = app.installationState;
-    return boxError.toPlainObject();
+    error.details.taskId = app.taskId;
+    error.details.installationState = app.installationState;
+    return error.toPlainObject();
 }
 
 // updates the app object and the database
@@ -140,7 +142,15 @@ function createContainer(app, callback) {
     docker.createContainer(app, function (error, container) {
         if (error) return callback(error);
 
-        updateApp(app, { containerId: container.id }, callback);
+        updateApp(app, { containerId: container.id }, function (error) {
+            if (error) return callback(error);
+
+            // re-generate configs that rely on container id
+            async.series([
+                addLogrotateConfig.bind(null, app),
+                addCollectdProfile.bind(null, app)
+            ], callback);
+        });
     });
 }
 
@@ -151,11 +161,14 @@ function deleteContainers(app, options, callback) {
 
     debugApp(app, 'deleting app containers (app, scheduler)');
 
-    docker.deleteContainers(app.id, options, function (error) {
-        if (error) return callback(error);
-
-        updateApp(app, { containerId: null }, callback);
-    });
+    async.series([
+        // remove configs that rely on container id
+        removeCollectdProfile.bind(null, app),
+        removeLogrotateConfig.bind(null, app),
+        docker.stopContainers.bind(null, app.id),
+        docker.deleteContainers.bind(null, app.id, options),
+        updateApp.bind(null, app, { containerId: null })
+    ], callback);
 }
 
 function createAppDir(app, callback) {
@@ -216,29 +229,14 @@ function addCollectdProfile(app, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir: apps.getDataDir(app, app.dataDir) });
-    fs.writeFile(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), collectdConf, function (error) {
-        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
-
-        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', app.id ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
-
-            callback(null);
-        });
-    });
+    collectd.addProfile(app.id, collectdConf, callback);
 }
 
 function removeCollectdProfile(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), function (error) {
-        if (error && error.code !== 'ENOENT') debugApp(app, 'Error removing collectd profile', error);
-        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', app.id ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
-
-            callback(null);
-        });
-    });
+    collectd.removeProfile(app.id, callback);
 }
 
 function addLogrotateConfig(app, callback) {
@@ -448,16 +446,18 @@ function waitForDnsPropagation(app, callback) {
     });
 }
 
-function moveDataDir(app, sourceDir, callback) {
+function moveDataDir(app, targetDir, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert(sourceDir === null || typeof sourceDir === 'string');
+    assert(targetDir === null || typeof targetDir === 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let resolvedSourceDir = apps.getDataDir(app, sourceDir);
-    let resolvedTargetDir = apps.getDataDir(app, app.dataDir);
+    let resolvedSourceDir = apps.getDataDir(app, app.dataDir);
+    let resolvedTargetDir = apps.getDataDir(app, targetDir);
 
     debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
 
+    if (resolvedSourceDir === resolvedTargetDir) return callback();
+
     shell.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}, function (error) {
         if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`));
 
@@ -492,17 +492,6 @@ function startApp(app, callback){
     docker.startContainer(app.id, callback);
 }
 
-// Ordering is based on the following rationale:
-//   - configure nginx, icon, oauth
-//   - register subdomain.
-//          at this point, the user can visit the site and the above nginx config can show some install screen.
-//          the icon can be displayed in this nginx page and oauth proxy means the page can be protected
-//   - download image
-//   - setup volumes
-//   - setup addons (requires the above volume)
-//   - setup the container (requires image, volumes, addons)
-//   - setup collectd (requires container id)
-// restore is also handled here since restore is just an install with some oldConfig to clean up
 function install(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
@@ -511,6 +500,7 @@ function install(app, args, progressCallback, callback) {
 
     const restoreConfig = args.restoreConfig; // has to be set when restoring
     const overwriteDns = args.overwriteDns;
+    const oldManifest = args.oldManifest;
 
     async.series([
         // this protects against the theoretical possibility of an app being marked for install/restore from
@@ -520,29 +510,29 @@ function install(app, args, progressCallback, callback) {
         // teardown for re-installs
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
         unconfigureReverseProxy.bind(null, app),
-        removeCollectdProfile.bind(null, app),
-        removeLogrotateConfig.bind(null, app),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
         function teardownAddons(next) {
             // when restoring, app does not require these addons anymore. remove carefully to preserve the db passwords
             let addonsToRemove;
-            if (restoreConfig && restoreConfig.oldManifest) { // oldManifest is null for clone
-                addonsToRemove = _.omit(restoreConfig.oldManifest.addons, Object.keys(app.manifest.addons));
+            if (oldManifest) {
+                addonsToRemove = _.omit(oldManifest.addons, Object.keys(app.manifest.addons));
             } else {
                 addonsToRemove = app.manifest.addons;
             }
 
             addons.teardownAddons(app, addonsToRemove, next);
         },
-        deleteAppDir.bind(null, app, { removeDirectory: false }), // do not remove any symlinked appdata dir
+
+        function deleteAppDirIfNeeded(done) {
+            if (restoreConfig && !restoreConfig.backupId) return done(); // in-place import should not delete data dir
+
+            deleteAppDir(app, { removeDirectory: false }, done); // do not remove any symlinked appdata dir
+        },
 
         function deleteImageIfChanged(done) {
-            if (!restoreConfig || !restoreConfig.oldManifest) return done();
+            if (!oldManifest || oldManifest.dockerImage === app.manifest.dockerImage) return done();
 
-            if (restoreConfig.oldManifest.dockerImage === app.manifest.dockerImage) return done();
-
-            docker.deleteImage(restoreConfig.oldManifest, done);
+            docker.deleteImage(oldManifest, done);
         },
 
         reserveHttpPort.bind(null, app),
@@ -565,14 +555,22 @@ function install(app, args, progressCallback, callback) {
                     progressCallback.bind(null, { percent: 60, message: 'Setting up addons' }),
                     addons.setupAddons.bind(null, app, app.manifest.addons),
                 ], next);
+            } else if (!restoreConfig.backupId) { // in-place import
+                async.series([
+                    progressCallback.bind(null, { percent: 60, message: 'Importing addons in-place' }),
+                    addons.setupAddons.bind(null, app, app.manifest.addons),
+                    addons.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
+                    addons.restoreAddons.bind(null, app, app.manifest.addons),
+                ], next);
             } else {
                 async.series([
                     progressCallback.bind(null, { percent: 65, message: 'Download backup and restoring addons' }),
                     addons.setupAddons.bind(null, app, app.manifest.addons),
                     addons.clearAddons.bind(null, app, app.manifest.addons),
-                    backups.restoreApp.bind(null, app, app.manifest.addons, restoreConfig, (progress) => {
-                        progressCallback({ percent: 65, message: `Restore - ${progress.message}` });
-                    })
+                    backups.downloadApp.bind(null, app, restoreConfig, (progress) => {
+                        progressCallback({ percent: 65, message: progress.message });
+                    }),
+                    addons.restoreAddons.bind(null, app, app.manifest.addons)
                 ], next);
             }
         },
@@ -580,12 +578,6 @@ function install(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 70, message: 'Creating container' }),
         createContainer.bind(null, app),
 
-        progressCallback.bind(null, { percent: 75, message: 'Setting up logrotate config' }),
-        addLogrotateConfig.bind(null, app),
-
-        progressCallback.bind(null, { percent: 80, message: 'Setting up collectd profile' }),
-        addCollectdProfile.bind(null, app),
-
         startApp.bind(null, app),
 
         progressCallback.bind(null, { percent: 85, message: 'Waiting for DNS propagation' }),
@@ -622,8 +614,8 @@ function backup(app, args, progressCallback, callback) {
     ], function seriesDone(error) {
         if (error) {
             debugApp(app, 'error backing up app: %s', error);
-            // return to installed state intentionally
-            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: error.toPlainObject ? error.toPlainObject() : error.message }, callback.bind(null, error));
+            // return to installed state intentionally. the error is stashed only in the task and not the app (the UI shows error state otherwise)
+            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }, callback.bind(null, makeTaskError(error, app)));
         }
         callback(null);
     });
@@ -637,7 +629,6 @@ function create(app, args, progressCallback, callback) {
 
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
 
         // FIXME: re-setup addons only because sendmail addon to re-inject env vars on mailboxName change
@@ -673,7 +664,6 @@ function changeLocation(app, args, progressCallback, callback) {
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
         unconfigureReverseProxy.bind(null, app),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
         function (next) {
             let obsoleteDomains = oldConfig.alternateDomains.filter(function (o) {
@@ -709,7 +699,7 @@ function changeLocation(app, args, progressCallback, callback) {
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error reconfiguring : %s', error);
+            debugApp(app, 'error changing location : %s', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
@@ -722,12 +712,11 @@ function migrateDataDir(app, args, progressCallback, callback) {
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    let oldDataDir = args.oldDataDir;
-    assert(oldDataDir === null || typeof oldDataDir === 'string');
+    let newDataDir = args.newDataDir;
+    assert(newDataDir === null || typeof newDataDir === 'string');
 
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
 
         progressCallback.bind(null, { percent: 45, message: 'Ensuring app data directory' }),
@@ -735,72 +724,43 @@ function migrateDataDir(app, args, progressCallback, callback) {
 
         // re-setup addons since this creates the localStorage volume
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        addons.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),
 
-        // migrate dataDir
-        function (next) {
-            const dataDirChanged = oldDataDir !== app.dataDir;
+        progressCallback.bind(null, { percent: 60, message: 'Moving data dir' }),
+        moveDataDir.bind(null, app, newDataDir),
 
-            if (!dataDirChanged) return next();
-
-            moveDataDir(app, oldDataDir, next);
-        },
-
-        progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
+        progressCallback.bind(null, { percent: 90, message: 'Creating container' }),
         createContainer.bind(null, app),
 
         startApp.bind(null, app),
 
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
-        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
+        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir })
     ], function seriesDone(error) {
         if (error) {
-            debugApp(app, 'error reconfiguring : %s', error);
+            debugApp(app, 'error migrating data dir : %s', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
         callback(null);
     });
 }
 
-// configure is called for an infra update and repair
+// configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local"
 function configure(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    const oldConfig = args.oldConfig || null;
-    const overwriteDns = args.overwriteDns;
-
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
         unconfigureReverseProxy.bind(null, app),
-        removeCollectdProfile.bind(null, app),
-        removeLogrotateConfig.bind(null, app),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
-        function (next) {
-            if (!oldConfig) return next();
-
-            let obsoleteDomains = oldConfig.alternateDomains.filter(function (o) {
-                return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
-            });
-
-            if (oldConfig.fqdn !== app.fqdn) obsoleteDomains.push({ subdomain: oldConfig.location, domain: oldConfig.domain });
-
-            if (obsoleteDomains.length === 0) return next();
-
-            unregisterSubdomains(app, obsoleteDomains, next);
-        },
-
         reserveHttpPort.bind(null, app),
 
         progressCallback.bind(null, { percent: 20, message: 'Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
-        registerSubdomains.bind(null, app, overwriteDns),
-
         progressCallback.bind(null, { percent: 40, message: 'Downloading image' }),
         downloadImage.bind(null, app.manifest),
 
@@ -814,12 +774,6 @@ function configure(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
         createContainer.bind(null, app),
 
-        progressCallback.bind(null, { percent: 65, message: 'Setting up logrotate config' }),
-        addLogrotateConfig.bind(null, app),
-
-        progressCallback.bind(null, { percent: 70, message: 'Add collectd profile' }),
-        addCollectdProfile.bind(null, app),
-
         startApp.bind(null, app),
 
         progressCallback.bind(null, { percent: 80, message: 'Waiting for DNS propagation' }),
@@ -856,7 +810,7 @@ function update(app, args, progressCallback, callback) {
     async.series([
         // this protects against the theoretical possibility of an app being marked for update from
         // a previous version of box code
-        progressCallback.bind(null, { percent: 0, message: 'Verify manifest' }),
+        progressCallback.bind(null, { percent: 5, message: 'Verify manifest' }),
         verifyManifest.bind(null, updateConfig.manifest),
 
         function (next) {
@@ -882,7 +836,6 @@ function update(app, args, progressCallback, callback) {
         // note: we cleanup first and then backup. this is done so that the app is not running should backup fail
         // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
         progressCallback.bind(null, { percent: 35, message: 'Cleaning up old install' }),
-        docker.stopContainers.bind(null, app.id),
         deleteContainers.bind(null, app, { managedOnly: true }),
         function deleteImageIfChanged(done) {
             if (app.manifest.dockerImage === updateConfig.manifest.dockerImage) return done();
@@ -952,6 +905,10 @@ function start(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 20, message: 'Starting container' }),
         docker.startContainer.bind(null, app.id),
 
+        // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
+        progressCallback.bind(null, { percent: 60, message: 'Configuring reverse proxy' }),
+        configureReverseProxy.bind(null, app),
+
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
@@ -984,6 +941,27 @@ function stop(app, args, progressCallback, callback) {
     });
 }
 
+function restart(app, args, progressCallback, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        progressCallback.bind(null, { percent: 20, message: 'Restarting container' }),
+        docker.restartContainer.bind(null, app.id),
+
+        progressCallback.bind(null, { percent: 100, message: 'Done' }),
+        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error starting app: %s', error);
+            return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
+        }
+        callback(null);
+    });
+}
+
 function uninstall(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
@@ -991,16 +969,8 @@ function uninstall(app, args, progressCallback, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     async.series([
-        progressCallback.bind(null, { percent: 0, message: 'Remove collectd profile' }),
-        removeCollectdProfile.bind(null, app),
-
-        progressCallback.bind(null, { percent: 5, message: 'Remove logrotate config' }),
-        removeLogrotateConfig.bind(null, app),
-
-        progressCallback.bind(null, { percent: 10, message: 'Stopping app' }),
-        docker.stopContainers.bind(null, app.id),
-
         progressCallback.bind(null, { percent: 20, message: 'Deleting container' }),
+        unconfigureReverseProxy.bind(null, app),
         deleteContainers.bind(null, app, {}),
 
         progressCallback.bind(null, { percent: 30, message: 'Teardown addons' }),
@@ -1018,9 +988,6 @@ function uninstall(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 70, message: 'Cleanup icon' }),
         removeIcon.bind(null, app),
 
-        progressCallback.bind(null, { percent: 80, message: 'Unconfiguring reverse proxy' }),
-        unconfigureReverseProxy.bind(null, app),
-
         progressCallback.bind(null, { percent: 90, message: 'Cleanup logs' }),
         cleanupLogs.bind(null, app),
 
@@ -1072,9 +1039,11 @@ function run(appId, args, progressCallback, callback) {
             return start(app, args, progressCallback, callback);
         case apps.ISTATE_PENDING_STOP:
             return stop(app, args, progressCallback, callback);
+        case apps.ISTATE_PENDING_RESTART:
+            return restart(app, args, progressCallback, callback);
         default:
             debugApp(app, 'apptask launched with invalid command');
-            return callback(new Error('Unknown install command in apptask:' + app.installationState));
+            return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState));
         }
     });
 }

src/apptaskmanager.js

@@ -5,6 +5,7 @@ exports = module.exports = {
 };
 
 let assert = require('assert'),
+    BoxError = require('./boxerror.js'),
     debug = require('debug')('box:apptaskmanager'),
     fs = require('fs'),
     locker = require('./locker.js'),
@@ -42,12 +43,12 @@ function scheduleTask(appId, taskId, callback) {
     if (!gInitialized) initializeSync();
 
     if (appId in gActiveTasks) {
-        return callback(new Error(`Task for %s is already active: ${appId}`));
+        return callback(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
     }
 
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 0, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
+        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
         gPendingTasks.push({ appId, taskId, callback });
         return;
     }
@@ -56,7 +57,7 @@ function scheduleTask(appId, taskId, callback) {
 
     if (lockError) {
         debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 0, message: waitText(lockError.operation) }, NOOP_CALLBACK);
+        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
         gPendingTasks.push({ appId, taskId, callback });
         return;
     }

src/auditsource.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     HEALTH_MONITOR: { userId: null, username: 'healthmonitor' },
     APP_TASK: { userId: null, username: 'apptask' },
     EXTERNAL_LDAP_TASK: { userId: null, username: 'externalldap' },
+    EXTERNAL_LDAP_AUTO_CREATE: { userId: null, username: 'externalldap' },
 
     fromRequest: fromRequest
 };

src/authcodedb.js

@@ -1,78 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-exports = module.exports = {
-    get: get,
-    add: add,
-    del: del,
-    delExpired: delExpired,
-
-    _clear: clear
-};
-
-var assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js');
-
-var AUTHCODES_FIELDS = [ 'authCode', 'userId', 'clientId', 'expiresAt' ].join(',');
-
-function get(authCode, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + AUTHCODES_FIELDS + ' FROM authcodes WHERE authCode = ? AND expiresAt > ?', [ authCode, Date.now() ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Authcode not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function add(authCode, clientId, userId, expiresAt, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof expiresAt, 'number');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('INSERT INTO authcodes (authCode, clientId, userId, expiresAt) VALUES (?, ?, ?, ?)',
-        [ authCode, clientId, userId, expiresAt ], function (error, result) {
-            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-            if (error || result.affectedRows !== 1) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            callback(null);
-        });
-}
-
-function del(authCode, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes WHERE authCode = ?', [ authCode ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Authcode not found'));
-
-        callback(null);
-    });
-}
-
-function delExpired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes WHERE expiresAt <= ?', [ Date.now() ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        return callback(null, result.affectedRows);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes', function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-

src/backups.js

@@ -2,6 +2,7 @@
 
 exports = module.exports = {
     testConfig: testConfig,
+    testProviderConfig: testProviderConfig,
 
     getByStatePaged: getByStatePaged,
     getByAppIdPaged: getByAppIdPaged,
@@ -14,7 +15,7 @@ exports = module.exports = {
     restore: restore,
 
     backupApp: backupApp,
-    restoreApp: restoreApp,
+    downloadApp: downloadApp,
 
     backupBoxAndApps: backupBoxAndApps,
 
@@ -29,6 +30,8 @@ exports = module.exports = {
 
     checkConfiguration: checkConfiguration,
 
+    configureCollectd: configureCollectd,
+
     SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8),
 
     // for testing
@@ -43,12 +46,14 @@ var addons = require('./addons.js'),
     assert = require('assert'),
     backupdb = require('./backupdb.js'),
     BoxError = require('./boxerror.js'),
+    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     crypto = require('crypto'),
     database = require('./database.js'),
     DataLayout = require('./datalayout.js'),
     debug = require('debug')('box:backups'),
     df = require('@sindresorhus/df'),
+    ejs = require('ejs'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
     locker = require('./locker.js'),
@@ -68,6 +73,7 @@ var addons = require('./addons.js'),
     zlib = require('zlib');
 
 const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');
+const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/cloudron-backup.ejs', { encoding: 'utf8' });
 
 function debugApp(app) {
     assert(typeof app === 'object');
@@ -87,6 +93,7 @@ function api(provider) {
     case 'exoscale-sos': return require('./storage/s3.js');
     case 'wasabi': return require('./storage/s3.js');
     case 'scaleway-objectstorage': return require('./storage/s3.js');
+    case 'linode-objectstorage': return require('./storage/s3.js');
     case 'noop': return require('./storage/noop.js');
     default: return null;
     }
@@ -118,6 +125,17 @@ function testConfig(backupConfig, callback) {
     api(backupConfig.provider).testConfig(backupConfig, callback);
 }
 
+
+function testProviderConfig(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var func = api(backupConfig.provider);
+    if (!func) return callback(new BoxError(BoxError.BAD_FIELD, 'unknown storage provider', { field: 'provider' }));
+
+    api(backupConfig.provider).testConfig(backupConfig, callback);
+}
+
 function getByStatePaged(state, page, perPage, callback) {
     assert.strictEqual(typeof state, 'string');
     assert(typeof page === 'number' && page > 0);
@@ -217,14 +235,14 @@ function createReadStream(sourceFile, key) {
 
     stream.on('error', function (error) {
         debug('createReadStream: read stream error.', error);
-        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, error.message));
     });
 
     if (key !== null) {
         var encrypt = crypto.createCipher('aes-256-cbc', key);
         encrypt.on('error', function (error) {
             debug('createReadStream: encrypt stream error.', error);
-            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
         });
         return stream.pipe(encrypt).pipe(ps);
     } else {
@@ -237,17 +255,25 @@ function createWriteStream(destFile, key) {
     assert(key === null || typeof key === 'string');
 
     var stream = fs.createWriteStream(destFile);
+    var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+
+    stream.on('error', function (error) {
+        debug('createWriteStream: write stream error.', error);
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, error.message));
+    });
 
     if (key !== null) {
         var decrypt = crypto.createDecipher('aes-256-cbc', key);
         decrypt.on('error', function (error) {
             debug('createWriteStream: decrypt stream error.', error);
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, error.message));
         });
-        decrypt.pipe(stream);
-        return decrypt;
+        ps.pipe(decrypt).pipe(stream);
     } else {
-        return stream;
+        ps.pipe(stream);
     }
+
+    return ps;
 }
 
 function tarPack(dataLayout, key, callback) {
@@ -338,7 +364,7 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
                     debug(`read stream error for ${task.path}: ${error.message}`);
                     retryCallback();
                 }); // ignore error if file disappears
-                stream.on('progress', function(progress) {
+                stream.on('progress', function (progress) {
                     const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
                     if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0Mbps looks wrong
                     progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}Mbps` }); // 0M@0Mbps looks wrong
@@ -437,7 +463,7 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback)
                     tarPack(dataLayout, backupConfig.key || null, function (error, tarStream) {
                         if (error) return retryCallback(error);
 
-                        tarStream.on('progress', function(progress) {
+                        tarStream.on('progress', function (progress) {
                             const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
                             if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0Mbps looks wrong
                             progressCallback({ message: `Uploading backup ${transferred}M@${speed}Mbps` });
@@ -545,24 +571,30 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
 
     debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
 
-    function downloadFile(entry, callback) {
+    function downloadFile(entry, done) {
         let relativePath = path.relative(backupFilePath, entry.fullPath);
         if (backupConfig.key) {
             relativePath = decryptFilePath(relativePath, backupConfig.key);
-            if (!relativePath) return callback(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file'));
+            if (!relativePath) return done(new BoxError(BoxError.BAD_STATE, 'Unable to decrypt file'));
         }
         const destFilePath = dataLayout.toLocalPath('./' + relativePath);
 
         mkdirp(path.dirname(destFilePath), function (error) {
-            if (error) return callback(new BoxError(BoxError.FS_ERROR, error.message));
+            if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));
 
             async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
                 let destStream = createWriteStream(destFilePath, backupConfig.key || null);
 
+                destStream.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0Mbps looks wrong
+                    progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}Mbps` });
+                });
+
                 // protect against multiple errors. must destroy the write stream so that a previous retry does not write
                 let closeAndRetry = once((error) => {
-                    if (error) progressCallback({ message: `Download ${entry.fullPath} errored: ${error.message}` });
-                    else progressCallback({ message: `Download ${entry.fullPath} finished` });
+                    if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                    else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` });
                     destStream.destroy();
                     retryCallback(error);
                 });
@@ -571,21 +603,21 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
                     if (error) return closeAndRetry(error);
 
                     sourceStream.on('error', closeAndRetry);
-                    destStream.on('error', closeAndRetry);
+                    destStream.on('error', closeAndRetry); // already emits BoxError
 
                     progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
 
                     sourceStream.pipe(destStream, { end: true }).on('finish', closeAndRetry);
                 });
-            }, callback);
+            }, done);
         });
     }
 
-    api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, done) {
+    api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
         // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
         const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
 
-        async.eachLimit(entries, concurrency, downloadFile, done);
+        async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
     }, callback);
 }
 
@@ -597,7 +629,7 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback,
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`download - Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`);
+    debug(`download: Downloading ${backupId} of format ${format} to ${dataLayout.toString()}`);
 
     const backupFilePath = getBackupFilePath(backupConfig, backupId, format);
 
@@ -651,9 +683,8 @@ function restore(backupConfig, backupId, progressCallback, callback) {
     });
 }
 
-function restoreApp(app, addonsToRestore, restoreConfig, progressCallback, callback) {
+function downloadApp(app, restoreConfig, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof addonsToRestore, 'object');
     assert.strictEqual(typeof restoreConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
@@ -662,30 +693,32 @@ function restoreApp(app, addonsToRestore, restoreConfig, progressCallback, callb
     if (!appDataDir) return callback(safe.error);
     const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
 
-    var startTime = new Date();
+    const startTime = new Date();
+    const getBackupConfigFunc = restoreConfig.backupConfig ? (next) => next(null, restoreConfig.backupConfig) : settings.getBackupConfig;
 
-    settings.getBackupConfig(function (error, backupConfig) {
+    getBackupConfigFunc(function (error, backupConfig) {
         if (error) return callback(error);
 
-        async.series([
-            download.bind(null, backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback),
-            addons.restoreAddons.bind(null, app, addonsToRestore)
-        ], function (error) {
-            debug('restoreApp: time: %s', (new Date() - startTime)/1000);
+        download(backupConfig, restoreConfig.backupId, restoreConfig.backupFormat, dataLayout, progressCallback, function (error) {
+            debug('downloadApp: time: %s', (new Date() - startTime)/1000);
 
             callback(error);
         });
     });
 }
 
-function runBackupUpload(backupId, format, dataLayout, progressCallback, callback) {
-    assert.strictEqual(typeof backupId, 'string');
-    assert.strictEqual(typeof format, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+function runBackupUpload(uploadConfig, progressCallback, callback) {
+    assert.strictEqual(typeof uploadConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    let result = '';
+    const { backupId, format, dataLayout, progressTag } = uploadConfig;
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof format, 'string');
+    assert.strictEqual(typeof progressTag, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    let result = ''; // the script communicates error result as a string
 
     shell.sudo(`backup-${backupId}`, [ BACKUP_UPLOAD_CMD, backupId, format, dataLayout.toString() ], { preserveEnv: true, ipc: true }, function (error) {
         if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
@@ -695,10 +728,10 @@ function runBackupUpload(backupId, format, dataLayout, progressCallback, callbac
         }
 
         callback();
-    }).on('message', function (message) {
-        if (!message.result) return progressCallback(message);
-        debug(`runBackupUpload: result - ${JSON.stringify(message)}`);
-        result = message.result;
+    }).on('message', function (progress) { // this is { message } or { result }
+        if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
+        debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
+        result = progress.result;
     });
 }
 
@@ -752,8 +785,14 @@ function uploadBoxSnapshot(backupConfig, progressCallback, callback) {
         const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
         if (!boxDataDir) return callback(safe.error);
 
-        const dataLayout = new DataLayout(boxDataDir, []);
-        runBackupUpload('snapshot/box', backupConfig.format, dataLayout, progressCallback, function (error) {
+        const uploadConfig = {
+            backupId: 'snapshot/box',
+            format: backupConfig.format,
+            dataLayout: new DataLayout(boxDataDir, []),
+            progressTag: 'box'
+        };
+
+        runBackupUpload(uploadConfig, progressCallback, function (error) {
             if (error) return callback(error);
 
             debug('uploadBoxSnapshot: time: %s secs', (new Date() - startTime)/1000);
@@ -782,7 +821,7 @@ function rotateBoxBackup(backupConfig, tag, appBackupIds, progressCallback, call
         if (error) return callback(error);
 
         var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, 'snapshot/box', format), getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message }));
+        copy.on('progress', (message) => progressCallback({ message: `box: ${message}` }));
         copy.on('done', function (copyBackupError) {
             const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
 
@@ -863,7 +902,7 @@ function rotateAppBackup(backupConfig, app, tag, options, progressCallback, call
         if (error) return callback(error);
 
         var copy = api(backupConfig.provider).copy(backupConfig, getBackupFilePath(backupConfig, `snapshot/app_${app.id}`, format), getBackupFilePath(backupConfig, backupId, format));
-        copy.on('progress', (message) => progressCallback({ message }));
+        copy.on('progress', (message) => progressCallback({ message: `${message} (${app.fqdn})` }));
         copy.on('done', function (copyBackupError) {
             const state = copyBackupError ? backupdb.BACKUP_STATE_ERROR : backupdb.BACKUP_STATE_NORMAL;
 
@@ -898,7 +937,14 @@ function uploadAppSnapshot(backupConfig, app, progressCallback, callback) {
 
         const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
 
-        runBackupUpload(backupId, backupConfig.format, dataLayout, progressCallback, function (error) {
+        const uploadConfig = {
+            backupId,
+            format: backupConfig.format,
+            dataLayout,
+            progressTag: app.fqdn
+        };
+
+        runBackupUpload(uploadConfig, progressCallback, function (error) {
             if (error) return callback(error);
 
             debugApp(app, 'uploadAppSnapshot: %s done time: %s secs', backupId, (new Date() - startTime)/1000);
@@ -1262,3 +1308,15 @@ function checkConfiguration(callback) {
         callback(null, message);
     });
 }
+
+function configureCollectd(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (backupConfig.provider === 'filesystem') {
+        const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { backupDir: backupConfig.backupFolder });
+        collectd.addProfile('cloudron-backup', collectdConf, callback);
+    } else {
+        collectd.removeProfile('cloudron-backup', callback);
+    }
+}

src/boxerror.js

@@ -33,6 +33,7 @@ function BoxError(reason, errorOrMessage, details) {
 }
 util.inherits(BoxError, Error);
 BoxError.ACCESS_DENIED = 'Access Denied';
+BoxError.ADDONS_ERROR = 'Addons Error';
 BoxError.ALREADY_EXISTS = 'Already Exists';
 BoxError.BAD_FIELD = 'Bad Field';
 BoxError.BAD_STATE = 'Bad State';
@@ -44,6 +45,7 @@ BoxError.DATABASE_ERROR = 'Database Error';
 BoxError.DNS_ERROR = 'DNS Error';
 BoxError.DOCKER_ERROR = 'Docker Error';
 BoxError.EXTERNAL_ERROR = 'External Error'; // use this for external API errors
+BoxError.FEATURE_DISABLED = 'Feature Disabled';
 BoxError.FS_ERROR = 'FileSystem Error';
 BoxError.INACTIVE = 'Inactive';
 BoxError.INTERNAL_ERROR = 'Internal Error';
@@ -58,9 +60,10 @@ BoxError.NOT_IMPLEMENTED = 'Not implemented';
 BoxError.NOT_SIGNED = 'Not Signed';
 BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.PLAN_LIMIT = 'Plan Limit';
+BoxError.SPAWN_ERROR = 'Spawn Error';
 BoxError.TASK_ERROR = 'Task Error';
+BoxError.TIMEOUT = 'Timeout';
 BoxError.TRY_AGAIN = 'Try Again';
-BoxError.UNKNOWN_ERROR = 'Unknown Error'; // only used for porting
 
 BoxError.prototype.toPlainObject = function () {
     return _.extend({}, { message: this.message, reason: this.reason }, this.details);
@@ -75,6 +78,8 @@ BoxError.toHttpError = function (error) {
         return new HttpError(402, error);
     case BoxError.NOT_FOUND:
         return new HttpError(404, error);
+    case BoxError.FEATURE_DISABLED:
+        return new HttpError(405, error);
     case BoxError.ALREADY_EXISTS:
     case BoxError.BAD_STATE:
     case BoxError.CONFLICT:
@@ -86,6 +91,7 @@ BoxError.toHttpError = function (error) {
     case BoxError.FS_ERROR:
     case BoxError.MAIL_ERROR:
     case BoxError.DOCKER_ERROR:
+    case BoxError.ADDONS_ERROR:
         return new HttpError(424, error);
     case BoxError.DATABASE_ERROR:
     case BoxError.INTERNAL_ERROR:

src/cert/acme2.js

@@ -9,8 +9,8 @@ var assert = require('assert'),
     fs = require('fs'),
     path = require('path'),
     paths = require('../paths.js'),
+    request = require('request'),
     safe = require('safetydance'),
-    superagent = require('superagent'),
     util = require('util'),
     _ = require('underscore');
 
@@ -41,15 +41,6 @@ function Acme2(options) {
     this.wildcard = !!options.wildcard;
 }
 
-Acme2.prototype.getNonce = function (callback) {
-    superagent.get(this.directory.newNonce).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(error);
-        if (response.statusCode !== 204) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
-
-        return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
-    });
-};
-
 // urlsafe base64 encoding (jose)
 function urlBase64Encode(string) {
     return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
@@ -96,8 +87,12 @@ Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
 
     var payload64 = b64(payload);
 
-    this.getNonce(function (error, nonce) {
-        if (error) return callback(error);
+    request.get(this.directory.newNonce, { json: true, timeout: 30000 }, function (error, response) {
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`));
+        if (response.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching nonce : ' + response.statusCode));
+
+        const nonce = response.headers['Replay-Nonce'.toLowerCase()];
+        if (!nonce) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response'));
 
         debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
 
@@ -113,14 +108,23 @@ Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
             signature: signature64
         };
 
-        superagent.post(url).set('Content-Type', 'application/jose+json').set('User-Agent', 'acme-cloudron').send(JSON.stringify(data)).timeout(30 * 1000).end(function (error, res) {
-            if (error && !error.response) return callback(error); // network errors
+        request.post(url, { headers: { 'Content-Type': 'application/jose+json', 'User-Agent': 'acme-cloudron' }, body: JSON.stringify(data), timeout: 30000 }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`)); // network error
 
-            callback(null, res);
+            // we don't set json: true in request because it ends up mangling the content-type
+            // we don't set json: true in request because it ends up mangling the content-type
+            if (response.headers['content-type'] === 'application/json') response.body = safe.JSON.parse(response.body);
+
+            callback(null, response);
         });
     });
 };
 
+// https://tools.ietf.org/html/rfc8555#section-6.3
+Acme2.prototype.postAsGet = function (url, callback) {
+    this.sendSignedRequest(url, '', callback);
+};
+
 Acme2.prototype.updateContact = function (registrationUri, callback) {
     assert.strictEqual(typeof registrationUri, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -134,7 +138,7 @@ Acme2.prototype.updateContact = function (registrationUri, callback) {
 
     const that = this;
     this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when updating contact: ${error.message}`));
+        if (error) return callback(error);
         if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 200, got %s %s', result.statusCode, result.text)));
 
         debug(`updateContact: contact of user updated to ${that.email}`);
@@ -154,7 +158,7 @@ Acme2.prototype.registerUser = function (callback) {
 
     var that = this;
     this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when registering user: ${error.message}`));
+        if (error) return callback(error);
         // 200 if already exists. 201 for new accounts
         if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register new account. Expecting 200 or 201, got %s %s', result.statusCode, result.text)));
 
@@ -180,7 +184,7 @@ Acme2.prototype.newOrder = function (domain, callback) {
     debug('newOrder: %s', domain);
 
     this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when creating new order: ${error.message}`));
+        if (error) return callback(error);
         if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending signed request: ${result.body.detail}`));
         if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
 
@@ -201,14 +205,15 @@ Acme2.prototype.waitForOrder = function (orderUrl, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     debug(`waitForOrder: ${orderUrl}`);
+    const that = this;
 
     async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
         debug('waitForOrder: getting status');
 
-        superagent.get(orderUrl).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) {
+        that.postAsGet(orderUrl, function (error, result) {
+            if (error) {
                 debug('waitForOrder: network error getting uri %s', orderUrl);
-                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for order: ${error.message}`)); // network error
+                return retryCallback(error);
             }
             if (result.statusCode !== 200) {
                 debug('waitForOrder: invalid response code getting uri %s', result.statusCode);
@@ -253,7 +258,7 @@ Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
     };
 
     this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when notifying challenge: ${error.message}`));
+        if (error) return callback(error);
         if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 200, got %s %s', result.statusCode, result.text)));
 
         callback();
@@ -265,21 +270,22 @@ Acme2.prototype.waitForChallenge = function (challenge, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     debug('waitingForChallenge: %j', challenge);
+    const that = this;
 
     async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
         debug('waitingForChallenge: getting status');
 
-        superagent.get(challenge.url).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) {
+        that.postAsGet(challenge.url, function (error, result) {
+            if (error) {
                 debug('waitForChallenge: network error getting uri %s', challenge.url);
-                return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error waiting for challenge: ${error.message}`));
+                return retryCallback(error);
             }
             if (result.statusCode !== 200) {
                 debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
                 return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
             }
 
-            debug('waitForChallenge: status is "%s %j', result.body.status, result.body);
+            debug('waitForChallenge: status is "%s" %j', result.body.status, result.body);
 
             if (result.body.status === 'pending') return retryCallback(new BoxError(BoxError.TRY_AGAIN));
             else if (result.body.status === 'valid') return retryCallback();
@@ -305,7 +311,7 @@ Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, cal
     debug('signCertificate: sending sign request');
 
     this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when signing certificate: ${error.message}`));
+        if (error) return callback(error);
         // 429 means we reached the cert limit for this domain
         if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 200, got %s %s', result.statusCode, result.text)));
 
@@ -348,20 +354,17 @@ Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var outdir = paths.APP_CERTS_DIR;
+    const that = this;
 
     async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
         debug('downloadCertificate: downloading certificate');
 
-        superagent.get(certUrl).buffer().parse(function (res, done) {
-            var data = [ ];
-            res.on('data', function(chunk) { data.push(chunk); });
-            res.on('end', function () { res.text = Buffer.concat(data); done(); });
-        }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
-            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry'));
+        that.postAsGet(certUrl, function (error, result) {
+            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
+            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
             if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
 
-            const fullChainPem = result.text;
+            const fullChainPem = result.body; // buffer
 
             const certName = hostname.replace('*.', '_.');
             var certificateFile = path.join(outdir, `${certName}.cert`);
@@ -488,8 +491,8 @@ Acme2.prototype.prepareChallenge = function (hostname, domain, authorizationUrl,
     debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
 
     const that = this;
-    superagent.get(authorizationUrl).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error when preparing challenge: ${error.message}`));
+    this.postAsGet(authorizationUrl, function (error, response) {
+        if (error) return callback(error);
         if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code getting authorization : ' + response.statusCode));
 
         const authorization = response.body;
@@ -569,13 +572,13 @@ Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
 Acme2.prototype.getDirectory = function (callback) {
     const that = this;
 
-    superagent.get(this.caDirectory).timeout(30 * 1000).end(function (error, response) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
+    request.get(this.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
         if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
 
         if (typeof response.body.newNonce !== 'string' ||
             typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') return callback(new Error(`Invalid response body : ${response.body}`));
+            typeof response.body.newAccount !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));
 
         that.directory = response.body;
 

src/cert/interface.js

@@ -10,7 +10,8 @@ exports = module.exports = {
     getCertificate: getCertificate
 };
 
-var assert = require('assert');
+var assert = require('assert'),
+    BoxError = require('../boxerror.js');
 
 function getCertificate(hostname, domain, options, callback) {
     assert.strictEqual(typeof hostname, 'string');
@@ -18,6 +19,6 @@ function getCertificate(hostname, domain, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    return callback(new Error('Not implemented'));
+    return callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'getCertificate is not implemented'));
 }
 

src/clientdb.js

@@ -1,179 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get: get,
-    getAll: getAll,
-    getAllWithTokenCount: getAllWithTokenCount,
-    getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
-    add: add,
-    del: del,
-    getByAppId: getByAppId,
-    getByAppIdAndType: getByAppIdAndType,
-
-    upsert: upsert,
-
-    delByAppId: delByAppId,
-    delByAppIdAndType: delByAppIdAndType,
-
-    _clear: clear
-};
-
-var assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js');
-
-var CLIENTS_FIELDS = [ 'id', 'appId', 'type', 'clientSecret', 'redirectURI', 'scope' ].join(',');
-var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.type', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE id = ?', [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, `Client not found: ${id}`));
-
-        callback(null, result[0]);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients ORDER BY appId', function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAllWithTokenCount(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAllWithTokenCountByIdentifier(identifier, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? LIMIT 1', [ appId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function getByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? AND type = ? LIMIT 1', [ appId, type ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function add(id, appId, type, clientSecret, redirectURI, scope, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof clientSecret, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
-
-    database.query('INSERT INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-        if (error || result.affectedRows === 0) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function upsert(id, appId, type, clientSecret, redirectURI, scope, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof clientSecret, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
-
-    database.query('REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-        if (error || result.affectedRows === 0) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients WHERE id = ?', [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, `Client not found: ${id}`));
-
-        callback(null);
-    });
-}
-
-function delByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients WHERE appId=?', [ appId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null);
-    });
-}
-
-function delByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients WHERE appId=? AND type=?', [ appId, type ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients', function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}

src/clients.js

@@ -1,329 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    add: add,
-    get: get,
-    del: del,
-    getAll: getAll,
-    getByAppIdAndType: getByAppIdAndType,
-    getTokensByUserId: getTokensByUserId,
-    delTokensByUserId: delTokensByUserId,
-    delByAppIdAndType: delByAppIdAndType,
-    addTokenByUserId: addTokenByUserId,
-    delToken: delToken,
-
-    issueDeveloperToken: issueDeveloperToken,
-
-    addDefaultClients: addDefaultClients,
-
-    removeTokenPrivateFields: removeTokenPrivateFields,
-
-    // client ids. we categorize them so we can have different restrictions based on the client
-    ID_WEBADMIN: 'cid-webadmin',    // dashboard oauth
-    ID_SDK: 'cid-sdk',              // created by user via dashboard
-    ID_CLI: 'cid-cli',              // created via cli tool
-
-    // client type enums
-    TYPE_EXTERNAL: 'external',
-    TYPE_BUILT_IN: 'built-in',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_PROXY: 'addon-proxy'
-};
-
-var apps = require('./apps.js'),
-    assert = require('assert'),
-    async = require('async'),
-    BoxError = require('./boxerror.js'),
-    clientdb = require('./clientdb.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:clients'),
-    eventlog = require('./eventlog.js'),
-    hat = require('./hat.js'),
-    accesscontrol = require('./accesscontrol.js'),
-    tokendb = require('./tokendb.js'),
-    users = require('./users.js'),
-    uuid = require('uuid'),
-    _ = require('underscore');
-
-function validateClientName(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 character', { field: 'name' });
-    if (name.length > 128) return new BoxError(BoxError.BAD_FIELD, 'name too long', { field: 'name' });
-
-    if (/[^a-zA-Z0-9-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'name can only contain alphanumerals and dash', { field: 'name' });
-
-    return null;
-}
-
-function validateTokenName(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (name.length > 64) return new BoxError(BoxError.BAD_FIELD, 'name too long', { field: 'name' });
-
-    return null;
-}
-
-function add(appId, type, redirectURI, scope, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = accesscontrol.validateScopeString(scope);
-    if (error) return callback(error);
-
-    error = validateClientName(appId);
-    if (error) return callback(error);
-
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(8 * 128);
-
-    clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
-        if (error) return callback(error);
-
-        var client = {
-            id: id,
-            appId: appId,
-            type: type,
-            clientSecret: clientSecret,
-            redirectURI: redirectURI,
-            scope: scope
-        };
-
-        callback(null, client);
-    });
-}
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.get(id, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.del(id, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.getAll(function (error, results) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, []);
-        if (error) return callback(error);
-
-        var tmp = [];
-        async.each(results, function (record, callback) {
-            if (record.type === exports.TYPE_EXTERNAL || record.type === exports.TYPE_BUILT_IN) {
-                // the appId in this case holds the name
-                record.name = record.appId;
-
-                tmp.push(record);
-
-                return callback(null);
-            }
-
-            apps.get(record.appId, function (error, result) {
-                if (error) {
-                    console.error('Failed to get app details for oauth client', record.appId, error);
-                    return callback(null);  // ignore error so we continue listing clients
-                }
-
-                if (record.type === exports.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === exports.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-
-                record.domain = result.fqdn;
-
-                tmp.push(record);
-
-                callback(null);
-            });
-        }, function (error) {
-            if (error) return callback(error);
-            callback(null, tmp);
-        });
-    });
-}
-
-function getByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.getByAppIdAndType(appId, type, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function getTokensByUserId(clientId, userId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.getByIdentifierAndClientId(userId, clientId, function (error, result) {
-        if (error && error.reason === BoxError.NOT_FOUND) {
-            // this can mean either that there are no tokens or the clientId is actually unknown
-            get(clientId, function (error/*, result*/) {
-                if (error) return callback(error);
-                callback(null, []);
-            });
-            return;
-        }
-        if (error) return callback(error);
-        callback(null, result || []);
-    });
-}
-
-function delTokensByUserId(clientId, userId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.delByIdentifierAndClientId(userId, clientId, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) {
-            // this can mean either that there are no tokens or the clientId is actually unknown
-            get(clientId, function (error/*, result*/) {
-                if (error) return callback(error);
-                callback(null);
-            });
-            return;
-        }
-        if (error) return callback(error);
-        callback(null);
-    });
-}
-
-function delByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    getByAppIdAndType(appId, type, function (error, result) {
-        if (error) return callback(error);
-
-        tokendb.delByClientId(result.id, function (error) {
-            if (error && error.reason !== BoxError.NOT_FOUND) return callback(error);
-
-            clientdb.delByAppIdAndType(appId, type, function (error) {
-                if (error) return callback(error);
-
-                callback(null);
-            });
-        });
-    });
-}
-
-function addTokenByUserId(clientId, userId, expiresAt, options, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof expiresAt, 'number');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const name = options.name || '';
-    let error = validateTokenName(name);
-    if (error) return callback(error);
-
-    get(clientId, function (error, result) {
-        if (error) return callback(error);
-
-        users.get(userId, function (error, user) {
-            if (error) return callback(error);
-
-            accesscontrol.scopesForUser(user, function (error, userScopes) {
-                if (error) return callback(error);
-
-                const scope = accesscontrol.canonicalScopeString(result.scope);
-                const authorizedScopes = accesscontrol.intersectScopes(userScopes, scope.split(','));
-
-                const token = {
-                    id: 'tid-' + uuid.v4(),
-                    accessToken: hat(8 * 32),
-                    identifier: userId,
-                    clientId: result.id,
-                    expires: expiresAt,
-                    scope: authorizedScopes.join(','),
-                    name: name
-                };
-
-                tokendb.add(token, function (error) {
-                    if (error) return callback(error);
-
-                    callback(null, {
-                        accessToken: token.accessToken,
-                        tokenScopes: authorizedScopes,
-                        identifier: userId,
-                        clientId: result.id,
-                        expires: expiresAt
-                    });
-                });
-            });
-        });
-    });
-}
-
-function issueDeveloperToken(userObject, auditSource, callback) {
-    assert.strictEqual(typeof userObject, 'object');
-    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const expiresAt = Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
-
-    addTokenByUserId(exports.ID_CLI, userObject.id, expiresAt, {}, function (error, result) {
-        if (error) return callback(error);
-
-        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: userObject.id, user: users.removePrivateFields(userObject) });
-
-        callback(null, result);
-    });
-}
-
-function delToken(clientId, tokenId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof tokenId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    get(clientId, function (error) {
-        if (error) return callback(error);
-
-        tokendb.del(tokenId, function (error) {
-            if (error) return callback(error);
-
-            callback(null);
-        });
-    });
-}
-
-function addDefaultClients(origin, callback) {
-    assert.strictEqual(typeof origin, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Adding default clients');
-
-    // The domain might have changed, therefor we have to update the record
-    // id, appId, type, clientSecret, redirectURI, scope
-    async.series([
-        clientdb.upsert.bind(null, exports.ID_WEBADMIN, 'Settings', 'built-in', 'secret-webadmin', origin, '*'),
-        clientdb.upsert.bind(null, exports.ID_SDK, 'SDK', 'built-in', 'secret-sdk', origin, '*'),
-        clientdb.upsert.bind(null, exports.ID_CLI, 'Cloudron Tool', 'built-in', 'secret-cli', origin, '*')
-    ], callback);
-}
-
-function removeTokenPrivateFields(token) {
-    return _.pick(token, 'id', 'identifier', 'clientId', 'scope', 'expires', 'name');
-}

src/cloudron.js

@@ -22,22 +22,20 @@ exports = module.exports = {
 };
 
 var apps = require('./apps.js'),
+    appstore = require('./appstore.js'),
     assert = require('assert'),
     async = require('async'),
     auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
-    clients = require('./clients.js'),
     constants = require('./constants.js'),
     cron = require('./cron.js'),
     debug = require('debug')('box:cloudron'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
-    custom = require('./custom.js'),
     fs = require('fs'),
     mail = require('./mail.js'),
     notifications = require('./notifications.js'),
-    os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
     platform = require('./platform.js'),
@@ -107,6 +105,12 @@ function runStartupTasks() {
     // configure nginx to be reachable by IP
     reverseProxy.writeDefaultConfig(NOOP_CALLBACK);
 
+    // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return console.error('Failed to read backup config.', error);
+        backups.configureCollectd(backupConfig, NOOP_CALLBACK);
+    });
+
     // always generate webadmin config since we have no versioning mechanism for the ejs
     if (settings.adminDomain()) reverseProxy.writeAdminConfig(settings.adminDomain(), NOOP_CALLBACK);
 
@@ -134,16 +138,20 @@ function getConfig(callback) {
             mailFqdn: settings.mailFqdn(),
             version: constants.VERSION,
             isDemo: settings.isDemo(),
-            memory: os.totalmem(),
             provider: settings.provider(),
             cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
-            uiSpec: custom.uiSpec()
+            footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
+            features: appstore.getFeatures()
         });
     });
 }
 
 function reboot(callback) {
-    shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
+    notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '', function (error) {
+        if (error) console.error('Failed to clear reboot notification.', error);
+
+        shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
+    });
 }
 
 function isRebootRequired(callback) {
@@ -154,20 +162,20 @@ function isRebootRequired(callback) {
 }
 
 // called from cron.js
-function runSystemChecks() {
+function runSystemChecks(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     async.parallel([
         checkBackupConfiguration,
         checkMailStatus,
         checkRebootRequired
-    ], function (error) {
-        debug('runSystemChecks: done', error);
-    });
+    ], callback);
 }
 
 function checkBackupConfiguration(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    debug('Checking backup configuration');
+    debug('checking backup configuration');
 
     backups.checkConfiguration(function (error, message) {
         if (error) return callback(error);
@@ -196,7 +204,7 @@ function checkRebootRequired(callback) {
     isRebootRequired(function (error, rebootRequired) {
         if (error) return callback(error);
 
-        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish security updates, a [reboot](/#/system) is necessary.' : '', callback);
+        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '', callback);
     });
 }
 
@@ -253,6 +261,8 @@ function prepareDashboardDomain(domain, auditSource, callback) {
 
     debug(`prepareDashboardDomain: ${domain}`);
 
+    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+
     domains.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
@@ -291,10 +301,7 @@ function setDashboardDomain(domain, auditSource, callback) {
 
             const fqdn = domains.fqdn(constants.ADMIN_LOCATION, domainObject);
 
-            async.series([
-                (done) => settings.setAdmin(domain, fqdn, done),
-                (done) => clients.addDefaultClients(settings.adminOrigin(), done)
-            ], function (error) {
+            settings.setAdmin(domain, fqdn, function (error) {
                 if (error) return callback(error);
 
                 eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain: domain, fqdn: fqdn });
@@ -313,6 +320,8 @@ function setDashboardAndMailDomain(domain, auditSource, callback) {
 
     debug(`setDashboardAndMailDomain: ${domain}`);
 
+    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+
     setDashboardDomain(domain, auditSource, function (error) {
         if (error) return callback(error);
 

src/collectd.js

@@ -0,0 +1,55 @@
+'use strict';
+
+exports = module.exports = {
+    addProfile,
+    removeProfile
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    debug = require('debug')('collectd'),
+    fs = require('fs'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js');
+
+const CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh');
+
+function addProfile(name, profile, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof profile, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const configFilePath = path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`);
+
+    // skip restarting collectd if the profile already exists with the same contents
+    const currentProfile = safe.fs.readFileSync(configFilePath, 'utf8') || '';
+    if (currentProfile === profile) return callback(null);
+
+    fs.writeFile(configFilePath, profile, function (error) {
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
+
+        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}, function (error) {
+            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
+
+            callback(null);
+        });
+    });
+
+}
+
+function removeProfile(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`), function (error) {
+        if (error && error.code !== 'ENOENT') debug('Error removing collectd profile', error);
+
+        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}, function (error) {
+            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
+
+            callback(null);
+        });
+    });
+}

src/collectd/cloudron-backup.ejs

@@ -0,0 +1,9 @@
+<Plugin python>
+  <Module du>
+    <Path>
+        Instance "cloudron-backup"
+        Dir "<%= backupDir %>"
+    </Path>
+  </Module>
+</Plugin>
+

src/constants.js

@@ -47,6 +47,10 @@ exports = module.exports = {
     CLOUDRON: CLOUDRON,
     TEST: TEST,
 
+    SUPPORT_EMAIL: 'support@cloudron.io',
+
+    FOOTER: '&copy; 2020  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
+
     VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '4.2.0-test'
 };
 

src/cron.js

@@ -12,18 +12,19 @@ var appHealthMonitor = require('./apphealthmonitor.js'),
     apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
+    async = require('async'),
     auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     cloudron = require('./cloudron.js'),
     constants = require('./constants.js'),
     CronJob = require('cron').CronJob,
     debug = require('debug')('box:cron'),
-    disks = require('./disks.js'),
     dyndns = require('./dyndns.js'),
     eventlog = require('./eventlog.js'),
     janitor = require('./janitor.js'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
+    system = require('./system.js'),
     updater = require('./updater.js'),
     updateChecker = require('./updatechecker.js');
 
@@ -59,150 +60,141 @@ var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 function startJobs(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    var randomHourMinute = Math.floor(60*Math.random());
+    const randomMinute = Math.floor(60*Math.random());
     gJobs.alive = new CronJob({
-        cronTime: '00 ' + randomHourMinute + ' * * * *', // every hour on a random minute
+        cronTime: '00 ' + randomMinute + ' * * * *', // every hour on a random minute
         onTick: appstore.sendAliveStatus,
         start: true
     });
 
+    gJobs.systemChecks = new CronJob({
+        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
+        onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.diskSpaceChecker = new CronJob({
+        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
+        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.boxUpdateCheckerJob = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkBoxUpdates(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.appUpdateChecker = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkAppUpdates(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.cleanupTokens = new CronJob({
+        cronTime: '00 */30 * * * *', // every 30 minutes
+        onTick: janitor.cleanupTokens,
+        start: true
+    });
+
+    gJobs.cleanupBackups = new CronJob({
+        cronTime: '00 45 1,3,5,23 * * *', // every 6 hours. try not to overlap with ensureBackup job
+        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.cleanupEventlog = new CronJob({
+        cronTime: '00 */30 * * * *', // every 30 minutes
+        onTick: eventlog.cleanup,
+        start: true
+    });
+
+    gJobs.dockerVolumeCleaner = new CronJob({
+        cronTime: '00 00 */12 * * *', // every 12 hours
+        onTick: janitor.cleanupDockerVolumes,
+        start: true
+    });
+
+    gJobs.schedulerSync = new CronJob({
+        cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
+        onTick: scheduler.sync,
+        start: true
+    });
+
+    gJobs.certificateRenew = new CronJob({
+        cronTime: '00 00 */12 * * *', // every 12 hours
+        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.appHealthMonitor = new CronJob({
+        cronTime: '*/10 * * * * *', // every 10 seconds
+        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
+        start: true
+    });
+
     settings.getAll(function (error, allSettings) {
         if (error) return callback(error);
 
-        recreateJobs(allSettings[settings.TIME_ZONE_KEY]);
-        appAutoupdatePatternChanged(allSettings[settings.APP_AUTOUPDATE_PATTERN_KEY]);
-        boxAutoupdatePatternChanged(allSettings[settings.BOX_AUTOUPDATE_PATTERN_KEY]);
+        const tz = allSettings[settings.TIME_ZONE_KEY];
+        backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
+        appAutoupdatePatternChanged(allSettings[settings.APP_AUTOUPDATE_PATTERN_KEY], tz);
+        boxAutoupdatePatternChanged(allSettings[settings.BOX_AUTOUPDATE_PATTERN_KEY], tz);
         dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
 
         callback();
     });
 }
 
+// eslint-disable-next-line no-unused-vars
 function handleSettingsChanged(key, value) {
     assert.strictEqual(typeof key, 'string');
-
     // value is a variant
+
     switch (key) {
-    case settings.TIME_ZONE_KEY: recreateJobs(value); break;
-    case settings.APP_AUTOUPDATE_PATTERN_KEY: appAutoupdatePatternChanged(value); break;
-    case settings.BOX_AUTOUPDATE_PATTERN_KEY: boxAutoupdatePatternChanged(value); break;
-    case settings.DYNAMIC_DNS_KEY: dynamicDnsChanged(value); break;
-    default: break;
+    case settings.TIME_ZONE_KEY:
+    case settings.BACKUP_CONFIG_KEY:
+    case settings.APP_AUTOUPDATE_PATTERN_KEY:
+    case settings.BOX_AUTOUPDATE_PATTERN_KEY:
+    case settings.DYNAMIC_DNS_KEY:
+        debug('handleSettingsChanged: recreating all jobs');
+        async.series([
+            stopJobs,
+            startJobs
+        ], NOOP_CALLBACK);
+        break;
+    default:
+        break;
     }
 }
 
-function recreateJobs(tz) {
+function backupConfigChanged(value, tz) {
+    assert.strictEqual(typeof value, 'object');
     assert.strictEqual(typeof tz, 'string');
 
-    debug('Creating jobs with timezone %s', tz);
+    debug(`backupConfigChanged: interval ${value.intervalSecs} (${tz})`);
 
     if (gJobs.backup) gJobs.backup.stop();
+    let pattern;
+    if (value.intervalSecs <= 6 * 60 * 60) {
+        pattern = '00 00 1,7,13,19 * * *'; // no option but to backup in the middle of the day
+    } else {
+        pattern = '00 00 1,3,5,23 * * *'; // avoid middle of the day backups
+    }
+
     gJobs.backup = new CronJob({
-        cronTime: '00 00 */6 * * *', // check every 6 hours
+        cronTime: pattern,
         onTick: backups.ensureBackup.bind(null, auditSource.CRON, NOOP_CALLBACK),
         start: true,
         timeZone: tz
     });
-
-    if (gJobs.systemChecks) gJobs.systemChecks.stop();
-    gJobs.systemChecks = new CronJob({
-        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: cloudron.runSystemChecks,
-        start: true,
-        runOnInit: true, // run system check immediately
-        timeZone: tz
-    });
-
-    if (gJobs.diskSpaceChecker) gJobs.diskSpaceChecker.stop();
-    gJobs.diskSpaceChecker = new CronJob({
-        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: () => disks.checkDiskSpace(NOOP_CALLBACK),
-        start: true,
-        runOnInit: true, // run system check immediately
-        timeZone: tz
-    });
-
-    // randomized pattern per cloudron every hour
-    var randomMinute = Math.floor(60*Math.random());
-
-    if (gJobs.boxUpdateCheckerJob) gJobs.boxUpdateCheckerJob.stop();
-    gJobs.boxUpdateCheckerJob = new CronJob({
-        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
-        onTick: () => updateChecker.checkBoxUpdates(NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.appUpdateChecker) gJobs.appUpdateChecker.stop();
-    gJobs.appUpdateChecker = new CronJob({
-        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
-        onTick: () => updateChecker.checkAppUpdates(NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.cleanupTokens) gJobs.cleanupTokens.stop();
-    gJobs.cleanupTokens = new CronJob({
-        cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: janitor.cleanupTokens,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.cleanupBackups) gJobs.cleanupBackups.stop();
-    gJobs.cleanupBackups = new CronJob({
-        cronTime: '00 45 */6 * * *', // every 6 hours. try not to overlap with ensureBackup job
-        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.cleanupEventlog) gJobs.cleanupEventlog.stop();
-    gJobs.cleanupEventlog = new CronJob({
-        cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: eventlog.cleanup,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.dockerVolumeCleaner) gJobs.dockerVolumeCleaner.stop();
-    gJobs.dockerVolumeCleaner = new CronJob({
-        cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: janitor.cleanupDockerVolumes,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.schedulerSync) gJobs.schedulerSync.stop();
-    gJobs.schedulerSync = new CronJob({
-        cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
-        onTick: scheduler.sync,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.certificateRenew) gJobs.certificateRenew.stop();
-    gJobs.certificateRenew = new CronJob({
-        cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.appHealthMonitor) gJobs.appHealthMonitor.stop();
-    gJobs.appHealthMonitor = new CronJob({
-        cronTime: '*/10 * * * * *', // every 10 seconds
-        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
 }
 
-function boxAutoupdatePatternChanged(pattern) {
+function boxAutoupdatePatternChanged(pattern, tz) {
     assert.strictEqual(typeof pattern, 'string');
-    assert(gJobs.boxUpdateCheckerJob);
+    assert.strictEqual(typeof tz, 'string');
 
-    debug('Box auto update pattern changed to %s', pattern);
+    debug(`boxAutoupdatePatternChanged: pattern - ${pattern} (${tz})`);
 
     if (gJobs.boxAutoUpdater) gJobs.boxAutoUpdater.stop();
 
@@ -220,15 +212,15 @@ function boxAutoupdatePatternChanged(pattern) {
             }
         },
         start: true,
-        timeZone: gJobs.boxUpdateCheckerJob.cronTime.zone // hack
+        timeZone: tz
     });
 }
 
-function appAutoupdatePatternChanged(pattern) {
+function appAutoupdatePatternChanged(pattern, tz) {
     assert.strictEqual(typeof pattern, 'string');
-    assert(gJobs.boxUpdateCheckerJob);
+    assert.strictEqual(typeof tz, 'string');
 
-    debug('Apps auto update pattern changed to %s', pattern);
+    debug(`appAutoupdatePatternChanged: pattern ${pattern} (${tz})`);
 
     if (gJobs.appAutoUpdater) gJobs.appAutoUpdater.stop();
 
@@ -246,13 +238,12 @@ function appAutoupdatePatternChanged(pattern) {
             }
         },
         start: true,
-        timeZone: gJobs.boxUpdateCheckerJob.cronTime.zone // hack
+        timeZone: tz
     });
 }
 
 function dynamicDnsChanged(enabled) {
     assert.strictEqual(typeof enabled, 'boolean');
-    assert(gJobs.boxUpdateCheckerJob);
 
     debug('Dynamic DNS setting changed to %s', enabled);
 
@@ -260,8 +251,7 @@ function dynamicDnsChanged(enabled) {
         gJobs.dynamicDns = new CronJob({
             cronTime: '5 * * * * *',    // we only update the records if the ip has changed.
             onTick: dyndns.sync.bind(null, auditSource.CRON, NOOP_CALLBACK),
-            start: true,
-            timeZone: gJobs.boxUpdateCheckerJob.cronTime.zone // hack
+            start: true
         });
     } else {
         if (gJobs.dynamicDns) gJobs.dynamicDns.stop();

src/custom.js

@@ -1,66 +0,0 @@
-'use strict';
-
-let debug = require('debug')('box:custom'),
-    lodash = require('lodash'),
-    paths = require('./paths.js'),
-    safe = require('safetydance'),
-    yaml = require('js-yaml');
-
-exports = module.exports = {
-    uiSpec: uiSpec,
-    spec: spec
-};
-
-const DEFAULT_SPEC = {
-    appstore: {
-        blacklist: [],
-        whitelist: null // null imples, not set. this is an object and not an array
-    },
-    backups: {
-        configurable: true
-    },
-    domains: {
-        dynamicDns: true,
-        changeDashboardDomain: true
-    },
-    subscription: {
-        configurable: true
-    },
-    support: {
-        email: 'support@cloudron.io',
-        remoteSupport: true,
-        ticketFormBody:
-            'Use this form to open support tickets. You can also write directly to [support@cloudron.io](mailto:support@cloudron.io).\n\n'
-            + '* [Knowledge Base & App Docs](https://cloudron.io/documentation/apps/?support_view)\n'
-            + '* [Custom App Packaging & API](https://cloudron.io/developer/packaging/?support_view)\n'
-            + '* [Forum](https://forum.cloudron.io/)\n\n',
-        submitTickets: true
-    },
-    alerts: {
-        email: '',
-        notifyCloudronAdmins: true
-    },
-    footer: {
-        body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
-    }
-};
-
-const gSpec = (function () {
-    try {
-        if (!safe.fs.existsSync(paths.CUSTOM_FILE)) return DEFAULT_SPEC;
-        const c = yaml.safeLoad(safe.fs.readFileSync(paths.CUSTOM_FILE, 'utf8'));
-        return lodash.merge({}, DEFAULT_SPEC, c);
-    } catch (e) {
-        debug(`Error loading features file from ${paths.CUSTOM_FILE} : ${e.message}`);
-        return DEFAULT_SPEC;
-    }
-})();
-
-// flags sent to the UI. this is separate because we have values that are secret to the backend
-function uiSpec() {
-    return gSpec;
-}
-
-function spec() {
-    return gSpec;
-}

src/database.js

@@ -14,6 +14,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
+    BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
     constants = require('./constants.js'),
     mysql = require('mysql'),
@@ -103,16 +104,13 @@ function clear(callback) {
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name,
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name);
 
-    async.series([
-        child_process.exec.bind(null, cmd),
-        require('./clients.js').addDefaultClients.bind(null, 'https://admin-localhost')
-    ], callback);
+    child_process.exec(cmd, callback);
 }
 
 function beginTransaction(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (gConnectionPool === null) return callback(new Error('No database connection pool.'));
+    if (gConnectionPool === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No database connection pool.'));
 
     gConnectionPool.getConnection(function (error, connection) {
         if (error) {
@@ -156,7 +154,7 @@ function query() {
     var callback = args[args.length - 1];
     assert.strictEqual(typeof callback, 'function');
 
-    if (gDefaultConnection === null) return callback(new Error('No connection to database'));
+    if (gDefaultConnection === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No connection to database'));
 
     args[args.length -1 ] = function (error, result) {
         if (error && error.fatal) {
@@ -203,7 +201,11 @@ function exportToFile(file, callback) {
     assert.strictEqual(typeof file, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
+    // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
+    // this option must not be set in production cloudrons which still use the old mysqldump
+    const disableColStats = (constants.TEST && process.env.DESKTOP_SESSION !== 'ubuntu') ? '--column-statistics=0' : '';
+
+    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${disableColStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
 
     child_process.exec(cmd, callback);
 }

src/dns/cloudflare.js

@@ -48,15 +48,29 @@ function translateRequestError(result, callback) {
     callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
 }
 
+function createRequest(method, url, dnsConfig) {
+    assert.strictEqual(typeof method, 'string');
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof dnsConfig, 'object');
+
+    let request = superagent(method, url)
+        .timeout(30 * 1000);
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        request.set('X-Auth-Key', dnsConfig.token).set('X-Auth-Email', dnsConfig.email);
+    } else {
+        request.set('Authorization', 'Bearer ' + dnsConfig.token);
+    }
+
+    return request;
+}
+
 function getZoneByName(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active')
-        .set('X-Auth-Key', dnsConfig.token)
-        .set('X-Auth-Email', dnsConfig.email)
-        .timeout(30 * 1000)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active', dnsConfig)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -74,11 +88,8 @@ function getDnsRecords(dnsConfig, zoneId, fqdn, type, callback) {
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-        .set('X-Auth-Key',dnsConfig.token)
-        .set('X-Auth-Email',dnsConfig.email)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
         .query({ type: type, name: fqdn })
-        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -132,11 +143,8 @@ function upsert(domainObject, location, type, values, callback) {
                 if (i >= dnsRecords.length) { // create a new record
                     debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: false`);
 
-                    superagent.post(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('POST', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             if (error && !error.response) return iteratorCallback(error);
                             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, iteratorCallback);
@@ -148,11 +156,8 @@ function upsert(domainObject, location, type, values, callback) {
 
                     debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
 
-                    superagent.put(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id)
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('PUT', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id, dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             ++i; // increment, as we have consumed the record
 
@@ -217,10 +222,7 @@ function del(domainObject, location, type, values, callback) {
             if (tmp.length === 0) return callback(null);
 
             async.eachSeries(tmp, function (record, callback) {
-                superagent.del(CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id)
-                    .set('X-Auth-Key', dnsConfig.token)
-                    .set('X-Auth-Email', dnsConfig.email)
-                    .timeout(30 * 1000)
+                createRequest('DELETE', CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id, dnsConfig)
                     .end(function (error, result) {
                         if (error && !error.response) return callback(error);
                         if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -277,14 +279,20 @@ function verifyDnsConfig(domainObject, callback) {
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
+    // token can be api token or global api key
     if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
-    if (!dnsConfig.email || typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    if (dnsConfig.tokenType !== 'GlobalApiKey' && dnsConfig.tokenType !== 'ApiToken')  return callback(new BoxError(BoxError.BAD_FIELD, 'tokenType is required', { field: 'tokenType' }));
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        if ('email' in dnsConfig && typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    }
 
     const ip = '127.0.0.1';
 
     var credentials = {
         token: dnsConfig.token,
-        email: dnsConfig.email
+        tokenType: dnsConfig.tokenType,
+        email: dnsConfig.email || null
     };
 
     if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here

src/dns/godaddy.js

@@ -126,7 +126,7 @@ function del(domainObject, location, type, values, callback) {
 
     debug(`get: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, new Error('Record deletion is not supported by GoDaddy API')));
+    if (type !== 'A' && type !== 'TXT') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Record deletion is not supported by GoDaddy API'));
 
     // check if the record exists at all so that we don't insert the "Dead" record for no reason
     get(domainObject, location, type, function (error, values) {

src/dns/interface.js

@@ -17,6 +17,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
     util = require('util');
 
 function removePrivateFields(domainObject) {
@@ -24,6 +25,7 @@ function removePrivateFields(domainObject) {
     return domainObject;
 }
 
+// eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
     // in-place injection of tokens and api keys which came in with domains.SECRET_PLACEHOLDER
 }
@@ -37,7 +39,7 @@ function upsert(domainObject, location, type, values, callback) {
 
     // Result: none
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upsert is not implemented'));
 }
 
 function get(domainObject, location, type, callback) {
@@ -48,7 +50,7 @@ function get(domainObject, location, type, callback) {
 
     // Result: Array of matching DNS records in string format
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'get is not implemented'));
 }
 
 function del(domainObject, location, type, values, callback) {
@@ -60,7 +62,7 @@ function del(domainObject, location, type, values, callback) {
 
     // Result: none
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'del is not implemented'));
 }
 
 function wait(domainObject, location, type, value, options, callback) {
@@ -80,5 +82,5 @@ function verifyDnsConfig(domainObject, callback) {
 
     // Result: dnsConfig object
 
-    callback(new Error('not implemented'));
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'verifyDnsConfig is not implemented'));
 }

src/docker.js

@@ -1,8 +1,6 @@
 'use strict';
 
 exports = module.exports = {
-    connection: connectionInstance(),
-
     testRegistryConfig: testRegistryConfig,
     setRegistryConfig: setRegistryConfig,
     injectPrivateFields: injectPrivateFields,
@@ -16,6 +14,7 @@ exports = module.exports = {
     downloadImage: downloadImage,
     createContainer: createContainer,
     startContainer: startContainer,
+    restartContainer: restartContainer,
     stopContainer: stopContainer,
     stopContainerByName: stopContainer,
     stopContainers: stopContainers,
@@ -27,6 +26,7 @@ exports = module.exports = {
     getContainerIdByIp: getContainerIdByIp,
     inspect: inspect,
     inspectByName: inspect,
+    execContainer: execContainer,
     getEvents: getEvents,
     memoryUsage: memoryUsage,
     createVolume: createVolume,
@@ -34,20 +34,14 @@ exports = module.exports = {
     clearVolume: clearVolume
 };
 
-// timeout is optional
-function connectionInstance(timeout) {
-    var Docker = require('dockerode');
-    var docker = new Docker({ socketPath: '/var/run/docker.sock', timeout: timeout });
-    return docker;
-}
-
 var addons = require('./addons.js'),
     async = require('async'),
     assert = require('assert'),
     BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
     constants = require('./constants.js'),
-    debug = require('debug')('box:docker.js'),
+    debug = require('debug')('box:docker'),
+    Docker = require('dockerode'),
     path = require('path'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
@@ -58,6 +52,9 @@ var addons = require('./addons.js'),
 const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
     MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');
 
+const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
+const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
+
 function debugApp(app) {
     assert(typeof app === 'object');
 
@@ -68,8 +65,7 @@ function testRegistryConfig(auth, callback) {
     assert.strictEqual(typeof auth, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-    docker.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
+    gConnection.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
         if (error) return callback(new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' }));
 
         callback();
@@ -108,13 +104,14 @@ function ping(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     // do not let the request linger
-    var docker = connectionInstance(1000);
+    const connection = new Docker({ socketPath: DOCKER_SOCKET_PATH, timeout: 1000 });
 
-    docker.ping(function (error, result) {
+    connection.ping(function (error, result) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
-        if (result !== 'OK') return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon'));
+        if (Buffer.isBuffer(result) && result.toString('utf8') === 'OK') return callback(null); // sometimes it returns buffer
+        if (result === 'OK') return callback(null);
 
-        callback(null);
+        callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon'));
     });
 }
 
@@ -140,15 +137,14 @@ function getRegistryConfig(image, callback) {
 }
 
 function pullImage(manifest, callback) {
-    var docker = exports.connection;
-
     getRegistryConfig(manifest.dockerImage, function (error, authConfig) {
         if (error) return callback(error);
 
         debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${authConfig ? 'yes' : 'no'}`);
 
-        docker.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
-            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to pull image. Please check the network or if the image needs authentication. statusCode: ' + error.statusCode));
+        gConnection.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
+            if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${manifest.dockerImage}. message: ${error.message} statusCode: ${error.statusCode}`));
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`));
 
             // https://github.com/dotcloud/docker/issues/1074 says each status message
             // is emitted as a chunk
@@ -185,14 +181,10 @@ function downloadImage(manifest, callback) {
 
     var attempt = 1;
 
-    async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
+    async.retry({ times: 10, interval: 5000, errorFilter: e => e.reason !== BoxError.NOT_FOUND }, function (retryCallback) {
         debug('Downloading image %s. attempt: %s', manifest.dockerImage, attempt++);
 
-        pullImage(manifest, function (error) {
-            if (error) console.error(error);
-
-            retryCallback(error);
-        });
+        pullImage(manifest, retryCallback);
     }, callback);
 }
 
@@ -203,8 +195,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection,
-        isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers
+    let isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers
 
     var manifest = app.manifest;
     var exposedPorts = {}, dockerPortBindings = { };
@@ -303,7 +294,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                     'Name': isAppContainer ? 'unless-stopped' : 'no',
                     'MaximumRetryCount': 0
                 },
-                CpuShares: 512, // relative to 1024 for system processes
+                CpuShares: app.cpuShares,
                 VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ],
                 NetworkMode: 'cloudron', // user defined bridge network
                 Dns: ['172.18.0.1'], // use internal dns
@@ -330,7 +321,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
 
         debugApp(app, 'Creating container for %s', app.manifest.dockerImage);
 
-        docker.createContainer(containerOptions, function (error, container) {
+        gConnection.createContainer(containerOptions, function (error, container) {
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
             callback(null, container);
@@ -346,15 +337,29 @@ function startContainer(containerId, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
     debug('Starting container %s', containerId);
 
     container.start(function (error) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
         if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
-        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); // 304 means already started
+
+        return callback(null);
+    });
+}
+
+function restartContainer(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = gConnection.getContainer(containerId);
+    debug('Restarting container %s', containerId);
+
+    container.restart(function (error) {
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+        if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
+        if (error && error.statusCode !== 204) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         return callback(null);
     });
@@ -369,8 +374,7 @@ function stopContainer(containerId, callback) {
         return callback();
     }
 
-    var docker = exports.connection;
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
     debug('Stopping container %s', containerId);
 
     var options = {
@@ -400,8 +404,7 @@ function deleteContainer(containerId, callback) {
 
     if (containerId === null) return callback(null);
 
-    var docker = exports.connection;
-    var container = docker.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
 
     var removeOptions = {
         force: true, // kill container if it's running
@@ -425,14 +428,12 @@ function deleteContainers(appId, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
     debug('deleting containers of %s', appId);
 
     let labels = [ 'appId=' + appId ];
     if (options.managedOnly) labels.push('isCloudronManaged=true');
 
-    docker.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) {
+    gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         async.eachSeries(containers, function (container, iteratorDone) {
@@ -445,11 +446,9 @@ function stopContainers(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
+    debug('Stopping containers of %s', appId);
 
-    debug('stopping containers of %s', appId);
-
-    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+    gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         async.eachSeries(containers, function (container, iteratorDone) {
@@ -465,8 +464,6 @@ function deleteImage(manifest, callback) {
     var dockerImage = manifest ? manifest.dockerImage : null;
     if (!dockerImage) return callback(null);
 
-    var docker = exports.connection;
-
     var removeOptions = {
         force: false, // might be shared with another instance of this app
         noprune: false // delete untagged parents
@@ -475,7 +472,7 @@ function deleteImage(manifest, callback) {
     // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
     // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
     // not created anymore after https://github.com/docker/docker/pull/10571
-    docker.getImage(dockerImage).remove(removeOptions, function (error) {
+    gConnection.getImage(dockerImage).remove(removeOptions, function (error) {
         if (error && error.statusCode === 400) return callback(null); // invalid image format. this can happen if user installed with a bad --docker-image
         if (error && error.statusCode === 404) return callback(null); // not found
         if (error && error.statusCode === 409) return callback(null); // another container using the image
@@ -493,10 +490,8 @@ function getContainerIdByIp(ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var docker = exports.connection;
-
-    docker.getNetwork('cloudron').inspect(function (error, bridge) {
-        if (error && error.statusCode === 404) return callback(new Error('Unable to find the cloudron network'));
+    gConnection.getNetwork('cloudron').inspect(function (error, bridge) {
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to find the cloudron network'));
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         var containerId;
@@ -506,7 +501,7 @@ function getContainerIdByIp(ip, callback) {
                 break;
             }
         }
-        if (!containerId) return callback(new Error('No container with that ip'));
+        if (!containerId) return callback(new BoxError(BoxError.DOCKER_ERROR, 'No container with that ip'));
 
         callback(null, containerId);
     });
@@ -516,7 +511,7 @@ function inspect(containerId, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = exports.connection.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
 
     container.inspect(function (error, result) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -526,13 +521,38 @@ function inspect(containerId, callback) {
     });
 }
 
+function execContainer(containerId, options, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = gConnection.getContainer(containerId);
+
+    container.exec(options.execOptions, function (error, exec) {
+        if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running
+        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+
+        exec.start(options.startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+
+            if (options.rows && options.columns) {
+                // there is a race where resizing too early results in a 404 "no such exec"
+                // https://git.cloudron.io/cloudron/box/issues/549
+                setTimeout(function () {
+                    exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
+                }, 2000);
+            }
+
+            callback(null, stream);
+        });
+    });
+}
+
 function getEvents(options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
-    docker.getEvents(options, function (error, stream) {
+    gConnection.getEvents(options, function (error, stream) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         callback(null, stream);
@@ -543,7 +563,7 @@ function memoryUsage(containerId, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = exports.connection.getContainer(containerId);
+    var container = gConnection.getContainer(containerId);
 
     container.stats({ stream: false }, function (error, result) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -559,8 +579,6 @@ function createVolume(app, name, volumeDataDir, callback) {
     assert.strictEqual(typeof volumeDataDir, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
     const volumeOptions = {
         Name: name,
         Driver: 'local',
@@ -577,9 +595,9 @@ function createVolume(app, name, volumeDataDir, callback) {
 
     // requires sudo because the path can be outside appsdata
     shell.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}, function (error) {
-        if (error) return callback(new Error(`Error creating app data dir: ${error.message}`));
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`));
 
-        docker.createVolume(volumeOptions, function (error) {
+        gConnection.createVolume(volumeOptions, function (error) {
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
             callback();
@@ -593,8 +611,7 @@ function clearVolume(app, name, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-    let volume = docker.getVolume(name);
+    let volume = gConnection.getVolume(name);
     volume.inspect(function (error, v) {
         if (error && error.statusCode === 404) return callback();
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
@@ -614,9 +631,7 @@ function removeVolume(app, name, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
-    let volume = docker.getVolume(name);
+    let volume = gConnection.getVolume(name);
     volume.remove(function (error) {
         if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume of ${app.id} ${error.message}`));
 
@@ -627,9 +642,7 @@ function removeVolume(app, name, callback) {
 function info(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    let docker = exports.connection;
-
-    docker.info(function (error, result) {
+    gConnection.info(function (error, result) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker'));
 
         callback(null, result);

src/domaindb.js

@@ -16,7 +16,7 @@ var assert = require('assert'),
     database = require('./database.js'),
     safe = require('safetydance');
 
-var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'locked' ].join(',');
+var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson' ].join(',');
 
 function postProcess(data) {
     data.config = safe.JSON.parse(data.configJson);
@@ -24,8 +24,6 @@ function postProcess(data) {
     delete data.configJson;
     delete data.tlsConfigJson;
 
-    data.locked = !!data.locked; // make it bool
-
     return data;
 }
 
@@ -103,7 +101,7 @@ function del(domain, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     database.query('DELETE FROM domains WHERE domain=?', [ domain ], function (error, result) {
-        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') return callback(new BoxError(BoxError.CONFLICT));
+        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') return callback(new BoxError(BoxError.CONFLICT, error.message));
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
         if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
 

src/domains.js

@@ -86,9 +86,9 @@ function verifyDnsConfig(dnsConfig, domain, zoneName, provider, callback) {
 
     const domainObject = { config: dnsConfig, domain: domain, zoneName: zoneName };
     api(provider).verifyDnsConfig(domainObject, function (error, result) {
-        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, 'Incorrect configuration. Access denied'));
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, 'Zone not found'));
-        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, 'Configuration error: ' + error.message));
+        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`));
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`));
+        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`));
         if (error) return callback(error);
 
         result.hyphenatedSubdomains = !!dnsConfig.hyphenatedSubdomains;
@@ -215,14 +215,14 @@ function get(domain, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     domaindb.get(domain, function (error, result) {
-        // TODO try to find subdomain entries maybe based on zoneNames or so
         if (error) return callback(error);
 
         reverseProxy.getFallbackCertificate(domain, function (_, bundle) { // never returns an error
             var cert = safe.fs.readFileSync(bundle.certFilePath, 'utf-8');
             var key = safe.fs.readFileSync(bundle.keyFilePath, 'utf-8');
 
-            if (!cert || !key) return callback(new BoxError(BoxError.FS_ERROR, 'unable to read certificates from disk'));
+            // do not error here. otherwise, there is no way to fix things up from the UI
+            if (!cert || !key) debug(`Unable to read fallback certificates of ${domain} from disk`);
 
             result.fallbackCertificate = { cert: cert, key: key };
 
@@ -253,6 +253,8 @@ function update(domain, data, auditSource, callback) {
 
     let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
 
+    if (settings.isDemo() && (domain === settings.adminDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+
     domaindb.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
@@ -307,6 +309,13 @@ function del(domain, auditSource, callback) {
     if (domain === settings.adminDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain'));
 
     domaindb.del(domain, function (error) {
+        if (error && error.reason === BoxError.CONFLICT) {
+            if (error.message.indexOf('apps_mailDomain_constraint') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use as the mailbox of an app. Check the Email section of each app.'));
+            if (error.message.indexOf('subdomains') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by an app. Move the app first to a different location.'));
+            if (error.message.indexOf('mail') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by a mailbox. Delete mailboxes first in the Email view.'));
+
+            // intentional fall through
+        }
         if (error) return callback(error);
 
         eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
@@ -440,13 +449,13 @@ function waitForDnsRecord(location, domain, type, value, options, callback) {
 
 // removes all fields that are strictly private and should never be returned by API calls
 function removePrivateFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate', 'locked');
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate');
     return api(result.provider).removePrivateFields(result);
 }
 
 // removes all fields that are not accessible by a normal user
 function removeRestrictedFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'locked');
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider');
 
     // always ensure config object
     result.config = { hyphenatedSubdomains: !!domain.config.hyphenatedSubdomains };

src/eventlog.js

@@ -40,8 +40,10 @@ exports = module.exports = {
     ACTION_MAIL_DISABLED: 'mail.disabled',
     ACTION_MAIL_MAILBOX_ADD: 'mail.box.add',
     ACTION_MAIL_MAILBOX_REMOVE: 'mail.box.remove',
+    ACTION_MAIL_MAILBOX_UPDATE: 'mail.box.update',
     ACTION_MAIL_LIST_ADD: 'mail.list.add',
     ACTION_MAIL_LIST_REMOVE: 'mail.list.remove',
+    ACTION_MAIL_LIST_UPDATE: 'mail.list.update',
 
     ACTION_PROVISION: 'cloudron.provision',
     ACTION_RESTORE: 'cloudron.restore', // unused
@@ -57,6 +59,9 @@ exports = module.exports = {
 
     ACTION_DYNDNS_UPDATE: 'dyndns.update',
 
+    ACTION_SUPPORT_TICKET: 'support.ticket',
+    ACTION_SUPPORT_SSH: 'support.ssh',
+
     ACTION_PROCESS_CRASH: 'system.crash'
 };
 
@@ -82,11 +87,9 @@ function add(action, source, data, callback) {
     api(uuid.v4(), action, source, data, function (error, id) {
         if (error) return callback(error);
 
-        notifications.onEvent(id, action, source, data, function (error) {
-            if (error) return callback(error);
+        callback(null, { id: id });
 
-            callback(null, { id: id });
-        });
+        notifications.onEvent(id, action, source, data, NOOP_CALLBACK);
     });
 }
 

src/externalldap.js

@@ -1,7 +1,9 @@
 'use strict';
 
 exports = module.exports = {
+    search: search,
     verifyPassword: verifyPassword,
+    createAndVerifyUserIfNotExist: createAndVerifyUserIfNotExist,
 
     testConfig: testConfig,
     startSyncer: startSyncer,
@@ -33,6 +35,25 @@ function removePrivateFields(ldapConfig) {
     return ldapConfig;
 }
 
+function translateUser(ldapConfig, ldapUser) {
+    assert.strictEqual(typeof ldapConfig, 'object');
+
+    return {
+        username: ldapUser[ldapConfig.usernameField],
+        email: ldapUser.mail,
+        displayName: ldapUser.cn // user.giveName + ' ' + user.sn
+    };
+}
+
+function validUserRequirements(user) {
+    if (!user.username || !user.email || !user.displayName) {
+        debug(`[LDAP user empty username/email/displayName] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+        return false;
+    } else {
+        return true;
+    }
+}
+
 // performs service bind if required
 function getClient(externalLdapConfig, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
@@ -60,6 +81,8 @@ function getClient(externalLdapConfig, callback) {
     });
 }
 
+
+// TODO support search by email
 function ldapSearch(externalLdapConfig, options, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof options, 'object');
@@ -136,6 +159,58 @@ function testConfig(config, callback) {
     });
 }
 
+function search(identifier, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
+        if (error) return callback(error);
+        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+
+        ldapSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
+            if (error) return callback(error);
+
+            // translate ldap properties to ours
+            let users = ldapUsers.map(function (u) { return translateUser(externalLdapConfig, u); });
+
+            callback(null, users);
+        });
+    });
+}
+
+function createAndVerifyUserIfNotExist(identifier, password, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getExternalLdapConfig(function (error, externalLdapConfig) {
+        if (error) return callback(error);
+        if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
+        if (!externalLdapConfig.autoCreate) return callback(new BoxError(BoxError.BAD_STATE, 'auto create not enabled'));
+
+        ldapSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
+            if (error) return callback(error);
+            if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
+            if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
+
+            let user = translateUser(externalLdapConfig, ldapUsers[0]);
+            if (!validUserRequirements(user)) return callback(new BoxError(BoxError.BAD_FIELD));
+
+            users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_AUTO_CREATE, function (error, user) {
+                if (error) {
+                    console.error('Failed to auto create user', user.username, error);
+                    return callback(new BoxError(BoxError.INTERNAL_ERROR));
+                }
+
+                verifyPassword(user, password, function (error) {
+                    if (error) return callback(error);
+                    callback(null, user);
+                });
+            });
+        });
+    });
+}
+
 function verifyPassword(user, password, callback) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof password, 'string');
@@ -150,14 +225,12 @@ function verifyPassword(user, password, callback) {
             if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
             if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
 
-            const userDn = ldapUsers[0].dn;
             let client = ldap.createClient({ url: externalLdapConfig.url });
-
-            client.bind(userDn, password, function (error) {
+            client.bind(ldapUsers[0].dn, password, function (error) {
                 if (error instanceof ldap.InvalidCredentialsError) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
                 if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
-                callback();
+                callback(null, translateUser(externalLdapConfig, ldapUsers[0]));
             });
         });
     });
@@ -201,46 +274,41 @@ function sync(progressCallback, callback) {
 
             // we ignore all errors here and just log them for now
             async.eachSeries(ldapUsers, function (user, iteratorCallback) {
-                const username = user[externalLdapConfig.usernameField];
-                const email = user.mail;
-                const displayName = user.cn; // user.giveName + ' ' + user.sn
+                user = translateUser(externalLdapConfig, user);
 
-                if (!username || !email || !displayName) {
-                    debug(`[empty username/email/displayName] username=${username} email=${email} displayName=${displayName} usernameField=${externalLdapConfig.usernameField}`);
-                    return iteratorCallback();
-                }
+                if (!validUserRequirements(user)) return iteratorCallback();
 
                 percent += step;
-                progressCallback({ percent, message: `Syncing... ${username}` });
+                progressCallback({ percent, message: `Syncing... ${user.username}` });
 
-                users.getByUsername(username, function (error, result) {
+                users.getByUsername(user.username, function (error, result) {
                     if (error && error.reason !== BoxError.NOT_FOUND) {
-                        debug(`Could not find user with username ${username}: ${error.message}`);
+                        debug(`Could not find user with username ${user.username}: ${error.message}`);
                         return iteratorCallback();
                     }
 
                     if (error) {
-                        debug(`[adding user] username=${username} email=${email} displayName=${displayName}`);
+                        debug(`[adding user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
 
-                        users.create(username, null /* password */, email, displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
+                        users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
                             if (error) console.error('Failed to create user', user, error);
                             iteratorCallback();
                         });
                     } else if (result.source !== 'ldap') {
-                        debug(`[conflicting user] username=${username} email=${email} displayName=${displayName}`);
+                        debug(`[conflicting user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
 
                         iteratorCallback();
-                    } else if (result.email !== email || result.displayName !== displayName) {
-                        debug(`[updating user] username=${username} email=${email} displayName=${displayName}`);
+                    } else if (result.email !== user.email || result.displayName !== user.displayName) {
+                        debug(`[updating user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
 
-                        users.update(result.id, { email: email, fallbackEmail: email, displayName: displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
+                        users.update(result, { email: user.email, fallbackEmail: user.email, displayName: user.displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
                             if (error) debug('Failed to update user', user, error);
 
                             iteratorCallback();
                         });
                     } else {
                         // user known and up-to-date
-                        debug(`[up-to-date user] username=${username} email=${email} displayName=${displayName}`);
+                        debug(`[up-to-date user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
 
                         iteratorCallback();
                     }

src/infra_version.js

@@ -6,7 +6,7 @@
 
 exports = module.exports = {
     // a version change recreates all containers with latest docker config
-    'version': '48.16.0',
+    'version': '48.17.0',
 
     'baseImages': [
         { repo: 'cloudron/base', tag: 'cloudron/base:1.0.0@sha256:147a648a068a2e746644746bbfb42eb7a50d682437cead3c67c933c546357617' }
@@ -15,11 +15,11 @@ exports = module.exports = {
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.0.2@sha256:a28320f313785816be60e3f865e09065504170a3d20ed37de675c719b32b01eb' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.1.0@sha256:eee0dfd3829d563f2063084bc0d7c8802c4bdd6e233159c6226a17ff7a9a3503' },
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:2.0.2@sha256:6dcee0731dfb9b013ed94d56205eee219040ee806c7e251db3b3886eaa4947ff' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:2.1.0@sha256:6d1bf221cfe6124957e2c58b57c0a47214353496009296acb16adf56df1da9d5' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.1.0@sha256:f2cda21bd15c21bbf44432df412525369ef831a2d53860b5c5b1675e6f384de2' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.5.0@sha256:086ae1c9433d90a820326aa43914a2afe94ad707074ef2bc05a7ef4798e83655' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.6.0@sha256:c881d513ddbdea73f29d92d392a7435039314b13e5e3659e9e85f6b26476e365' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.2.0@sha256:fc9ca69d16e6ebdbd98ed53143d4a0d2212eef60cb638dc71219234e6f427a2c' },
         'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:0.1.0@sha256:e177c5bf5f38c84ce1dea35649c22a1b05f96eec67a54a812c5a35e585670f0f' }
     }

src/janitor.js

@@ -2,9 +2,9 @@
 
 var assert = require('assert'),
     async = require('async'),
-    authcodedb = require('./authcodedb.js'),
+    BoxError = require('./boxerror.js'),
     debug = require('debug')('box:janitor'),
-    docker = require('./docker.js').connection,
+    Docker = require('dockerode'),
     tokendb = require('./tokendb.js');
 
 exports = module.exports = {
@@ -12,7 +12,9 @@ exports = module.exports = {
     cleanupDockerVolumes: cleanupDockerVolumes
 };
 
-var NOOP_CALLBACK = function () { };
+const NOOP_CALLBACK = function () { };
+
+const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });
 
 function ignoreError(func) {
     return function (callback) {
@@ -36,26 +38,13 @@ function cleanupExpiredTokens(callback) {
     });
 }
 
-function cleanupExpiredAuthCodes(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    authcodedb.delExpired(function (error, result) {
-        if (error) return callback(error);
-
-        debug('Cleaned up %s expired authcodes.', result);
-
-        callback(null);
-    });
-}
-
 function cleanupTokens(callback) {
     assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
 
     debug('Cleaning up expired tokens');
 
     async.series([
-        ignoreError(cleanupExpiredTokens),
-        ignoreError(cleanupExpiredAuthCodes)
+        ignoreError(cleanupExpiredTokens)
     ], callback);
 }
 
@@ -67,16 +56,16 @@ function cleanupTmpVolume(containerInfo, callback) {
 
     debug('cleanupTmpVolume %j', containerInfo.Names);
 
-    docker.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
-        if (error) return callback(new Error('Failed to exec container : ' + error.message));
+    gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
+        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`));
 
         execContainer.start({ hijack: true }, function (error, stream) {
-            if (error) return callback(new Error('Failed to start exec container : ' + error.message));
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Failed to start exec container: ${error.message}`));
 
             stream.on('error', callback);
             stream.on('end', callback);
 
-            docker.modem.demuxStream(stream, process.stdout, process.stderr);
+            gConnection.modem.demuxStream(stream, process.stdout, process.stderr);
         });
     });
 }
@@ -88,7 +77,7 @@ function cleanupDockerVolumes(callback) {
 
     debug('Cleaning up docker volumes');
 
-    docker.listContainers({ all: 0 }, function (error, containers) {
+    gConnection.listContainers({ all: 0 }, function (error, containers) {
         if (error) return callback(error);
 
         async.eachSeries(containers, function (container, iteratorDone) {

src/ldap.js

@@ -126,16 +126,16 @@ function userSearch(req, res, next) {
         var results = [];
 
         // send user objects
-        result.forEach(function (entry) {
+        result.forEach(function (user) {
             // skip entries with empty username. Some apps like owncloud can't deal with this
-            if (!entry.username) return;
+            if (!user.username) return;
 
-            var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+            var dn = ldap.parseDN('cn=' + user.id + ',ou=users,dc=cloudron');
 
             var groups = [ GROUP_USERS_DN ];
-            if (entry.admin) groups.push(GROUP_ADMINS_DN);
+            if (users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) groups.push(GROUP_ADMINS_DN);
 
-            var displayName = entry.displayName || entry.username || ''; // displayName can be empty and username can be null
+            var displayName = user.displayName || user.username || ''; // displayName can be empty and username can be null
             var nameParts = displayName.split(' ');
             var firstName = nameParts[0];
             var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
@@ -143,17 +143,18 @@ function userSearch(req, res, next) {
             var obj = {
                 dn: dn.toString(),
                 attributes: {
-                    objectclass: ['user'],
+                    objectclass: ['user', 'inetorgperson', 'person' ],
                     objectcategory: 'person',
-                    cn: entry.id,
-                    uid: entry.id,
-                    mail: entry.email,
-                    mailAlternateAddress: entry.fallbackEmail,
+                    cn: user.id,
+                    uid: user.id,
+                    entryuuid: user.id, // to support OpenLDAP clients
+                    mail: user.email,
+                    mailAlternateAddress: user.fallbackEmail,
                     displayname: displayName,
                     givenName: firstName,
-                    username: entry.username,
-                    samaccountname: entry.username,      // to support ActiveDirectory clients
-                    isadmin: entry.admin,
+                    username: user.username,
+                    samaccountname: user.username,      // to support ActiveDirectory clients
+                    isadmin: users.compareRoles(user.role, users.ROLE_ADMIN) >= 0,
                     memberof: groups
                 }
             };
@@ -193,7 +194,7 @@ function groupSearch(req, res, next) {
 
         groups.forEach(function (group) {
             var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-            var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+            var members = group.admin ? result.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : result;
 
             var obj = {
                 dn: dn.toString(),
@@ -241,8 +242,8 @@ function groupAdminsCompare(req, res, next) {
 
         // we only support memberuid here, if we add new group attributes later add them here
         if (req.attribute === 'memberuid') {
-            var found = result.find(function (u) { return u.id === req.value; });
-            if (found && found.admin) return res.end(true);
+            var user = result.find(function (u) { return u.id === req.value; });
+            if (user && users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) return res.end(true);
         }
 
         res.end(false);
@@ -283,10 +284,11 @@ function mailboxSearch(req, res, next) {
                 res.end();
             }
         });
-    } else if (req.dn.rdns[0].attrs.domain) {
+    } else if (req.dn.rdns[0].attrs.domain) { // legacy ldap mailbox search for old sogo
         var domain = req.dn.rdns[0].attrs.domain.value.toLowerCase();
 
         mailboxdb.listMailboxes(domain, 1, 1000, function (error, result) {
+
             if (error) return next(new ldap.OperationsError(error.toString()));
 
             var results = [];
@@ -309,7 +311,6 @@ function mailboxSearch(req, res, next) {
                 // ensure all filter values are also lowercase
                 var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
                 if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
                 if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
                     results.push(obj);
                 }
@@ -317,8 +318,55 @@ function mailboxSearch(req, res, next) {
 
             finalSend(results, req, res, next);
         });
-    } else {
-        return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    } else { // new sogo
+        mailboxdb.listAllMailboxes(1, 1000, function (error, mailboxes) {
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            var results = [];
+
+            // send mailbox objects
+            async.eachSeries(mailboxes, function (mailbox, callback) {
+                var dn = ldap.parseDN(`cn=${mailbox.name}@${mailbox.domain},ou=mailboxes,dc=cloudron`);
+
+                users.get(mailbox.ownerId, function (error, userObject) {
+                    if (error) return callback(); // skip mailboxes with unknown owner
+
+                    var obj = {
+                        dn: dn.toString(),
+                        attributes: {
+                            objectclass: ['mailbox'],
+                            objectcategory: 'mailbox',
+                            displayname: userObject.displayName,
+                            cn: `${mailbox.name}@${mailbox.domain}`,
+                            uid: `${mailbox.name}@${mailbox.domain}`,
+                            mail: `${mailbox.name}@${mailbox.domain}`
+                        }
+                    };
+
+                    mailboxdb.getAliasesForName(mailbox.name, mailbox.domain, function (error, aliases) {
+                        if (error) return callback(error);
+
+                        aliases.forEach(function (a, idx) {
+                            obj.attributes['mail' + idx] = `${a}@${mailbox.domain}`;
+                        });
+
+                        // ensure all filter values are also lowercase
+                        var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+                        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+                        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                            results.push(obj);
+                        }
+
+                        callback();
+                    });
+                });
+            }, function (error) {
+                if (error) return next(new ldap.OperationsError(error.toString()));
+
+                finalSend(results, req, res, next);
+            });
+        });
     }
 }
 
@@ -419,7 +467,7 @@ function authenticateUser(req, res, next) {
         api = users.verifyWithUsername;
     }
 
-    api(commonName, req.credentials || '', function (error, user) {
+    api(commonName, req.credentials || '', req.app.id, function (error, user) {
         if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.message));
@@ -465,7 +513,7 @@ function authenticateUserMailbox(req, res, next) {
             if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
             if (error) return next(new ldap.OperationsError(error.message));
 
-            users.verify(mailbox.ownerId, req.credentials || '', function (error, result) {
+            users.verify(mailbox.ownerId, req.credentials || '', users.AP_MAIL, function (error, result) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
@@ -487,7 +535,7 @@ function authenticateSftp(req, res, next) {
     if (parts.length !== 2) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
     // actual user bind
-    users.verifyWithUsername(parts[0], req.credentials, function (error) {
+    users.verifyWithUsername(parts[0], req.credentials, users.AP_SFTP, function (error) {
         if (error) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
 
         debug('sftp auth: success');
@@ -576,7 +624,7 @@ function authenticateMailAddon(req, res, next) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
 
-                users.verify(mailbox.ownerId, req.credentials || '', function (error, result) {
+                users.verify(mailbox.ownerId, req.credentials || '', users.AP_MAIL, function (error, result) {
                     if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                     if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                     if (error) return next(new ldap.OperationsError(error.message));

src/locker.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
     debug = require('debug')('box:locker'),
     EventEmitter = require('events').EventEmitter,
     util = require('util');
@@ -23,7 +24,7 @@ Locker.prototype.lock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
     if (this._operation !== null) {
-        let error = new Error(`Locked for ${this._operation}`);
+        let error = new BoxError(BoxError.CONFLICT, `Locked for ${this._operation}`);
         error.operation = this._operation;
         return error;
     }
@@ -54,7 +55,7 @@ Locker.prototype.recursiveLock = function (operation) {
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
-    if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
+    if (this._operation !== operation) throw BoxError(BoxError.BAD_STATE, 'Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
     if (--this._lockDepth === 0) {
         debug('Released : %s', this._operation);

src/mail.js

@@ -123,13 +123,13 @@ function checkOutboundPort25(callback) {
         relay.status = false;
         relay.value = `Connect to ${smtpServer} timed out. Check if port 25 (outbound) is blocked`;
         client.destroy();
-        callback(new Error('Timeout'), relay);
+        callback(new BoxError(BoxError.TIMEOUT, `Connect to ${smtpServer} timed out.`), relay);
     });
     client.on('error', function (error) {
         relay.status = false;
         relay.value = `Connect to ${smtpServer} failed: ${error.message}. Check if port 25 (outbound) is blocked`;
         client.destroy();
-        callback(error, relay);
+        callback(new BoxError(BoxError.NETWORK_ERROR, `Connect to ${smtpServer} failed.`), relay);
     });
 }
 
@@ -199,7 +199,7 @@ function checkDkim(mailDomain, callback) {
     };
 
     var dkimKey = readDkimPublicKeySync(domain);
-    if (!dkimKey) return callback(new Error('Failed to read dkim public key'), dkim);
+    if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, `Failed to read dkim public key of ${domain}`), dkim);
 
     dkim.expected = 'v=DKIM1; t=s; p=' + dkimKey;
 
@@ -456,7 +456,7 @@ function getStatus(domain, callback) {
 
     // ensure we always have a valid toplevel properties for the api
     var results = {
-        dns: {}, // { mx/dmar/dkim/spf/ptr: { expected, value, name, domain, type } }
+        dns: {}, // { mx/dmarc/dkim/spf/ptr: { expected, value, name, domain, type, status } }
         rbl: {}, // { status, ip, servers: [{name,site,dns}]} optional. only for cloudron-smtp
         relay: {} // { status, value } always checked
     };
@@ -466,7 +466,7 @@ function getStatus(domain, callback) {
             func(function (error, result) {
                 if (error) debug('Ignored error - ' + what + ':', error);
 
-                safe.set(results, what, result);
+                safe.set(results, what, result || {});
 
                 callback();
             });
@@ -567,12 +567,12 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
         // mail_domain is used for SRS
         if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
             `mail_in_domains=${mailInDomains}\nmail_out_domains=${mailOutDomains}\nmail_server_name=${mailFqdn}\nmail_domain=${mailDomain}\n\n`, 'utf8')) {
-            return callback(new Error('Could not create mail var file:' + safe.error.message));
+            return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
         }
 
         // enable_outbound makes plugin forward email for relayed mail. non-relayed mail always hits LMTP plugin first
         if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/smtp_forward.ini'), 'enable_outbound=false\ndomain_selector=mail_from\n', 'utf8')) {
-            return callback(new Error('Could not create smtp forward file:' + safe.error.message));
+            return callback(new BoxError(BoxError.FS_ERROR, 'Could not create smtp forward file:' + safe.error.message));
         }
 
         // create sections for per-domain configuration
@@ -582,7 +582,7 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
 
             if (!safe.fs.appendFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
                 `[${domain.domain}]\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\n\n`, 'utf8')) {
-                return callback(new Error('Could not create mail var file:' + safe.error.message));
+                return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
             }
 
             const relay = domain.relay;
@@ -598,7 +598,7 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
 
             if (!safe.fs.appendFileSync(paths.ADDON_CONFIG_DIR + '/mail/smtp_forward.ini',
                 `[${domain.domain}]\nenable_outbound=true\nhost=${host}\nport=${port}\nenable_tls=true\nauth_type=${authType}\nauth_user=${username}\nauth_pass=${password}\n\n`, 'utf8')) {
-                return callback(new Error('Could not create mail var file:' + safe.error.message));
+                return callback(new BoxError(BoxError.FS_ERROR, 'Could not create mail var file:' + safe.error.message));
             }
         });
 
@@ -627,8 +627,8 @@ function configureMail(mailFqdn, mailDomain, callback) {
         const mailCertFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_cert.pem');
         const mailKeyFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_key.pem');
 
-        if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new Error('Could not create cert file:' + safe.error.message));
-        if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new Error('Could not create key file:' + safe.error.message));
+        if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create cert file:' + safe.error.message));
+        if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create key file:' + safe.error.message));
 
         shell.exec('startMail', 'docker rm -f mail || true', function (error) {
             if (error) return callback(error);
@@ -654,7 +654,6 @@ function configureMail(mailFqdn, mailDomain, callback) {
                             -v "${paths.MAIL_DATA_DIR}:/app/data" \
                             -v "${paths.PLATFORM_DATA_DIR}/addons/mail:/etc/mail" \
                             ${ports} \
-                            -p 127.0.0.1:2020:2020 \
                             --label isCloudronManaged=true \
                             --read-only -v /run -v /tmp ${tag}`;
 
@@ -846,7 +845,7 @@ function upsertDnsRecords(domain, mailFqdn, callback) {
         if (process.env.BOX_ENV === 'test') return callback();
 
         var dkimKey = readDkimPublicKeySync(domain);
-        if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, new Error('Failed to read dkim public key')));
+        if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, 'Failed to read dkim public key'));
 
         // t=s limits the domainkey to this domain and not it's subdomains
         var dkimRecord = { subdomain: `${mailDomain.dkimSelector}._domainkey`, domain: domain, type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
@@ -1106,18 +1105,25 @@ function addMailbox(name, domain, userId, auditSource, callback) {
     });
 }
 
-function updateMailboxOwner(name, domain, userId, callback) {
+function updateMailboxOwner(name, domain, userId, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     name = name.toLowerCase();
 
-    mailboxdb.updateMailboxOwner(name, domain, userId, function (error) {
+    getMailbox(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        callback(null);
+        mailboxdb.updateMailboxOwner(name, domain, userId, function (error) {
+            if (error) return callback(error);
+
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: result.userId, userId });
+
+            callback(null);
+        });
     });
 }
 
@@ -1196,12 +1202,12 @@ function getLists(domain, callback) {
     });
 }
 
-function getList(domain, listName, callback) {
+function getList(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof listName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    mailboxdb.getList(listName, domain, function (error, result) {
+    mailboxdb.getList(name, domain, function (error, result) {
         if (error) return callback(error);
 
         callback(null, result);
@@ -1227,16 +1233,17 @@ function addList(name, domain, members, auditSource, callback) {
     mailboxdb.addList(name, domain, members, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members });
 
         callback();
     });
 }
 
-function updateList(name, domain, members, callback) {
+function updateList(name, domain, members, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert(Array.isArray(members));
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     name = name.toLowerCase();
@@ -1248,10 +1255,16 @@ function updateList(name, domain, members, callback) {
         if (!validator.isEmail(members[i])) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid email: ' + members[i]));
     }
 
-    mailboxdb.updateList(name, domain, members, function (error) {
+    getList(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        callback(null);
+        mailboxdb.updateList(name, domain, members, function (error) {
+            if (error) return callback(error);
+
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members });
+
+            callback(null);
+        });
     });
 }
 
@@ -1264,7 +1277,7 @@ function removeList(name, domain, auditSource, callback) {
     mailboxdb.del(name, domain, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
 
         callback();
     });
@@ -1315,4 +1328,4 @@ function resolveList(listName, listDomain, callback) {
             });
         });
     });
-}
+}

src/mail_templates/welcome_user.ejs

@@ -5,7 +5,7 @@ Dear <%= user.displayName || user.username || user.email %>,
 Welcome to <%= cloudronName %>!
 
 Follow the link to get started.
-<%- setupLink %>
+<%- inviteLink %>
 
 <% if (invitor && invitor.email) { %>
 You are receiving this email because you were invited by <%= invitor.email %>.
@@ -25,7 +25,7 @@ Powered by https://cloudron.io
 <h2>Welcome to <%= cloudronName %>!</h2>
 
 <p>
-    <a href="<%= setupLink %>">Get started</a>.
+    <a href="<%= inviteLink %>">Get started</a>.
 </p>
 
 <br/>

src/mailboxdb.js

@@ -12,6 +12,8 @@ exports = module.exports = {
     listMailboxes: listMailboxes,
     getLists: getLists,
 
+    listAllMailboxes: listAllMailboxes,
+
     get: get,
     getMailbox: getMailbox,
     getList: getList,
@@ -214,6 +216,21 @@ function listMailboxes(domain, page, perPage, callback) {
         });
 }
 
+function listAllMailboxes(page, perPage, callback) {
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? ORDER BY name LIMIT ${(page-1)*perPage},${perPage}`,
+        [ exports.TYPE_MAILBOX ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
 function getLists(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');

src/mailer.js

@@ -3,7 +3,7 @@
 exports = module.exports = {
     userAdded: userAdded,
     userRemoved: userRemoved,
-    adminChanged: adminChanged,
+    roleChanged: roleChanged,
     passwordReset: passwordReset,
     appUpdatesAvailable: appUpdatesAvailable,
 
@@ -26,7 +26,6 @@ exports = module.exports = {
 
 var assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    custom = require('./custom.js'),
     debug = require('debug')('box:mailer'),
     ejs = require('ejs'),
     mail = require('./mail.js'),
@@ -47,15 +46,16 @@ function getMailConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     settings.getCloudronName(function (error, cloudronName) {
-        // this is not fatal
-        if (error) {
-            debug(error);
-            cloudronName = 'Cloudron';
-        }
+        if (error) debug('Error getting cloudron name: ', error);
 
-        callback(null, {
-            cloudronName: cloudronName,
-            notificationFrom: `"${cloudronName}" <no-reply@${settings.adminDomain()}>`
+        settings.getSupportConfig(function (error, supportConfig) {
+            if (error) debug('Error getting support config: ', error);
+
+            callback(null, {
+                cloudronName: cloudronName || '',
+                notificationFrom: `"${cloudronName}" <no-reply@${settings.adminDomain()}>`,
+                supportEmail: supportConfig.email
+            });
         });
     });
 }
@@ -125,9 +125,10 @@ function mailUserEvent(mailTo, user, event) {
     });
 }
 
-function sendInvite(user, invitor) {
+function sendInvite(user, invitor, inviteLink) {
     assert.strictEqual(typeof user, 'object');
-    assert(typeof invitor === 'object');
+    assert.strictEqual(typeof invitor, 'object');
+    assert.strictEqual(typeof inviteLink, 'string');
 
     debug('Sending invite mail');
 
@@ -137,7 +138,7 @@ function sendInvite(user, invitor) {
         var templateData = {
             user: user,
             webadminUrl: settings.adminOrigin(),
-            setupLink: `${settings.adminOrigin()}/api/v1/session/account/setup.html?reset_token=${user.resetToken}&email=${encodeURIComponent(user.email)}`,
+            inviteLink: inviteLink,
             invitor: invitor,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
@@ -203,14 +204,13 @@ function userRemoved(mailTo, user) {
     mailUserEvent(mailTo, user, 'was removed');
 }
 
-function adminChanged(mailTo, user, isAdmin) {
+function roleChanged(mailTo, user) {
     assert.strictEqual(typeof mailTo, 'string');
     assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof isAdmin, 'boolean');
 
-    debug('Sending mail for adminChanged');
+    debug('Sending mail for roleChanged');
 
-    mailUserEvent(mailTo, user, isAdmin ? 'is now an admin' : 'is no more an admin');
+    mailUserEvent(mailTo, user, `now has the role '${user.role}`);
 }
 
 function passwordReset(user) {
@@ -223,7 +223,7 @@ function passwordReset(user) {
 
         var templateData = {
             user: user,
-            resetLink: `${settings.adminOrigin()}/api/v1/session/password/reset.html?reset_token=${user.resetToken}&email=${encodeURIComponent(user.email)}`,
+            resetLink: `${settings.adminOrigin()}/login.html?resetToken=${user.resetToken}`,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
         };
@@ -279,7 +279,7 @@ function appDied(mailTo, app) {
             from: mailConfig.notificationFrom,
             to: mailTo,
             subject: util.format('[%s] App %s is down', mailConfig.cloudronName, app.fqdn),
-            text: render('app_down.ejs', { title: app.manifest.title, appFqdn: app.fqdn, supportEmail: custom.spec().support.email, format: 'text' })
+            text: render('app_down.ejs', { title: app.manifest.title, appFqdn: app.fqdn, supportEmail: mailConfig.supportEmail, format: 'text' })
         };
 
         sendMail(mailOptions);

src/middleware/index.js

@@ -1,15 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    cookieParser: require('cookie-parser'),
     cors: require('./cors'),
-    csrf: require('csurf'),
     json: require('body-parser').json,
     morgan: require('morgan'),
     proxy: require('proxy-middleware'),
     lastMile: require('connect-lastmile'),
     multipart: require('./multipart.js'),
-    session: require('express-session'),
     timeout: require('connect-timeout'),
     urlencoded: require('body-parser').urlencoded
 };

src/nginxconfig.ejs

@@ -108,7 +108,9 @@ server {
 <% } -%>
 
     proxy_http_version 1.1;
+    # intercept errors (>= 400) and use the error_page handler
     proxy_intercept_errors on;
+    # nginx will return 504 on connect/timeout errors
     proxy_read_timeout       3500;
     proxy_connect_timeout    3250;
 
@@ -125,7 +127,15 @@ server {
 
     # only serve up the status page if we get proxy gateway errors
     root <%= sourceDir %>/dashboard/dist;
-    error_page 502 503 504 /appstatus.html;
+    # some apps use 503 to indicate updating or maintenance
+    error_page 502 504 /app_error_page;
+    location /app_error_page {
+        root /home/yellowtent/boxdata;
+        # the first argument looks for file under the root
+        try_files /custom_pages/$request_uri /custom_pages/app_not_responding.html /appstatus.html;
+        # internal means this is for internal routing and cannot be accessed as URL from browser
+        internal;
+    }
     location /appstatus.html {
         internal;
     }

src/notifications.js

@@ -25,7 +25,6 @@ let assert = require('assert'),
     auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     changelog = require('./changelog.js'),
-    custom = require('./custom.js'),
     debug = require('debug')('box:notifications'),
     eventlog = require('./eventlog.js'),
     mailer = require('./mailer.js'),
@@ -100,7 +99,8 @@ function actionForAllAdmins(skippingUserIds, iterator, callback) {
     assert.strictEqual(typeof iterator, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    users.getAllAdmins(function (error, result) {
+    users.getAdmins(function (error, result) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback();
         if (error) return callback(error);
 
         // filter out users we want to skip (like the user who did the action or the user the action was performed on)
@@ -134,14 +134,14 @@ function userRemoved(performedBy, eventId, user, callback) {
     }, callback);
 }
 
-function adminChanged(performedBy, eventId, user, callback) {
+function roleChanged(performedBy, eventId, user, callback) {
     assert.strictEqual(typeof performedBy, 'string');
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     actionForAllAdmins([ performedBy, user.id ], function (admin, done) {
-        mailer.adminChanged(admin.email, user, user.admin);
-        add(admin.id, eventId, `User '${user.displayName} ' ${user.admin ? 'is now an admin' : 'is no more an admin'}`, `User '${user.username || user.email || user.fallbackEmail}' ${user.admin ? 'is now an admin' : 'is no more an admin'}.`, done);
+        mailer.roleChanged(admin.email, user);
+        add(admin.id, eventId, `User '${user.displayName}'s role changed`, `User '${user.username || user.email || user.fallbackEmail}' now has the role '${user.role}'.`, done);
     }, callback);
 }
 
@@ -167,9 +167,6 @@ function oomEvent(eventId, app, addon, containerId, event, callback) {
         message = 'The container has been restarted automatically. Consider increasing the [memory limit](https://docs.docker.com/v17.09/edge/engine/reference/commandline/update/#update-a-containers-kernel-memory-constraints)';
     }
 
-    if (custom.spec().alerts.email) mailer.oomEvent(custom.spec().alerts.email, program, event);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, done) {
         mailer.oomEvent(admin.email, program, event);
 
@@ -182,9 +179,6 @@ function appUp(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.appUp(custom.spec().alerts.email, app);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, done) {
         mailer.appUp(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is back online`, `The application ${app.manifest.title} installed at ${app.fqdn} is back online.`, done);
@@ -196,9 +190,6 @@ function appDied(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.appDied(custom.spec().alerts.email, app);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, callback) {
         mailer.appDied(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is down`, `The application ${app.manifest.title} installed at ${app.fqdn} is not responding.`, callback);
@@ -246,9 +237,6 @@ function boxUpdateError(eventId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.boxUpdateError(custom.spec().alerts.email, errorMessage);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, done) {
         mailer.boxUpdateError(admin.email, errorMessage);
         add(admin.id, eventId, 'Cloudron update failed', `Failed to update Cloudron: ${errorMessage}. Update will be retried in 4 hours`, done);
@@ -261,9 +249,6 @@ function certificateRenewalError(eventId, vhost, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.certificateRenewalError(custom.spec().alerts.email, vhost, errorMessage);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, callback) {
         mailer.certificateRenewalError(admin.email, vhost, errorMessage);
         add(admin.id, eventId, `Certificate renewal of ${vhost} failed`, `Failed to new certs of ${vhost}: ${errorMessage}. Renewal will be retried in 12 hours`, callback);
@@ -276,9 +261,6 @@ function backupFailed(eventId, taskId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.backupFailed(custom.spec().alerts.email, errorMessage, `${settings.adminOrigin()}/logs.html?taskId=${taskId}`);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, callback) {
         mailer.backupFailed(admin.email, errorMessage, `${settings.adminOrigin()}/logs.html?taskId=${taskId}`);
         add(admin.id, eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}). Will be retried in 4 hours`, callback);
@@ -343,8 +325,8 @@ function onEvent(id, action, source, data, callback) {
         return userRemoved(source.userId, id, data.user, callback);
 
     case eventlog.ACTION_USER_UPDATE:
-        if (!data.adminStatusChanged) return callback();
-        return adminChanged(source.userId, id, data.user, callback);
+        if (!data.roleChanged) return callback();
+        return roleChanged(source.userId, id, data.user, callback);
 
     case eventlog.ACTION_APP_OOM:
         return oomEvent(id, data.app, data.addon, data.containerId, data.event, callback);

src/oauth2views/account_setup.ejs

@@ -1,82 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<script>
-
-'use strict';
-
-// very basic angular app
-var app = angular.module('Application', []);
-app.controller('Controller', ['$scope', function ($scope) {
-    $scope.username = '<%= (user && user.username) ? user.username : '' %>';
-    $scope.displayName = '<%= (user && user.displayName) ? user.displayName : '' %>';
-}]);
-
-</script>
-
-<div class="layout-content">
-
-  <center>
-    <br/>
-    <h4>Hello <%= (user && user.email) ? user.email : '' %>, welcome to <%= cloudronName %>!</h4>
-    <h2>Setup your account and password</h2>
-  </center>
-
-  <div class="container" ng-app="Application" ng-controller="Controller">
-    <div class="row">
-      <div class="col-md-6 col-md-offset-3">
-        <form action="/api/v1/session/account/setup" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
-          <input type="password" style="display: none;">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <input type="hidden" name="email" value="<%= email %>"/>
-          <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
-
-          <center><p class="has-error"><%= error %></p></center>
-
-<% if (user && user.username) { %>
-          <div class="form-group"">
-            <label class="control-label">Username</label>
-            <input type="text" class="form-control" ng-model="username" name="username" readonly required>
-          </div>
-<% } else { %>
-          <div class="form-group" ng-class="{ 'has-error': (setupForm.username.$dirty && setupForm.username.$invalid) }">
-            <label class="control-label">Username</label>
-            <div class="control-label" ng-show="setupForm.username.$dirty && setupForm.username.$invalid">
-              <small ng-show="setupForm.username.$error.minlength">The username is too short</small>
-              <small ng-show="setupForm.username.$error.maxlength">The username is too long</small>
-              <small ng-show="setupForm.username.$dirty && setupForm.username.$invalid">Not a valid username</small>
-            </div>
-            <input type="text" class="form-control" ng-model="username" name="username" required autofocus>
-          </div>
-<% } %>
-
-          <div class="form-group">
-            <label class="control-label">Full Name</label>
-            <input type="displayName" class="form-control" ng-model="displayName" name="displayName" required>
-          </div>
-
-          <div class="form-group" ng-class="{ 'has-error': (setupForm.password.$dirty && setupForm.password.$invalid) }">
-            <label class="control-label">New Password</label>
-            <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
-              <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be atleast 8 characters</small>
-            </div>
-            <input type="password" class="form-control" ng-model="password" name="password" ng-pattern="/^.{8,}$/" required>
-          </div>
-
-          <div class="form-group" ng-class="{ 'has-error': (setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)) }">
-            <label class="control-label">Repeat Password</label>
-            <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
-              <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
-            </div>
-            <input type="password" class="form-control" ng-model="passwordRepeat" name="passwordRepeat" required>
-          </div>
-
-          <center><input class="btn btn-primary btn-outline" type="submit" value="Setup" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/></center>
-        </form>
-      </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/callback.ejs

@@ -1,45 +0,0 @@
-<!-- callback tester -->
-
-<script>
-
-'use strict';
-
-var code = null;
-var redirectURI = null;
-var accessToken = null;
-var tokenType = null;
-var state = null;
-
-var args = window.location.search.slice(1).split('&');
-args.forEach(function (arg) {
-    var tmp = arg.split('=');
-    if (tmp[0] === 'redirectURI') {
-        redirectURI = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'code') {
-        code = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'state') {
-        state = window.decodeURIComponent(tmp[1]);
-    }
-});
-
-args = window.location.hash.slice(1).split('&');
-args.forEach(function (arg) {
-    var tmp = arg.split('=');
-    if (tmp[0] === 'access_token') {
-        accessToken = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'token_type') {
-        tokenType = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'state') {
-        state = window.decodeURIComponent(tmp[1]);
-    }
-});
-
-if (code && redirectURI) {
-    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'code=' + code + (state ? '&state=' + state : '');
-} else if (redirectURI && accessToken) {
-    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'token=' + accessToken + (state ? '&state=' + state : '');
-} else {
-    window.location.href = '/api/v1/session/login';
-}
-
-</script>

src/oauth2views/error.ejs

@@ -1,27 +0,0 @@
-<% include header %>
-
-<!-- error tester -->
-
-<div class="layout-content">
-
-  <div class="container" style="margin-top: 50px;">
-    <div class="row">
-      <div class="col-md-2"></div>
-      <div class="col-md-8">
-        <div class="alert alert-danger">
-          <%- message %>
-        </div>
-      </div>
-      <div class="col-md-2"></div>
-    </div>
-    <div class="row">
-      <div class="col-md-2"></div>
-      <div class="col-md-8 text-center">
-        <a href="<%- adminOrigin %>">Back</a>
-      </div>
-      <div class="col-md-2"></div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/footer.ejs

@@ -1,9 +0,0 @@
-
-    <footer class="text-center">
-      <span class="text-muted">&copy; 2016-19 <a href="https://cloudron.io" target="_blank">Cloudron</a></span>
-    </footer>
-
-	</div>
-
-</body>
-</html>

src/oauth2views/header.ejs

@@ -1,41 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, height=device-height" />
-    <meta http-equiv="Content-Security-Policy" content="default-src 'unsafe-inline' 'unsafe-eval' 'self'; img-src 'self'" />
-
-    <title> <%= title %> </title>
-
-    <link href="/api/v1/cloudron/avatar?<%= Math.random() %>" rel="icon" type="image/png">
-
-    <!-- Theme CSS -->
-    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
-
-    <!-- Custom Fonts -->
-    <link href="<%= adminOrigin %>/3rdparty/fontawesome/css/all.min.css" rel="stylesheet" rel="stylesheet" type="text/css">
-
-    <!-- jQuery-->
-    <script src="<%= adminOrigin %>/3rdparty/js/jquery.min.js"></script>
-
-    <!-- Bootstrap Core JavaScript -->
-    <script src="<%= adminOrigin %>/3rdparty/js/bootstrap.min.js"></script>
-
-    <!-- Angularjs scripts -->
-    <script src="<%= adminOrigin %>/3rdparty/js/angular.min.js"></script>
-    <script src="<%= adminOrigin %>/3rdparty/js/angular-loader.min.js"></script>
-
-</head>
-
-<body>
-
-  <div class="layout-root">
-
-    <nav class="navbar navbar-default navbar-static-top shadow" role="navigation" style="margin-bottom: 0">
-      <div class="container-fluid">
-        <div class="navbar-header">
-          <a href="/" class="navbar-brand navbar-brand-icon"><img src="/api/v1/cloudron/avatar?<%= Math.random() %>" width="40" height="40"/></a>
-          <a href="/" class="navbar-brand"><%= cloudronName %></a>
-        </div>
-      </div>
-    </nav>

src/oauth2views/login.ejs

@@ -1,58 +0,0 @@
-<% include header %>
-
-<!-- login tester -->
-
-<div class="layout-content">
-  <div class="card" style="padding: 20px; margin-top: 50px; max-width: 620px;">
-    <div class="row">
-      <div class="col-md-12" style="text-align: center;">
-        <img width="128" height="128" src="<%= applicationLogo %>?<%= Math.random() %>"/>
-        <h1><small>Login to</small> <%= applicationName %></h1>
-        <br/>
-      </div>
-    </div>
-    <br/>
-<% if (error) { -%>
-    <div class="row">
-      <div class="col-md-12">
-        <h4 class="has-error"><%= error %></h4>
-      </div>
-    </div>
-<% } -%>
-    <div class="row">
-      <div class="col-md-12">
-        <form id="loginForm" action="" method="post">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <div class="form-group">
-            <label class="control-label" for="inputUsername">Username</label>
-            <input type="text" class="form-control" id="inputUsername" name="username" value="<%= username %>" autofocus required>
-          </div>
-          <div class="form-group">
-            <label class="control-label" for="inputPassword">Password</label>
-            <input type="password" class="form-control" name="password" id="inputPassword" value="<%= password %>" required>
-          </div>
-          <div class="form-group">
-            <label class="control-label" for="inputPassword">2FA Token (if enabled)</label>
-            <input type="text" class="form-control" name="totpToken" id="inputTotpToken" value="">
-          </div>
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Sign in"/>
-        </form>
-        <a href="/api/v1/session/password/resetRequest.html">Reset password</a>
-      </div>
-    </div>
-  </div>
-</div>
-
-<script>
-
-(function () {
-    'use strict';
-
-    var search = window.location.search.slice(1).split('&').map(function (item) { return item.split('='); }).reduce(function (o, k) { o[k[0]] = k[1]; return o; }, {});
-
-    document.getElementById('loginForm').action = '/api/v1/session/login?returnTo=' + search.returnTo;
-})();
-
-</script>
-
-<% include footer %>

src/oauth2views/password_reset.ejs

@@ -1,53 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<script>
-
-'use strict';
-
-// very basic angular app
-var app = angular.module('Application', []);
-app.controller('Controller', [function () {}]);
-
-</script>
-
-<div class="layout-content">
-
-  <center>
-    <h2>Hello <%= user.username %>, set a new password</h2>
-  </center>
-
-  <br/>
-
-  <div class="container" ng-app="Application" ng-controller="Controller">
-    <div class="row">
-      <div class="col-md-6 col-md-offset-3">
-        <form action="/api/v1/session/password/reset" method="post" name="resetForm" autocomplete="off" role="form" novalidate>
-          <input type="password" style="display: none;">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <input type="hidden" name="email" value="<%= email %>"/>
-          <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
-
-          <div class="form-group" ng-class="{ 'has-error': resetForm.password.$dirty && resetForm.password.$invalid }">
-            <label class="control-label" for="inputPassword">New Password</label>
-            <div class="control-label" ng-show="resetForm.password.$dirty && resetForm.password.$invalid">
-              <small ng-show="resetForm.password.$dirty && resetForm.password.$invalid">Password must be atleast 8 characters</small>
-            </div>
-            <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^.{8,30}$/" autofocus required>
-          </div>
-          <div class="form-group" ng-class="{ 'has-error': resetForm.passwordRepeat.$dirty && (password !== passwordRepeat) }">
-            <label class="control-label" for="inputPasswordRepeat">Repeat Password</label>
-            <div class="control-label" ng-show="resetForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
-              <small ng-show="resetForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
-            </div>
-            <input type="password" class="form-control" id="inputPasswordRepeat" ng-model="passwordRepeat" name="passwordRepeat" required>
-          </div>
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Set New Password" ng-disabled="resetForm.$invalid || password !== passwordRepeat"/>
-        </form>
-      </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/password_reset_request.ejs

@@ -1,30 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<div class="layout-content">
-
-  <center>
-    <h2>Reset password</h2>
-  </center>
-
-  <br/>
-
-  <div class="container">
-    <div class="row">
-      <div class="col-md-6 col-md-offset-3">
-        <form action="/api/v1/session/password/resetRequest" method="post" autocomplete="off">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <div class="form-group">
-            <label class="control-label" for="inputIdentifier">Username</label>
-            <input type="text" class="form-control" id="inputIdentifier" name="identifier" autofocus required>
-          </div>
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Reset"/>
-        </form>
-        <a href="/api/v1/session/login">Login</a>
-      </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/password_reset_sent.ejs

@@ -1,25 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<div class="layout-content">
-
-  <center>
-    <h2>Password reset successful</h2>
-  </center>
-
-  <br/>
-
-  <div class="container">
-    <div class="row">
-        <div class="col-md-6 col-md-offset-3 text-center">
-          <p>An email was sent to you with a link to set a new password.</p>
-          <br/>
-          <br/>
-          If you have not received any email, simply <a href="/api/v1/session/password/resetRequest.html">try again</a>.
-        </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/paths.js

@@ -24,8 +24,6 @@ exports = module.exports = {
     APPS_DATA_DIR: path.join(baseDir(), 'appsdata'),
     BOX_DATA_DIR: path.join(baseDir(), 'boxdata'),
 
-    CUSTOM_FILE: path.join(baseDir(), 'boxdata/custom.yml'),
-
     ACME_CHALLENGES_DIR: path.join(baseDir(), 'platformdata/acme'),
     ADDON_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons'),
     COLLECTD_APPCONFIG_DIR: path.join(baseDir(), 'platformdata/collectd/collectd.conf.d'),
@@ -37,13 +35,12 @@ exports = module.exports = {
     UPDATE_DIR: path.join(baseDir(), 'platformdata/update'),
     SNAPSHOT_INFO_FILE: path.join(baseDir(), 'platformdata/backup/snapshot-info.json'),
     DYNDNS_INFO_FILE: path.join(baseDir(), 'platformdata/dyndns-info.json'),
+    FEATURES_INFO_FILE: path.join(baseDir(), 'platformdata/features-info.json'),
     VERSION_FILE: path.join(baseDir(), 'platformdata/VERSION'),
 
-    SESSION_SECRET_FILE: path.join(baseDir(), 'boxdata/session.secret'),
-    SESSION_DIR: path.join(baseDir(), 'platformdata/sessions'),
-
     // this is not part of appdata because an icon may be set before install
     APP_ICONS_DIR: path.join(baseDir(), 'boxdata/appicons'),
+    PROFILE_ICONS_DIR: path.join(baseDir(), 'boxdata/profileicons'),
     MAIL_DATA_DIR: path.join(baseDir(), 'boxdata/mail'),
     ACME_ACCOUNT_KEY_FILE: path.join(baseDir(), 'boxdata/acme/acme.key'),
     APP_CERTS_DIR: path.join(baseDir(), 'boxdata/certs'),

src/provision.js

@@ -15,7 +15,6 @@ var appstore = require('./appstore.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    clients = require('./clients.js'),
     cloudron = require('./cloudron.js'),
     debug = require('debug')('box:provision'),
     domains = require('./domains.js'),
@@ -27,9 +26,9 @@ var appstore = require('./appstore.js'),
     semver = require('semver'),
     settings = require('./settings.js'),
     sysinfo = require('./sysinfo.js'),
-    superagent = require('superagent'),
     users = require('./users.js'),
     tld = require('tldjs'),
+    tokens = require('./tokens.js'),
     _ = require('underscore');
 
 const NOOP_CALLBACK = function (error) { if (error) debug(error); };
@@ -151,31 +150,6 @@ function setup(dnsConfig, sysinfoConfig, auditSource, callback) {
     });
 }
 
-function setTimeZone(ip, callback) {
-    assert.strictEqual(typeof ip, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('setTimeZone ip:%s', ip);
-
-    superagent.get('https://geolocation.cloudron.io/json').query({ ip: ip }).timeout(10 * 1000).end(function (error, result) {
-        if ((error && !error.response) || result.statusCode !== 200) {
-            debug('Failed to get geo location: %s', error.message);
-            return callback(null);
-        }
-
-        var timezone = safe.query(result.body, 'location.time_zone');
-
-        if (!timezone || typeof timezone !== 'string') {
-            debug('No timezone in geoip response : %j', result.body);
-            return callback(null);
-        }
-
-        debug('Setting timezone to ', timezone);
-
-        settings.setTimeZone(timezone, callback);
-    });
-}
-
 function activate(username, password, email, displayName, ip, auditSource, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
@@ -187,13 +161,11 @@ function activate(username, password, email, displayName, ip, auditSource, callb
 
     debug('activating user:%s email:%s', username, email);
 
-    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
-
     users.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
         if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(new BoxError(BoxError.CONFLICT, 'Already activated'));
         if (error) return callback(error);
 
-        clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+        tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
             if (error) return callback(error);
 
             eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
@@ -218,7 +190,7 @@ function restore(backupConfig, backupId, version, sysinfoConfig, auditSource, ca
     assert.strictEqual(typeof callback, 'function');
 
     if (!semver.valid(version)) return callback(new BoxError(BoxError.BAD_FIELD, 'version is not a valid semver', { field: 'version' }));
-    if (semver.major(constants.VERSION) !== semver.major(version) || semver.minor(constants.VERSION) !== semver.minor(version)) return callback(new BoxError(BoxError.BAD_STATE, `Run cloudron-setup with --version ${version} to restore from this backup`));
+    if (constants.VERSION !== version) return callback(new BoxError(BoxError.BAD_STATE, `Run cloudron-setup with --version ${version} to restore from this backup`));
 
     if (gProvisionStatus.setup.active || gProvisionStatus.restore.active) return callback(new BoxError(BoxError.BAD_STATE, 'Already setting up or restoring'));
 
@@ -249,7 +221,7 @@ function restore(backupConfig, backupId, version, sysinfoConfig, auditSource, ca
                     backups.restore.bind(null, backupConfig, backupId, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)),
                     settings.setSysinfoConfig.bind(null, sysinfoConfig),
                     cloudron.setupDashboard.bind(null, auditSource, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)),
-                    settings.setBackupConfig.bind(null, backupConfig), // update with the latest backupConfig
+                    settings.setBackupCredentials.bind(null, backupConfig), // update just the credentials and not the policy and flags
                     eventlog.add.bind(null, eventlog.ACTION_RESTORE, auditSource, { backupId }),
                 ], function (error) {
                     gProvisionStatus.restore.active = false;
@@ -268,14 +240,16 @@ function getStatus(callback) {
     users.isActivated(function (error, activated) {
         if (error) return callback(error);
 
-        settings.getCloudronName(function (error, cloudronName) {
+        settings.getAll(function (error, allSettings) {
             if (error) return callback(error);
 
             callback(null, _.extend({
                 version: constants.VERSION,
                 apiServerOrigin: settings.apiServerOrigin(), // used by CaaS tool
+                webServerOrigin: settings.webServerOrigin(), // used by CaaS tool
                 provider: settings.provider(),
-                cloudronName: cloudronName,
+                cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
+                footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
                 adminFqdn: settings.adminDomain() ? settings.adminFqdn() : null,
                 activated: activated,
             }, gProvisionStatus));

src/reverseproxy.js

@@ -171,7 +171,7 @@ function reload(callback) {
     if (process.env.BOX_ENV === 'test') return callback();
 
     shell.sudo('reload', [ RELOAD_NGINX_CMD ], {}, function (error) {
-        if (error) return callback(new BoxError(BoxError.NGINX_ERROR, error));
+        if (error) return callback(new BoxError(BoxError.NGINX_ERROR, `Error reloading nginx: ${error.message}`));
 
         callback();
     });
@@ -346,7 +346,7 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
                 debug('ensureCertificate: getting certificate for %s with options %j', vhost, apiOptions);
 
                 api.getCertificate(vhost, domain, apiOptions, function (error, certFilePath, keyFilePath) {
-                    debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath}`);
+                    debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath || 'null'}`);
 
                     eventlog.add(currentBundle ? eventlog.ACTION_CERTIFICATE_RENEWAL : eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: vhost, errorMessage: error ? error.message : '' });
 
@@ -582,6 +582,8 @@ function renewCerts(options, auditSource, progressCallback, callback) {
 
         // add app main
         allApps.forEach(function (app) {
+            if (app.runState === apps.RSTATE_STOPPED) return; // do not renew certs of stopped apps
+
             appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'main', app: app, nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf') });
 
             app.alternateDomains.forEach(function (alternateDomain) {

src/routes/accesscontrol.js

@@ -1,138 +1,134 @@
 'use strict';
 
 exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize,
+    passwordAuth: passwordAuth,
+    tokenAuth: tokenAuth,
 
-    scope: scope,
+    authorize: authorize,
     websocketAuth: websocketAuth
 };
 
 var accesscontrol = require('../accesscontrol.js'),
     assert = require('assert'),
-    BasicStrategy = require('passport-http').BasicStrategy,
-    BearerStrategy = require('passport-http-bearer').Strategy,
     BoxError = require('../boxerror.js'),
-    clients = require('../clients.js'),
-    ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    externalLdap = require('../externalldap.js'),
     HttpError = require('connect-lastmile').HttpError,
-    LocalStrategy = require('passport-local').Strategy,
-    passport = require('passport'),
-    users = require('../users.js');
+    users = require('../users.js'),
+    speakeasy = require('speakeasy');
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function passwordAuth(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
 
-    // serialize user into session
-    passport.serializeUser(function (user, callback) {
-        callback(null, user.id);
-    });
+    if (!req.body.username || typeof req.body.username !== 'string') return next(new HttpError(400, 'A username must be non-empty string'));
+    if (!req.body.password || typeof req.body.password !== 'string') return next(new HttpError(400, 'A password must be non-empty string'));
 
-    // deserialize user from session
-    passport.deserializeUser(function(userId, callback) {
-        users.get(userId, function (error, result) {
-            if (error) return callback(error);
+    const username = req.body.username;
+    const password = req.body.password;
 
-            callback(null, result);
-        });
-    });
+    function check2FA(user) {
+        assert.strictEqual(typeof user, 'object');
 
-    // used when username/password is sent in request body. used in CLI tool login route
-    passport.use(new LocalStrategy(function (username, password, callback) {
-        if (username.indexOf('@') === -1) {
-            users.verifyWithUsername(username, password, function (error, result) {
-                if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === BoxError.INVALID_CREDENTIALS) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, result);
-            });
-        } else {
-            users.verifyWithEmail(username, password, function (error, result) {
-                if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === BoxError.INVALID_CREDENTIALS) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, result);
-            });
+        if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
+            if (!req.body.totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
+
+            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
+            if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
         }
-    }));
-
-    // Used to authenticate a OAuth2 client which uses clientId and clientSecret in the Authorization header
-    passport.use(new BasicStrategy(function (clientId, clientSecret, callback) {
-        clients.get(clientId, function (error, client) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-            if (error) return callback(error);
-            if (client.clientSecret !== clientSecret) return callback(null, false);
-            callback(null, client);
-        });
-    }));
-
-    // Used to authenticate a OAuth2 client which uses clientId and clientSecret in the request body (client_id, client_secret)
-    passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clients.get(clientId, function(error, client) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-            if (error) { return callback(error); }
-            if (client.clientSecret !== clientSecret) { return callback(null, false); }
-            callback(null, client);
-        });
-    }));
-
-    // used for "Authorization: Bearer token" or access_token query param authentication
-    passport.use(new BearerStrategy(function (token, callback) {
-        accesscontrol.validateToken(token, callback);
-    }));
-
-    callback(null);
-}
-
-function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    callback(null);
-}
-
-//  The scope middleware provides an auth middleware for routes.
-//
-//  It is used for API routes, which are authenticated using accesstokens.
-//  Those accesstokens carry OAuth scopes and the middleware takes the required
-//  scope as an argument and will verify the accesstoken against it.
-//
-//  See server.js:
-//    var profileScope = routes.oauth2.scope('profile');
-//
-function scope(requiredScope) {
-    assert.strictEqual(typeof requiredScope, 'string');
-
-    var requiredScopes = requiredScope.split(',');
-
-    return [
-        passport.authenticate(['bearer'], { session: false }),
-
-        function (req, res, next) {
-            assert(req.authInfo && typeof req.authInfo === 'object');
-
-            var error = accesscontrol.hasScopes(req.authInfo.authorizedScopes, requiredScopes);
-            if (error) return next(new HttpError(403, error.message));
-
-            next();
-        }
-    ];
-}
-
-function websocketAuth(requiredScopes, req, res, next) {
-    assert(Array.isArray(requiredScopes));
-
-    if (typeof req.query.access_token !== 'string') return next(new HttpError(401, 'Unauthorized'));
-
-    accesscontrol.validateToken(req.query.access_token, function (error, user, info) {
-        if (error) return next(new HttpError(500, error.message));
-        if (!user) return next(new HttpError(401, 'Unauthorized'));
 
         req.user = user;
 
-        var e = accesscontrol.hasScopes(info.authorizedScopes, requiredScopes);
-        if (e) return next(new HttpError(403, e.message));
+        next();
+    }
+
+    function createAndVerifyUserIfNotExist(identifier, password) {
+        assert.strictEqual(typeof identifier, 'string');
+        assert.strictEqual(typeof password, 'string');
+
+        externalLdap.createAndVerifyUserIfNotExist(identifier.toLowerCase(), password, function (error, result) {
+            if (error && error.reason === BoxError.BAD_STATE) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.CONFLICT) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.NOT_FOUND) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+            if (error) return next(new HttpError(500, error));
+
+            check2FA(result);
+        });
+    }
+
+    if (username.indexOf('@') === -1) {
+        users.verifyWithUsername(username, password, users.AP_WEBADMIN, function (error, result) {
+            if (error && error.reason === BoxError.NOT_FOUND) return createAndVerifyUserIfNotExist(username, password);
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+            if (error) return next(new HttpError(500, error));
+            if (!result) return next(new HttpError(401, 'Unauthorized'));
+
+            check2FA(result);
+        });
+    } else {
+        users.verifyWithEmail(username, password, users.AP_WEBADMIN, function (error, result) {
+            if (error && error.reason === BoxError.NOT_FOUND) return createAndVerifyUserIfNotExist(username, password);
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+            if (error) return next(new HttpError(500, error));
+            if (!result) return next(new HttpError(401, 'Unauthorized'));
+
+            check2FA(result);
+        });
+    }
+}
+
+function tokenAuth(req, res, next) {
+    var token;
+
+    // this determines the priority
+    if (req.body && req.body.access_token) token = req.body.access_token;
+    if (req.query && req.query.access_token) token = req.query.access_token;
+    if (req.headers && req.headers.authorization) {
+        var parts = req.headers.authorization.split(' ');
+        if (parts.length == 2) {
+            var scheme = parts[0];
+            var credentials = parts[1];
+
+            if (/^Bearer$/i.test(scheme)) token = credentials;
+        }
+    }
+
+    if (!token) return next(new HttpError(401, 'Unauthorized'));
+
+    accesscontrol.verifyToken(token, function (error, user) {
+        if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+        if (error) return next(new HttpError(500, error.message));
+
+        req.user = user;
+
+        next();
+    });
+}
+
+function authorize(requiredRole) {
+    assert.strictEqual(typeof requiredRole, 'string');
+
+    return function (req, res, next) {
+        assert.strictEqual(typeof req.user, 'object');
+
+        if (users.compareRoles(req.user.role, requiredRole) < 0) return next(new HttpError(403, `role '${requiredRole}' is required but user has only '${req.user.role}'`));
+
+        next();
+    };
+}
+
+function websocketAuth(requiredRole, req, res, next) {
+    assert.strictEqual(typeof requiredRole, 'string');
+
+    if (typeof req.query.access_token !== 'string') return next(new HttpError(401, 'Unauthorized'));
+
+    accesscontrol.verifyToken(req.query.access_token, function (error, user) {
+        if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+        if (error) return next(new HttpError(500, error.message));
+
+        req.user = user;
+
+        if (users.compareRoles(req.user.role, requiredRole) < 0) return next(new HttpError(403, `role '${requiredRole}' is required but user has only '${user.role}'`));
 
         next();
     });

src/routes/apppasswords.js

@@ -0,0 +1,61 @@
+'use strict';
+
+exports = module.exports = {
+    list: list,
+    get: get,
+    del: del,
+    add: add
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    users = require('../users.js');
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    users.getAppPassword(req.params.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function add(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+    if (typeof req.body.identifier !== 'string') return next(new HttpError(400, 'identifier must be string'));
+
+    users.addAppPassword(req.user.id, req.body.identifier, req.body.name, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(201, result));
+    });
+}
+
+function list(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    users.getAppPasswords(req.user.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { appPasswords: result }));
+    });
+}
+
+function del(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    // TODO: verify userId owns the id ?
+    users.delAppPassword(req.params.id, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(204, {}));
+    });
+}

src/routes/apps.js

@@ -20,6 +20,7 @@ exports = module.exports = {
     setTags: setTags,
     setIcon: setIcon,
     setMemoryLimit: setMemoryLimit,
+    setCpuShares: setCpuShares,
     setAutomaticBackup: setAutomaticBackup,
     setAutomaticUpdate: setAutomaticUpdate,
     setReverseProxyConfig: setReverseProxyConfig,
@@ -32,6 +33,7 @@ exports = module.exports = {
 
     stopApp: stopApp,
     startApp: startApp,
+    restartApp: restartApp,
     exec: exec,
     execWebSocket: execWebSocket,
 
@@ -206,6 +208,19 @@ function setMemoryLimit(req, res, next) {
     });
 }
 
+function setCpuShares(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    if (typeof req.body.cpuShares !== 'number') return next(new HttpError(400, 'cpuShares is not a number'));
+
+    apps.setCpuShares(req.params.id, req.body.cpuShares, auditSource.fromRequest(req), function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, { taskId: result.taskId }));
+    });
+}
+
 function setAutomaticBackup(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
@@ -349,18 +364,13 @@ function repairApp(req, res, next) {
 
     const data = req.body;
 
-    if (data.backupId && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
-    if (data.backupFormat && typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string or null'));
-
-    if (data.location && typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (data.domain && typeof data.domain !== 'string') return next(new HttpError(400, 'domain is required'));
-
-    if ('alternateDomains' in data) {
-        if (!Array.isArray(data.alternateDomains)) return next(new HttpError(400, 'alternateDomains must be an array'));
-        if (data.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
+    if ('manifest' in data) {
+        if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
     }
 
-    if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
+    if ('dockerImage' in data) {
+        if (!data.dockerImage || typeof data.dockerImage !== 'string') return next(new HttpError(400, 'dockerImage must be a string'));
+    }
 
     apps.repair(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -377,10 +387,9 @@ function restoreApp(req, res, next) {
 
     debug('Restore app id:%s', req.params.id);
 
-    if (!('backupId' in req.body)) return next(new HttpError(400, 'backupId is required'));
-    if (data.backupId !== null && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
+    if (!data.backupId || typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be non-empty string'));
 
-    apps.restore(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
+    apps.restore(req.params.id, data.backupId, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -395,8 +404,23 @@ function importApp(req, res, next) {
 
     debug('Importing app id:%s', req.params.id);
 
-    if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string'));
-    if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
+    if ('backupId' in data) { // if not provided, we import in-place
+        if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string'));
+        if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
+
+        if ('backupConfig' in data && typeof data.backupConfig !== 'object') return next(new HttpError(400, 'backupConfig must be an object'));
+
+        const backupConfig = req.body.backupConfig;
+
+        if (req.body.backupConfig) {
+            if (typeof backupConfig.provider !== 'string') return next(new HttpError(400, 'provider is required'));
+            if ('key' in backupConfig && typeof backupConfig.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+            if ('acceptSelfSignedCerts' in backupConfig && typeof backupConfig.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
+
+            // testing backup config can take sometime
+            req.clearTimeout();
+        }
+    }
 
     apps.importApp(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -475,6 +499,18 @@ function stopApp(req, res, next) {
     });
 }
 
+function restartApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    debug('Restart app id:%s', req.params.id);
+
+    apps.restart(req.params.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, { taskId: result.taskId }));
+    });
+}
+
 function updateApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
     assert.strictEqual(typeof req.body, 'object');

src/routes/appstore.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     getApp: getApp,
     getAppVersion: getAppVersion,
 
+    createUserToken: createUserToken,
     registerCloudron: registerCloudron,
     getSubscription: getSubscription
 };
@@ -12,35 +13,20 @@ exports = module.exports = {
 var appstore = require('../appstore.js'),
     assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    custom = require('../custom.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
-function isAppAllowed(appstoreId) {
-    if (custom.spec().appstore.blacklist.includes(appstoreId)) return false;
-
-    if (!custom.spec().appstore.whitelist) return true;
-    if (!custom.spec().appstore.whitelist[appstoreId]) return false;
-
-    return true;
-}
-
 function getApps(req, res, next) {
     appstore.getApps(function (error, apps) {
         if (error) return next(BoxError.toHttpError(error));
 
-        let filteredApps = apps.filter((app) => !custom.spec().appstore.blacklist.includes(app.id));
-        if (custom.spec().appstore.whitelist) filteredApps = filteredApps.filter((app) => app.id in custom.spec().appstore.whitelist);
-
-        next(new HttpSuccess(200, { apps: filteredApps }));
+        next(new HttpSuccess(200, { apps }));
     });
 }
 
 function getApp(req, res, next) {
     assert.strictEqual(typeof req.params.appstoreId, 'string');
 
-    if (!isAppAllowed(req.params.appstoreId)) return next(new HttpError(405, 'feature disabled by admin'));
-
     appstore.getApp(req.params.appstoreId, function (error, app) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -52,8 +38,6 @@ function getAppVersion(req, res, next) {
     assert.strictEqual(typeof req.params.appstoreId, 'string');
     assert.strictEqual(typeof req.params.versionId, 'string');
 
-    if (!isAppAllowed(req.params.appstoreId)) return next(new HttpError(405, 'feature disabled by admin'));
-
     appstore.getAppVersion(req.params.appstoreId, req.params.versionId, function (error, manifest) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -61,6 +45,14 @@ function getAppVersion(req, res, next) {
     });
 }
 
+function createUserToken(req, res, next) {
+    appstore.getUserToken(function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(201, { accessToken: result }));
+    });
+}
+
 function registerCloudron(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
@@ -68,6 +60,7 @@ function registerCloudron(req, res, next) {
     if (typeof req.body.password !== 'string' || !req.body.password) return next(new HttpError(400, 'password must be string'));
     if ('totpToken' in req.body && typeof req.body.totpToken !== 'string') return next(new HttpError(400, 'totpToken must be string'));
     if (typeof req.body.signup !== 'boolean') return next(new HttpError(400, 'signup must be a boolean'));
+    if ('purpose' in req.body && typeof req.body.purpose !== 'string') return next(new HttpError(400, 'purpose must be string'));
 
     appstore.registerWithLoginCredentials(req.body, function (error) {
         if (error) return next(BoxError.toHttpError(error));
@@ -82,6 +75,6 @@ function getSubscription(req, res, next) {
     appstore.getSubscription(function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
-        next(new HttpSuccess(200, result)); // { email, cloudronId, plan, cancel_at, status }
+        next(new HttpSuccess(200, result)); // { email, cloudronId, cloudronCreatedAt, plan, current_period_end, canceled_at, cancel_at, status, features }
     });
 }

src/routes/clients.js

@@ -1,124 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    add: add,
-    get: get,
-    del: del,
-    getAll: getAll,
-    addToken: addToken,
-    getTokens: getTokens,
-    delTokens: delTokens,
-    delToken: delToken
-};
-
-var assert = require('assert'),
-    BoxError = require('../boxerror.js'),
-    clients = require('../clients.js'),
-    constants = require('../constants.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    validUrl = require('valid-url');
-
-function add(req, res, next) {
-    var data = req.body;
-
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (typeof data.appId !== 'string' || !data.appId) return next(new HttpError(400, 'appId is required'));
-    if (typeof data.redirectURI !== 'string' || !data.redirectURI) return next(new HttpError(400, 'redirectURI is required'));
-    if (typeof data.scope !== 'string' || !data.scope) return next(new HttpError(400, 'scope is required'));
-    if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));
-
-    clients.add(data.appId, clients.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(201, result));
-    });
-}
-
-function get(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-
-    clients.get(req.params.clientId, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, result));
-    });
-}
-
-function del(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-
-    clients.get(req.params.clientId, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        // we do not allow to use the REST API to delete addon clients
-        if (result.type !== clients.TYPE_EXTERNAL) return next(new HttpError(405, 'Deleting app addon clients is not allowed.'));
-
-        clients.del(req.params.clientId, function (error, result) {
-            if (error) return next(BoxError.toHttpError(error));
-
-            next(new HttpSuccess(204, result));
-        });
-    });
-}
-
-function getAll(req, res, next) {
-    clients.getAll(function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, { clients: result }));
-    });
-}
-
-function addToken(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-    assert.strictEqual(typeof req.body, 'object');
-
-    var data = req.body;
-    var expiresAt = data.expiresAt ? parseInt(data.expiresAt, 10) : Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
-    if (isNaN(expiresAt) || expiresAt <= Date.now()) return next(new HttpError(400, 'expiresAt must be a timestamp in the future'));
-    if ('name' in req.body && typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be a string'));
-
-    clients.addTokenByUserId(req.params.clientId, req.user.id, expiresAt, { name: req.body.name || '' }, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(201, { token: result }));
-    });
-}
-
-function getTokens(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-
-    clients.getTokensByUserId(req.params.clientId, req.user.id, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        result = result.map(clients.removeTokenPrivateFields);
-
-        next(new HttpSuccess(200, { tokens: result }));
-    });
-}
-
-function delTokens(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-
-    clients.delTokensByUserId(req.params.clientId, req.user.id, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(204));
-    });
-}
-
-function delToken(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.params.tokenId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-
-    clients.delToken(req.params.clientId, req.params.tokenId, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(204));
-    });
-}

src/routes/cloudron.js

@@ -1,10 +1,16 @@
 'use strict';
 
 exports = module.exports = {
+    login: login,
+    logout: logout,
+    passwordResetRequest: passwordResetRequest,
+    passwordReset: passwordReset,
+    setupAccount: setupAccount,
     reboot: reboot,
     isRebootRequired: isRebootRequired,
     getConfig: getConfig,
     getDisks: getDisks,
+    getMemory: getMemory,
     getUpdateInfo: getUpdateInfo,
     update: update,
     checkForUpdates: checkForUpdates,
@@ -22,15 +28,135 @@ let assert = require('assert'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
     cloudron = require('../cloudron.js'),
-    custom = require('../custom.js'),
-    disks = require('../disks.js'),
+    constants = require('../constants.js'),
+    debug = require('debug')('box:routes/cloudron'),
+    eventlog = require('../eventlog.js'),
     externalLdap = require('../externalldap.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     sysinfo = require('../sysinfo.js'),
+    system = require('../system.js'),
+    tokendb = require('../tokendb.js'),
+    tokens = require('../tokens.js'),
     updater = require('../updater.js'),
+    users = require('../users.js'),
     updateChecker = require('../updatechecker.js');
 
+function login(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    if ('type' in req.body && typeof req.body.type !== 'string') return next(new HttpError(400, 'type must be a string'));
+
+    const type = req.body.type || tokens.ID_WEBADMIN;
+    const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    const auditSource = { authType: 'basic', ip: ip };
+
+    const error = tokens.validateTokenType(type);
+    if (error) return next(new HttpError(400, error.message));
+
+    tokens.add(type, req.user.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function logout(req, res) {
+    var token;
+
+    // this determines the priority
+    if (req.body && req.body.access_token) token = req.body.access_token;
+    if (req.query && req.query.access_token) token = req.query.access_token;
+    if (req.headers && req.headers.authorization) {
+        var parts = req.headers.authorization.split(' ');
+        if (parts.length == 2) {
+            var scheme = parts[0];
+            var credentials = parts[1];
+
+            if (/^Bearer$/i.test(scheme)) token = credentials;
+        }
+    }
+
+    if (!token) return res.redirect('/login.html');
+
+    tokendb.delByAccessToken(token, function () { res.redirect('/login.html'); });
+}
+
+function passwordResetRequest(req, res, next) {
+    if (!req.body.identifier || typeof req.body.identifier !== 'string') return next(new HttpError(401, 'A identifier must be non-empty string'));
+
+    users.resetPasswordByIdentifier(req.body.identifier, function (error) {
+        if (error && error.reason !== BoxError.NOT_FOUND) console.error(error);
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
+function passwordReset(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
+    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
+
+    users.getByResetToken(req.body.resetToken, function (error, userObject) {
+        if (error) return next(new HttpError(401, 'Invalid resetToken'));
+
+        if (!userObject.username) return next(new HttpError(409, 'No username set'));
+
+        // setPassword clears the resetToken
+        users.setPassword(userObject, req.body.password, function (error) {
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+            if (error) return next(new HttpError(500, error));
+
+            tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+                if (error) return next(new HttpError(500, error));
+
+                next(new HttpSuccess(202, { accessToken: result.accessToken }));
+            });
+        });
+    });
+}
+
+function setupAccount(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.email || typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be a non-empty string'));
+    if (!req.body.resetToken || typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'resetToken must be a non-empty string'));
+    if (!req.body.password || typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be a non-empty string'));
+    if (!req.body.username || typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be a non-empty string'));
+    if (!req.body.displayName || typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be a non-empty string'));
+
+    debug(`setupAccount: for email ${req.body.email} and username ${req.body.username} with token ${req.body.resetToken}`);
+
+    users.getByResetToken(req.body.resetToken, function (error, userObject) {
+        if (error) return next(new HttpError(401, 'Invalid Reset Token'));
+
+        users.update(userObject, { username: req.body.username, displayName: req.body.displayName }, auditSource.fromRequest(req), function (error) {
+            if (error && error.reason === BoxError.ALREADY_EXISTS) return next(new HttpError(409, 'Username already used'));
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+            if (error && error.reason === BoxError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
+            if (error) return next(new HttpError(500, error));
+
+            userObject.username = req.body.username;
+            userObject.displayName = req.body.displayName;
+
+            // setPassword clears the resetToken
+            users.setPassword(userObject, req.body.password, function (error) {
+                if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+                if (error) return next(new HttpError(500, error));
+
+                tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+                    if (error) return next(new HttpError(500, error));
+
+                    next(new HttpSuccess(201, { accessToken: result.accessToken }));
+                });
+            });
+        });
+    });
+}
+
 function reboot(req, res, next) {
     // Finish the request, to let the appstore know we triggered the reboot
     next(new HttpSuccess(202, {}));
@@ -55,7 +181,15 @@ function getConfig(req, res, next) {
 }
 
 function getDisks(req, res, next) {
-    disks.getDisks(function (error, result) {
+    system.getDisks(function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function getMemory(req, res, next) {
+    system.getMemory(function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, result));
@@ -156,8 +290,6 @@ function getLogStream(req, res, next) {
 function setDashboardAndMailDomain(req, res, next) {
     if (!req.body.domain || typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
 
-    if (!custom.spec().domains.changeDashboardDomain) return next(new HttpError(405, 'feature disabled by admin'));
-
     cloudron.setDashboardAndMailDomain(req.body.domain, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -168,8 +300,6 @@ function setDashboardAndMailDomain(req, res, next) {
 function prepareDashboardDomain(req, res, next) {
     if (!req.body.domain || typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
 
-    if (!custom.spec().domains.changeDashboardDomain) return next(new HttpError(405, 'feature disabled by admin'));
-
     cloudron.prepareDashboardDomain(req.body.domain, auditSource.fromRequest(req), function (error, taskId) {
         if (error) return next(BoxError.toHttpError(error));
 

src/routes/developer.js

@@ -1,35 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    login: login
-};
-
-var clients = require('../clients.js'),
-    passport = require('passport'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    speakeasy = require('speakeasy');
-
-function login(req, res, next) {
-    passport.authenticate('local', function (error, user) {
-        if (error) return next(new HttpError(500, error));
-        if (!user) return next(new HttpError(401, 'Invalid credentials'));
-
-        var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-
-        if (!user.ghost && user.twoFactorAuthenticationEnabled) {
-            if (!req.body.totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
-
-            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
-            if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
-        }
-
-        const auditSource = { authType: 'cli', ip: ip };
-        clients.issueDeveloperToken(user, auditSource, function (error, result) {
-            if (error) return next(new HttpError(500, error));
-
-            next(new HttpSuccess(200, result));
-        });
-    })(req, res, next);
-}
-

src/routes/domains.js

@@ -8,8 +8,6 @@ exports = module.exports = {
     del: del,
 
     checkDnsRecords: checkDnsRecords,
-
-    verifyDomainLock: verifyDomainLock
 };
 
 var assert = require('assert'),
@@ -19,18 +17,6 @@ var assert = require('assert'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
-function verifyDomainLock(req, res, next) {
-    assert.strictEqual(typeof req.params.domain, 'string');
-
-    domains.get(req.params.domain, function (error, domain) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        if (domain.locked) return next(new HttpError(423, 'This domain is locked'));
-
-        next();
-    });
-}
-
 function add(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
@@ -69,7 +55,7 @@ function add(req, res, next) {
     domains.add(req.body.domain, data, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
-        next(new HttpSuccess(201, { domain: req.body.domain, config: req.body.config }));
+        next(new HttpSuccess(201, {}));
     });
 }
 
@@ -159,4 +145,4 @@ function checkDnsRecords(req, res, next) {
 
         next(new HttpSuccess(200, { needsOverwrite: result.needsOverwrite }));
     });
-}
+}

src/routes/index.js

@@ -2,18 +2,17 @@
 
 exports = module.exports = {
     accesscontrol: require('./accesscontrol.js'),
+    appPasswords: require('./apppasswords.js'),
     apps: require('./apps.js'),
     appstore: require('./appstore.js'),
     backups: require('./backups.js'),
-    clients: require('./clients.js'),
     cloudron: require('./cloudron.js'),
-    developer: require('./developer.js'),
     domains: require('./domains.js'),
     eventlog: require('./eventlog.js'),
     graphs: require('./graphs.js'),
     groups: require('./groups.js'),
-    oauth2: require('./oauth2.js'),
     mail: require('./mail.js'),
+    mailserver: require('./mailserver.js'),
     notifications: require('./notifications.js'),
     profile: require('./profile.js'),
     provision: require('./provision.js'),
@@ -21,5 +20,6 @@ exports = module.exports = {
     settings: require('./settings.js'),
     support: require('./support.js'),
     tasks: require('./tasks.js'),
+    tokens: require('./tokens.js'),
     users: require('./users.js')
 };

src/routes/mail.js

@@ -3,7 +3,6 @@
 exports = module.exports = {
     getDomain: getDomain,
     addDomain: addDomain,
-    getDomainStats: getDomainStats,
     removeDomain: removeDomain,
 
     setDnsRecords: setDnsRecords,
@@ -31,7 +30,7 @@ exports = module.exports = {
     getList: getList,
     addList: addList,
     updateList: updateList,
-    removeList: removeList
+    removeList: removeList,
 };
 
 var assert = require('assert'),
@@ -39,11 +38,7 @@ var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     mail = require('../mail.js'),
     HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    middleware = require('../middleware/index.js'),
-    url = require('url');
-
-var mailProxy = middleware.proxy(url.parse('http://127.0.0.1:2020'));
+    HttpSuccess = require('connect-lastmile').HttpSuccess;
 
 function getDomain(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
@@ -67,19 +62,6 @@ function addDomain(req, res, next) {
     });
 }
 
-function getDomainStats(req, res, next) {
-    assert.strictEqual(typeof req.params.domain, 'string');
-
-    var parsedUrl = url.parse(req.url, true /* parseQueryString */);
-    delete parsedUrl.query['access_token'];
-    delete req.headers['authorization'];
-    delete req.headers['cookies'];
-
-    req.url = url.format({ pathname: req.params.domain, query: parsedUrl.query });
-
-    mailProxy(req, res, next);
-}
-
 function setDnsRecords(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.domain, 'string');
@@ -197,10 +179,10 @@ function listMailboxes(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
-    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a positive number'));
 
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
-    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a positive number'));
 
     mail.listMailboxes(req.params.domain, page, perPage, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -239,7 +221,7 @@ function updateMailbox(req, res, next) {
 
     if (typeof req.body.userId !== 'string') return next(new HttpError(400, 'userId must be a string'));
 
-    mail.updateMailboxOwner(req.params.name, req.params.domain, req.body.userId, function (error) {
+    mail.updateMailboxOwner(req.params.name, req.params.domain, req.body.userId, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));
@@ -261,10 +243,10 @@ function listAliases(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
-    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a positive number'));
 
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
-    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a positive number'));
 
     mail.listAliases(req.params.domain, page, perPage, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -316,7 +298,7 @@ function getList(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.params.name, 'string');
 
-    mail.getList(req.params.domain, req.params.name, function (error, result) {
+    mail.getList(req.params.name, req.params.domain, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { list: result }));
@@ -353,7 +335,7 @@ function updateList(req, res, next) {
         if (typeof req.body.members[i] !== 'string') return next(new HttpError(400, 'member must be a string'));
     }
 
-    mail.updateList(req.params.name, req.params.domain, req.body.members, function (error) {
+    mail.updateList(req.params.name, req.params.domain, req.body.members, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));

src/routes/mailserver.js

@@ -0,0 +1,43 @@
+'use strict';
+
+exports = module.exports = {
+    proxy
+};
+
+var addons = require('../addons.js'),
+    assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    middleware = require('../middleware/index.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    url = require('url');
+
+function proxy(req, res, next) {
+    assert.strictEqual(typeof req.params.pathname, 'string');
+
+    let parsedUrl = url.parse(req.url, true /* parseQueryString */);
+
+    // do not proxy protected values
+    delete parsedUrl.query['access_token'];
+    delete req.headers['authorization'];
+    delete req.headers['cookies'];
+
+    addons.getServiceDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        parsedUrl.query['access_token'] = addonDetails.token;
+        req.url = url.format({ pathname: req.params.pathname, query: parsedUrl.query });
+
+        const proxyOptions = url.parse(`https://${addonDetails.ip}:3000`);
+        proxyOptions.rejectUnauthorized = false;
+        const mailserverProxy = middleware.proxy(proxyOptions);
+
+        mailserverProxy(req, res, function (error) {
+            if (!error) return next();
+
+            if (error.code === 'ECONNREFUSED') return next(new HttpError(424, 'Unable to connect to mail server'));
+            if (error.code === 'ECONNRESET') return next(new HttpError(424, 'Unable to query mail server'));
+
+            next(new HttpError(500, error));
+        });
+    });
+}

src/routes/oauth2.js

@@ -1,563 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize,
-    loginForm: loginForm,
-    login: login,
-    logout: logout,
-    sessionCallback: sessionCallback,
-    passwordResetRequestSite: passwordResetRequestSite,
-    passwordResetRequest: passwordResetRequest,
-    passwordSentSite: passwordSentSite,
-    passwordResetSite: passwordResetSite,
-    passwordReset: passwordReset,
-    accountSetupSite: accountSetupSite,
-    accountSetup: accountSetup,
-    authorization: authorization,
-    token: token,
-    csrf: csrf
-};
-
-var apps = require('../apps.js'),
-    assert = require('assert'),
-    async = require('async'),
-    authcodedb = require('../authcodedb.js'),
-    BoxError = require('../boxerror.js'),
-    clients = require('../clients'),
-    constants = require('../constants.js'),
-    debug = require('debug')('box:routes/oauth2'),
-    eventlog = require('../eventlog.js'),
-    hat = require('../hat.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    middleware = require('../middleware/index.js'),
-    oauth2orize = require('oauth2orize'),
-    passport = require('passport'),
-    querystring = require('querystring'),
-    session = require('connect-ensure-login'),
-    settings = require('../settings.js'),
-    speakeasy = require('speakeasy'),
-    url = require('url'),
-    users = require('../users.js'),
-    util = require('util'),
-    _ = require('underscore');
-
-// appId is optional here
-function auditSource(req, appId) {
-    var tmp = {
-        authType: 'oauth',
-        ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress || null
-    };
-
-    if (appId) tmp.appId = appId;
-
-    return tmp;
-}
-
-var gServer = null;
-
-function initialize() {
-    assert.strictEqual(gServer, null);
-
-    gServer = oauth2orize.createServer();
-
-    gServer.serializeClient(function (client, callback) {
-        return callback(null, client.id);
-    });
-
-    gServer.deserializeClient(function (id, callback) {
-        clients.get(id, callback);
-    });
-
-    // grant authorization code that can be exchanged for access tokens. this is used by external oauth clients
-    gServer.grant(oauth2orize.grant.code({ scopeSeparator: ',' }, function (client, redirectURI, user, ares, callback) {
-        debug('grant code:', client.id, redirectURI, user.id, ares);
-
-        var code = hat(256);
-        var expiresAt = Date.now() + 60 * 60000; // 1 hour
-
-        authcodedb.add(code, client.id, user.id, expiresAt, function (error) {
-            if (error) return callback(error);
-
-            debug('grant code: new auth code for client %s code %s', client.id, code);
-
-            callback(null, code);
-        });
-    }));
-
-    // exchange authorization codes for access tokens. this is used by external oauth clients
-    gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI, callback) {
-        authcodedb.get(code, function (error, authCode) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-            if (error) return callback(error);
-            if (client.id !== authCode.clientId) return callback(null, false);
-
-            authcodedb.del(code, function (error) {
-                if(error) return callback(error);
-
-                clients.addTokenByUserId(client.id, authCode.userId, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                    if (error) return callback(error);
-
-                    debug('exchange: new access token for client %s user %s token %s', client.id, authCode.userId, result.accessToken.slice(0, 6)); // partial token for security
-
-                    callback(null, result.accessToken);
-                });
-            });
-        });
-    }));
-
-    // implicit token grant that skips issuing auth codes. this is used by our webadmin
-    gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client, user, ares, callback) {
-        clients.addTokenByUserId(client.id, user.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-            if (error) return callback(error);
-
-            debug('grant token: new access token for client %s user %s token %s', client.id, user.id, result.accessToken.slice(0, 6)); // partial token for security
-
-            callback(null, result.accessToken);
-        });
-    }));
-
-    // overwrite the session.ensureLoggedIn to not use res.redirect() due to a chrome bug not sending cookies on redirects
-    session.ensureLoggedIn = function (redirectTo) {
-        assert.strictEqual(typeof redirectTo, 'string');
-
-        return function (req, res, next) {
-            if (!req.isAuthenticated || !req.isAuthenticated()) {
-                if (req.session) {
-                    req.session.returnTo = req.originalUrl || req.url;
-                }
-
-                res.status(200).send(util.format('<script>window.location.href = "%s";</script>', redirectTo));
-            } else {
-                next();
-            }
-        };
-    };
-}
-
-function uninitialize() {
-    gServer = null;
-}
-
-function renderTemplate(res, template, data) {
-    assert.strictEqual(typeof res, 'object');
-    assert.strictEqual(typeof template, 'string');
-    assert.strictEqual(typeof data, 'object');
-
-    settings.getCloudronName(function (error, cloudronName) {
-        if (error) {
-            console.error(error);
-            cloudronName = 'Cloudron';
-        }
-
-        // amend template properties, for example used in the header
-        data.title = data.title || 'Cloudron';
-        data.adminOrigin = settings.adminOrigin();
-        data.cloudronName = cloudronName;
-
-        res.render(template, data);
-    });
-}
-
-function sendErrorPageOrRedirect(req, res, message) {
-    assert.strictEqual(typeof req, 'object');
-    assert.strictEqual(typeof res, 'object');
-    assert.strictEqual(typeof message, 'string');
-
-    debug('sendErrorPageOrRedirect: returnTo %s.', req.query.returnTo, message);
-
-    if (typeof req.query.returnTo !== 'string') {
-        renderTemplate(res, 'error', {
-            message: message,
-            title: 'Cloudron Error'
-        });
-    } else {
-        var u = url.parse(req.query.returnTo);
-        if (!u.protocol || !u.host) {
-            return renderTemplate(res, 'error', {
-                message: 'Invalid request. returnTo query is not a valid URI. ' + message,
-                title: 'Cloudron Error'
-            });
-        }
-
-        res.redirect(util.format('%s//%s', u.protocol, u.host));
-    }
-}
-
-// use this instead of sendErrorPageOrRedirect(), in case we have a returnTo provided in the query, to avoid login loops
-// This usually happens when the OAuth client ID is wrong
-function sendError(req, res, message) {
-    assert.strictEqual(typeof req, 'object');
-    assert.strictEqual(typeof res, 'object');
-    assert.strictEqual(typeof message, 'string');
-
-    renderTemplate(res, 'error', {
-        message: message,
-        title: 'Cloudron Error'
-    });
-}
-
-// -> GET /api/v1/session/login
-function loginForm(req, res) {
-    if (typeof req.session.returnTo !== 'string') return sendErrorPageOrRedirect(req, res, 'Invalid login request. No returnTo provided.');
-
-    var u = url.parse(req.session.returnTo, true);
-    if (!u.query.client_id) return sendErrorPageOrRedirect(req, res, 'Invalid login request. No client_id provided.');
-
-    function render(applicationName, applicationLogo) {
-        var error = req.query.error || null;
-
-        renderTemplate(res, 'login', {
-            csrf: req.csrfToken(),
-            applicationName: applicationName,
-            applicationLogo: applicationLogo,
-            error: error,
-            username: settings.isDemo() ? constants.DEMO_USERNAME : '',
-            password: settings.isDemo() ? 'cloudron' : '',
-            title: applicationName + ' Login'
-        });
-    }
-
-    function renderBuiltIn() {
-        settings.getCloudronName(function (error, cloudronName) {
-            if (error) {
-                console.error(error);
-                cloudronName = 'Cloudron';
-            }
-
-            render(cloudronName, '/api/v1/cloudron/avatar');
-        });
-    }
-
-    clients.get(u.query.client_id, function (error, result) {
-        if (error) return sendError(req, res, 'Unknown OAuth client');
-
-        switch (result.type) {
-        case clients.TYPE_BUILT_IN: return renderBuiltIn();
-        case clients.TYPE_EXTERNAL: return render(result.appId, '/api/v1/cloudron/avatar');
-        default: break;
-        }
-
-        apps.get(result.appId, function (error, result) {
-            if (error) return sendErrorPageOrRedirect(req, res, 'Unknown Application for those OAuth credentials');
-
-            var applicationName = result.fqdn;
-            render(applicationName, '/api/v1/apps/' + result.id + '/icon');
-        });
-    });
-}
-
-// -> POST /api/v1/session/login
-function login(req, res) {
-    var returnTo = req.session.returnTo || req.query.returnTo;
-
-    var failureQuery = querystring.stringify({ error: 'Invalid username or password', returnTo: returnTo });
-    passport.authenticate('local', {
-        failureRedirect: '/api/v1/session/login?' + failureQuery
-    })(req, res, function (error) {
-        if (error) return res.redirect('/api/v1/session/login?' + failureQuery); // on some exception in the handlers
-
-        if (!req.user.ghost && req.user.twoFactorAuthenticationEnabled) {
-            if (!req.body.totpToken) {
-                let failureQuery = querystring.stringify({ error: 'A 2FA token is required', returnTo: returnTo });
-                return res.redirect('/api/v1/session/login?' + failureQuery);
-            }
-
-            let verified = speakeasy.totp.verify({ secret: req.user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
-            if (!verified) {
-                let failureQuery = querystring.stringify({ error: 'The 2FA token is invalid', returnTo: returnTo });
-                return res.redirect('/api/v1/session/login?' + failureQuery);
-            }
-        }
-
-        res.redirect(returnTo);
-    });
-}
-
-// -> GET /api/v1/session/logout
-function logout(req, res) {
-    function done() {
-        req.logout();
-
-        if (req.query && req.query.redirect) res.redirect(req.query.redirect);
-        else res.redirect('/');
-    }
-
-    if (!req.query.all) return done();
-
-    // find and destroy all login sessions by this user - this got rather complex quickly
-    req.sessionStore.list(function (error, result) {
-        if (error) {
-            console.error('Error listing sessions', error);
-            return done();
-        }
-
-        // WARNING fix this if we change the storage backend - Great stuff!
-        var sessionIds = result.map(function(s) { return s.replace('.json', ''); });
-
-        async.each(sessionIds, function (id, callback) {
-            req.sessionStore.get(id, function (error, result) {
-                if (error) {
-                    console.error(`Error getting session ${id}`, error);
-                    return callback();
-                }
-
-                // ignore empty or non passport sessions
-                if (!result || !result.passport || !result.passport.user) return callback();
-
-                // not this user
-                if (result.passport.user !== req.user.id) return callback();
-
-                req.sessionStore.destroy(id, function (error) {
-                    if (error) console.error(`Unable to destroy session ${id}`, error);
-                    callback();
-                });
-            });
-        }, done);
-    });
-}
-
-// Form to enter email address to send a password reset request mail
-// -> GET /api/v1/session/password/resetRequest.html
-function passwordResetRequestSite(req, res) {
-    var data = {
-        csrf: req.csrfToken(),
-        title: 'Password Reset'
-    };
-
-    renderTemplate(res, 'password_reset_request', data);
-}
-
-// This route is used for above form submission
-// -> POST /api/v1/session/password/resetRequest
-function passwordResetRequest(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.identifier !== 'string') return next(new HttpError(400, 'Missing identifier')); // email or username
-
-    debug('passwordResetRequest: email or username %s.', req.body.identifier);
-
-    users.resetPasswordByIdentifier(req.body.identifier, function (error) {
-        if (error && error.reason !== BoxError.NOT_FOUND) {
-            console.error(error);
-            return sendErrorPageOrRedirect(req, res, 'User not found');
-        }
-
-        res.redirect('/api/v1/session/password/sent.html');
-    });
-}
-
-// -> GET /api/v1/session/password/sent.html
-function passwordSentSite(req, res) {
-    renderTemplate(res, 'password_reset_sent', { title: 'Cloudron Password Reset' });
-}
-
-function renderAccountSetupSite(res, req, userObject, error) {
-    renderTemplate(res, 'account_setup', {
-        user: userObject,
-        error: error,
-        csrf: req.csrfToken(),
-        resetToken: req.query.reset_token || req.body.resetToken,
-        email: req.query.email || req.body.email,
-        title: 'Account Setup'
-    });
-}
-
-// -> GET /api/v1/session/account/setup.html
-function accountSetupSite(req, res) {
-    if (!req.query.reset_token) return sendError(req, res, 'Missing Reset Token');
-    if (!req.query.email) return sendError(req, res, 'Missing Email');
-
-    users.getByResetToken(req.query.email, req.query.reset_token, function (error, userObject) {
-        if (error) return sendError(req, res, 'Invalid Email or Reset Token');
-
-        renderAccountSetupSite(res, req, userObject, '');
-    });
-}
-
-// -> POST /api/v1/session/account/setup
-function accountSetup(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.email !== 'string') return next(new HttpError(400, 'Missing email'));
-    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
-    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
-    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'Missing username'));
-    if (typeof req.body.displayName !== 'string') return next(new HttpError(400, 'Missing displayName'));
-
-    debug(`acountSetup: for email ${req.body.email} with token ${req.body.resetToken}`);
-
-    users.getByResetToken(req.body.email, req.body.resetToken, function (error, userObject) {
-        if (error) return sendError(req, res, 'Invalid Reset Token');
-
-        var data = _.pick(req.body, 'username', 'displayName');
-        users.update(userObject.id, data, auditSource(req), function (error) {
-            if (error && error.reason === BoxError.ALREADY_EXISTS) return renderAccountSetupSite(res, req, userObject, 'Username already exists');
-            if (error && error.reason === BoxError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
-            if (error && error.reason === BoxError.NOT_FOUND) return renderAccountSetupSite(res, req, userObject, 'No such user');
-            if (error) return next(new HttpError(500, error));
-
-            userObject.username = req.body.username;
-            userObject.displayName = req.body.displayName;
-
-            // setPassword clears the resetToken
-            users.setPassword(userObject.id, req.body.password, function (error) {
-                if (error && error.reason === BoxError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
-                if (error) return next(new HttpError(500, error));
-
-                clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                    if (error) return next(new HttpError(500, error));
-
-                    res.redirect(`${settings.adminOrigin()}?accessToken=${result.accessToken}&expiresAt=${result.expires}`);
-                });
-            });
-        });
-    });
-}
-
-// -> GET /api/v1/session/password/reset.html
-function passwordResetSite(req, res, next) {
-    if (!req.query.email) return next(new HttpError(400, 'Missing email'));
-    if (!req.query.reset_token) return next(new HttpError(400, 'Missing reset_token'));
-
-    users.getByResetToken(req.query.email, req.query.reset_token, function (error, user) {
-        if (error) return next(new HttpError(401, 'Invalid email or reset token'));
-
-        renderTemplate(res, 'password_reset', {
-            user: user,
-            csrf: req.csrfToken(),
-            resetToken: req.query.reset_token,
-            email: req.query.email,
-            title: 'Password Reset'
-        });
-    });
-}
-
-// -> POST /api/v1/session/password/reset
-function passwordReset(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.email !== 'string') return next(new HttpError(400, 'Missing email'));
-    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
-    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
-
-    debug(`passwordReset: for ${req.body.email} with token ${req.body.resetToken}`);
-
-    users.getByResetToken(req.body.email, req.body.resetToken, function (error, userObject) {
-        if (error) return next(new HttpError(401, 'Invalid email or resetToken'));
-
-        if (!userObject.username) return next(new HttpError(401, 'No username set'));
-
-        // setPassword clears the resetToken
-        users.setPassword(userObject.id, req.body.password, function (error) {
-            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(406, error.message));
-            if (error) return next(new HttpError(500, error));
-
-            clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                if (error) return next(new HttpError(500, error));
-
-                res.redirect(`${settings.adminOrigin()}?accessToken=${result.accessToken}&expiresAt=${result.expires}`);
-            });
-        });
-    });
-}
-
-
-// The callback page takes the redirectURI and the authCode and redirects the browser accordingly
-//
-// -> GET /api/v1/session/callback
-function sessionCallback() {
-    return [
-        session.ensureLoggedIn('/api/v1/session/login'),
-        function (req, res) {
-            renderTemplate(res, 'callback', { callbackServer: req.query.redirectURI });
-        }
-    ];
-}
-
-// The authorization endpoint is the entry point for an OAuth login.
-//
-// Each app would start OAuth by redirecting the user to:
-//
-//    /api/v1/oauth/dialog/authorize?response_type=code&client_id=<clientId>&redirect_uri=<callbackURL>&scope=<ignored>
-//
-//  - First, this will ensure the user is logged in.
-//  - Then it will redirect the browser to the given <callbackURL> containing the authcode in the query
-//
-// -> GET /api/v1/oauth/dialog/authorize
-function authorization() {
-    return [
-        function (req, res, next) {
-            if (!req.query.redirect_uri) return sendErrorPageOrRedirect(req, res, 'Invalid request. redirect_uri query param is not set.');
-            if (!req.query.client_id) return sendErrorPageOrRedirect(req, res, 'Invalid request. client_id query param is not set.');
-            if (!req.query.response_type) return sendErrorPageOrRedirect(req, res, 'Invalid request. response_type query param is not set.');
-            if (req.query.response_type !== 'code' && req.query.response_type !== 'token') return sendErrorPageOrRedirect(req, res, 'Invalid request. Only token and code response types are supported.');
-
-            session.ensureLoggedIn('/api/v1/session/login?returnTo=' + req.query.redirect_uri)(req, res, next);
-        },
-        gServer.authorization({}, function (clientId, redirectURI, callback) {
-            debug('authorization: client %s with callback to %s.', clientId, redirectURI);
-
-            clients.get(clientId, function (error, client) {
-                if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-                if (error) return callback(error);
-
-                // ignore the origin passed into form the client, but use the one from the clientdb
-                var redirectPath = url.parse(redirectURI).path;
-                var redirectOrigin = client.redirectURI;
-
-                callback(null, client, '/api/v1/session/callback?redirectURI=' + encodeURIComponent(url.resolve(redirectOrigin, redirectPath)));
-            });
-        }),
-        function (req, res, next) {
-            // Handle our different types of oauth clients
-            var type = req.oauth2.client.type;
-
-            if (type === clients.TYPE_EXTERNAL || type === clients.TYPE_BUILT_IN) {
-                eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, req.oauth2.client.appId), { userId: req.oauth2.user.id, user: users.removePrivateFields(req.oauth2.user) });
-                return next();
-            }
-
-            apps.get(req.oauth2.client.appId, function (error, appObject) {
-                if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
-
-                apps.hasAccessTo(appObject, req.oauth2.user, function (error, access) {
-                    if (error) return sendError(req, res, 'Internal error');
-                    if (!access) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
-
-                    eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, appObject.id), { userId: req.oauth2.user.id, user: users.removePrivateFields(req.oauth2.user) });
-
-                    next();
-                });
-            });
-        },
-        gServer.decision({ loadTransaction: false })
-    ];
-}
-
-//  The token endpoint allows an OAuth client to exchange an authcode with an accesstoken.
-//
-//  Authcodes are obtained using the authorization endpoint. The route is authenticated by
-//  providing a Basic auth with clientID as username and clientSecret as password.
-//  An authcode is only good for one such exchange to an accesstoken.
-//
-// -> POST /api/v1/oauth/token
-function token() {
-    return [
-        passport.authenticate(['basic', 'oauth2-client-password'], { session: false }),
-        gServer.token(), // will call the token grant callback registered in initialize()
-        gServer.errorHandler()
-    ];
-}
-
-// Cross-site request forgery protection middleware for login form
-function csrf() {
-    return [
-        middleware.csrf(),
-        function (err, req, res, next) {
-            if (err.code !== 'EBADCSRFTOKEN') return next(err);
-
-            sendErrorPageOrRedirect(req, res, 'Form expired');
-        }
-    ];
-}

src/routes/profile.js

@@ -3,6 +3,9 @@
 exports = module.exports = {
     get: get,
     update: update,
+    getAvatar: getAvatar,
+    setAvatar: setAvatar,
+    clearAvatar: clearAvatar,
     changePassword: changePassword,
     setTwoFactorAuthenticationSecret: setTwoFactorAuthenticationSecret,
     enableTwoFactorAuthentication: enableTwoFactorAuthentication,
@@ -12,14 +15,21 @@ exports = module.exports = {
 var assert = require('assert'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
+    fs = require('fs'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    path = require('path'),
+    paths = require('../paths.js'),
+    safe = require('safetydance'),
     users = require('../users.js'),
+    settings = require('../settings.js'),
     _ = require('underscore');
 
 function get(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
+    const emailHash = require('crypto').createHash('md5').update(req.user.email).digest('hex');
+
     next(new HttpSuccess(200, {
         id: req.user.id,
         username: req.user.username,
@@ -27,8 +37,9 @@ function get(req, res, next) {
         fallbackEmail: req.user.fallbackEmail,
         displayName: req.user.displayName,
         twoFactorAuthenticationEnabled: req.user.twoFactorAuthenticationEnabled,
-        admin: req.user.admin,
-        source: req.user.source
+        role: req.user.role,
+        source: req.user.source,
+        avatarUrl: fs.existsSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id)) ? `${settings.adminOrigin()}/api/v1/profile/avatar/${req.user.id}` : `https://www.gravatar.com/avatar/${emailHash}.jpg`
     }));
 }
 
@@ -42,20 +53,42 @@ function update(req, res, next) {
 
     var data = _.pick(req.body, 'email', 'fallbackEmail', 'displayName');
 
-    users.update(req.user.id, data, auditSource.fromRequest(req), function (error) {
+    users.update(req.user, data, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));
     });
 }
 
+function setAvatar(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    if (!req.files.avatar) return next(new HttpError(400, 'avatar is missing'));
+
+    if (!safe.fs.renameSync(req.files.avatar.path, path.join(paths.PROFILE_ICONS_DIR, req.user.id))) return next(new HttpError(500, safe.error));
+
+    next(new HttpSuccess(202, {}));
+}
+
+function clearAvatar(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    safe.fs.unlinkSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id));
+
+    next(new HttpSuccess(202, {}));
+}
+
+function getAvatar(req, res) {
+    res.sendFile(path.join(paths.PROFILE_ICONS_DIR, req.params.identifier));
+}
+
 function changePassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.user, 'object');
 
     if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'newPassword must be a string'));
 
-    users.setPassword(req.user.id, req.body.newPassword, function (error) {
+    users.setPassword(req.user, req.body.newPassword, function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));

src/routes/provision.js

@@ -9,14 +9,15 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    appstore = require('../appstore.js'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:routes/setup'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     provision = require('../provision.js'),
-    settings = require('../settings.js'),
-    superagent = require('superagent');
+    request = require('request'),
+    settings = require('../settings.js');
 
 function providerTokenAuth(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
@@ -24,11 +25,11 @@ function providerTokenAuth(req, res, next) {
     if (settings.provider() === 'ami') {
         if (typeof req.body.providerToken !== 'string' || !req.body.providerToken) return next(new HttpError(400, 'providerToken must be a non empty string'));
 
-        superagent.get('http://169.254.169.254/latest/meta-data/instance-id').timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return next(new HttpError(500, error));
-            if (result.statusCode !== 200) return next(new HttpError(500, 'Unable to get meta data'));
+        request.get('http://169.254.169.254/latest/meta-data/instance-id', { timeout: 30 * 1000 }, function (error, result) {
+            if (error) return next(new HttpError(422, `Network error getting EC2 metadata: ${error.message}`));
+            if (result.statusCode !== 200) return next(new HttpError(422, `Unable to get EC2 meta data. statusCode: ${result.statusCode}`));
 
-            if (result.text !== req.body.providerToken) return next(new HttpError(401, 'Invalid providerToken'));
+            if (result.body !== req.body.providerToken) return next(new HttpError(422, 'Instance ID does not match'));
 
             next();
         });
@@ -62,6 +63,8 @@ function setup(req, res, next) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
+
+        appstore.trackFinishedSetup(dnsConfig.domain);
     });
 }
 
@@ -116,5 +119,10 @@ function getStatus(req, res, next) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, status));
+
+        // check if Cloudron is not in setup state nor activated and let appstore know of the attempt
+        if (!status.activated && !status.setup.active && !status.restore.active) {
+            appstore.trackBeginSetup(status.provider);
+        }
     });
 }

src/routes/services.js

@@ -40,10 +40,11 @@ function configure(req, res, next) {
     assert.strictEqual(typeof req.params.service, 'string');
 
     if (typeof req.body.memory !== 'number') return next(new HttpError(400, 'memory must be a number'));
+    if (typeof req.body.memorySwap !== 'number') return next(new HttpError(400, 'memorySwap must be a number'));
 
     const data = {
         memory: req.body.memory,
-        memorySwap: req.body.memory * 2
+        memorySwap: req.body.memorySwap
     };
 
     addons.configureService(req.params.service, data, function (error) {

src/routes/settings.js

@@ -10,7 +10,6 @@ exports = module.exports = {
 var assert = require('assert'),
     backups = require('../backups.js'),
     BoxError = require('../boxerror.js'),
-    custom = require('../custom.js'),
     docker = require('../docker.js'),
     externalLdap = require('../externalldap.js'),
     HttpError = require('connect-lastmile').HttpError,
@@ -98,6 +97,34 @@ function setTimeZone(req, res, next) {
     });
 }
 
+function getFooter(req, res, next) {
+    settings.getFooter(function (error, footer) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { footer }));
+    });
+}
+
+function setFooter(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.footer !== 'string') return next(new HttpError(400, 'footer is required'));
+
+    settings.setFooter(req.body.footer, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, {}));
+    });
+}
+
+function getSupportConfig(req, res, next) {
+    settings.getSupportConfig(function (error, supportConfig) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, supportConfig));
+    });
+}
+
 function setCloudronAvatar(req, res, next) {
     assert.strictEqual(typeof req.files, 'object');
 
@@ -127,11 +154,6 @@ function getBackupConfig(req, res, next) {
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return next(BoxError.toHttpError(error));
 
-        // always send provider as it is used by the UI to figure if backups are disabled ('noop' backend)
-        if (!custom.spec().backups.configurable) {
-            return next(new HttpSuccess(200, { provider: backupConfig.provider }));
-        }
-
         next(new HttpSuccess(200, backups.removePrivateFields(backupConfig)));
     });
 }
@@ -139,8 +161,6 @@ function getBackupConfig(req, res, next) {
 function setBackupConfig(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (!custom.spec().backups.configurable) return next(new HttpError(405, 'feature disabled by admin'));
-
     if (typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider is required'));
     if (typeof req.body.retentionSecs !== 'number') return next(new HttpError(400, 'retentionSecs is required'));
     if (typeof req.body.intervalSecs !== 'number') return next(new HttpError(400, 'intervalSecs is required'));
@@ -225,8 +245,6 @@ function getDynamicDnsConfig(req, res, next) {
 function setDynamicDnsConfig(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (!custom.spec().domains.dynamicDns) return next(new HttpError(405, 'feature disabled by admin'));
-
     if (typeof req.body.enabled !== 'boolean') return next(new HttpError(400, 'enabled boolean is required'));
 
     settings.setDynamicDnsConfig(req.body.enabled, function (error) {
@@ -316,8 +334,12 @@ function get(req, res, next) {
     case settings.TIME_ZONE_KEY: return getTimeZone(req, res, next);
     case settings.CLOUDRON_NAME_KEY: return getCloudronName(req, res, next);
 
+    case settings.FOOTER_KEY: return getFooter(req, res, next);
+
     case settings.CLOUDRON_AVATAR_KEY: return getCloudronAvatar(req, res, next);
 
+    case settings.SUPPORT_CONFIG_KEY: return getSupportConfig(req, res, next);
+
     default: return next(new HttpError(404, 'No such setting'));
     }
 }
@@ -339,6 +361,8 @@ function set(req, res, next) {
     case settings.TIME_ZONE_KEY: return setTimeZone(req, res, next);
     case settings.CLOUDRON_NAME_KEY: return setCloudronName(req, res, next);
 
+    case settings.FOOTER_KEY: return setFooter(req, res, next);
+
     case settings.CLOUDRON_AVATAR_KEY: return setCloudronAvatar(req, res, next);
 
     default: return next(new HttpError(404, 'No such setting'));

src/routes/support.js

@@ -4,22 +4,35 @@ exports = module.exports = {
     createTicket: createTicket,
 
     getRemoteSupport: getRemoteSupport,
-    enableRemoteSupport: enableRemoteSupport
+    enableRemoteSupport: enableRemoteSupport,
+
+    canCreateTicket: canCreateTicket,
+    canEnableRemoteSupport: canEnableRemoteSupport
 };
 
 var appstore = require('../appstore.js'),
     assert = require('assert'),
-    custom = require('../custom.js'),
+    auditSource = require('../auditsource.js'),
+    constants = require('../constants.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    settings = require('../settings.js'),
     support = require('../support.js'),
     _ = require('underscore');
 
+function canCreateTicket(req, res, next) {
+    settings.getSupportConfig(function (error, supportConfig) {
+        if (error) return next(new HttpError(503, error.message));
+
+        if (!supportConfig.submitTickets) return next(new HttpError(405, 'feature disabled by admin'));
+
+        next();
+    });
+}
+
 function createTicket(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    if (!custom.spec().support.submitTickets) return next(new HttpError(405, 'feature disabled by admin'));
-
     const VALID_TYPES = [ 'feedback', 'ticket', 'app_missing', 'app_error', 'upgrade_request', 'email_error' ];
 
     if (typeof req.body.type !== 'string' || !req.body.type) return next(new HttpError(400, 'type must be string'));
@@ -29,21 +42,34 @@ function createTicket(req, res, next) {
     if (req.body.appId && typeof req.body.appId !== 'string') return next(new HttpError(400, 'appId must be string'));
     if (req.body.altEmail && typeof req.body.altEmail !== 'string') return next(new HttpError(400, 'altEmail must be string'));
 
-    appstore.createTicket(_.extend({ }, req.body, { email: req.user.email, displayName: req.user.displayName }), function (error) {
-        if (error) return next(new HttpError(503, `Error contacting cloudron.io: ${error.message}. Please email ${custom.spec().support.email}`));
+    settings.getSupportConfig(function (error, supportConfig) {
+        if (error) return next(new HttpError(503, `Error getting support config: ${error.message}`));
+        if (supportConfig.email !== constants.SUPPORT_EMAIL) return next(new HttpError(503, 'Sending to non-cloudron email not implemented yet'));
 
-        next(new HttpSuccess(201, { message: `An email for sent to ${custom.spec().support.email}. We will get back shortly!` }));
+        appstore.createTicket(_.extend({ }, req.body, { email: req.user.email, displayName: req.user.displayName }), auditSource.fromRequest(req), function (error, result) {
+            if (error) return next(new HttpError(503, `Error contacting cloudron.io: ${error.message}. Please email ${constants.SUPPORT_EMAIL}`));
+
+            next(new HttpSuccess(201, result));
+        });
+    });
+}
+
+function canEnableRemoteSupport(req, res, next) {
+    settings.getSupportConfig(function (error, supportConfig) {
+        if (error) return next(new HttpError(503, error.message));
+
+        if (!supportConfig.remoteSupport) return next(new HttpError(405, 'feature disabled by admin'));
+
+        next();
     });
 }
 
 function enableRemoteSupport(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (!custom.spec().support.remoteSupport) return next(new HttpError(405, 'feature disabled by admin'));
-
     if (typeof req.body.enable !== 'boolean') return next(new HttpError(400, 'enabled is required'));
 
-    support.enableRemoteSupport(req.body.enable, function (error) {
+    support.enableRemoteSupport(req.body.enable, auditSource.fromRequest(req), function (error) {
         if (error) return next(new HttpError(503, 'Error enabling remote support. Try running "cloudron-support --enable-ssh" on the server'));
 
         next(new HttpSuccess(202, {}));

src/routes/test/accesscontrol-test.js

@@ -1,97 +1,27 @@
 /* jslint node:true */
 /* global it:false */
 /* global describe:false */
-/* global before:false */
-/* global after:false */
 
 'use strict';
 
 var accesscontrol = require('../accesscontrol.js'),
     expect = require('expect.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    passport = require('passport');
+    HttpError = require('connect-lastmile').HttpError;
 
-describe('scopes middleware', function () {
-    var passportAuthenticateSave = null;
-
-    before(function () {
-        passportAuthenticateSave = passport.authenticate;
-        passport.authenticate = function () {
-            return function (req, res, next) { next(); };
-        };
+describe('access control middleware', function () {
+    describe('passwordAuth', function () {
+        // TBD
     });
 
-    after(function () {
-        passport.authenticate = passportAuthenticateSave;
+    describe('tokenAuth', function () {
+        // TBD
     });
 
-    it('fails due to empty scope in request', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { authorizedScopes: [ ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.be.a(HttpError);
-            done();
-        });
+    describe('authorize', function () {
+        // TBD
     });
 
-    it('fails due to wrong scope in request', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { authorizedScopes: [ 'foobar', 'something' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.be.a(HttpError);
-            done();
-        });
-    });
-
-    it('fails due to wrong scope in request', function (done) {
-        var mw = accesscontrol.scope('admin,users')[1];
-        var req = { authInfo: { authorizedScopes: [ 'foobar', 'admin' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.be.a(HttpError);
-            done();
-        });
-    });
-
-    it('succeeds with one requested scope and one provided scope', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { authorizedScopes: [ 'admin' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.not.be.ok();
-            done();
-        });
-    });
-
-    it('succeeds with one requested scope and two provided scopes', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { authorizedScopes: [ 'foobar', 'admin' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.not.be.ok();
-            done();
-        });
-    });
-
-    it('succeeds with two requested scope and two provided scopes', function (done) {
-        var mw = accesscontrol.scope('admin,foobar')[1];
-        var req = { authInfo: { authorizedScopes: [ 'foobar', 'admin' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.not.be.ok();
-            done();
-        });
-    });
-
-    it('succeeds with two requested scope and provided wildcard scope', function (done) {
-        var mw = accesscontrol.scope('admin,foobar')[1];
-        var req = { authInfo: { authorizedScopes: [ '*' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.not.be.ok();
-            done();
-        });
+    describe('websocketAuth', function () {
+        // TBD
     });
 });

src/routes/test/apps-test.js

@@ -7,11 +7,10 @@
 let apps = require('../../apps.js'),
     async = require('async'),
     child_process = require('child_process'),
-    clients = require('../../clients.js'),
     constants = require('../../constants.js'),
     crypto = require('crypto'),
     database = require('../../database.js'),
-    docker = require('../../docker.js').connection,
+    Docker = require('dockerode'),
     expect = require('expect.js'),
     fs = require('fs'),
     hat = require('../../hat.js'),
@@ -29,13 +28,16 @@ let apps = require('../../apps.js'),
     settingsdb = require('../../settingsdb.js'),
     superagent = require('superagent'),
     tokendb = require('../../tokendb.js'),
+    tokens = require('../../tokens.js'),
     url = require('url');
 
 var SERVER_URL = 'http://localhost:' + constants.PORT;
 
+const docker = new Docker({ socketPath: '/var/run/docker.sock' });
+
 // Test image information
-var TEST_IMAGE_REPO = 'cloudron/test';
-var TEST_IMAGE_TAG = '25.19.0';
+var TEST_IMAGE_REPO = 'docker.io/cloudron/io.cloudron.testapp';
+var TEST_IMAGE_TAG = '20200207-233155-725d9e015';
 var TEST_IMAGE = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
 
 const DOMAIN_0 = {
@@ -211,7 +213,7 @@ function startBox(done) {
                     token_1 = hat(8 * 32);
 
                     // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                    tokendb.add({ id: 'tid-1', accessToken: token_1, identifier: user_1_id, clientId: clients.ID_SDK, expires: Date.now() + 1000000, scope: 'apps', name: '' }, callback);
+                    tokendb.add({ id: 'tid-1', accessToken: token_1, identifier: user_1_id, clientId: tokens.ID_SDK, expires: Date.now() + 1000000, scope: 'apps', name: '' }, callback);
                 });
         },
 
@@ -631,22 +633,6 @@ describe('App API', function () {
             });
         });
 
-        it('oauth addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                expect(error).to.not.be.ok();
-
-                clients.getByAppIdAndType(APP_ID, clients.TYPE_OAUTH, function (error, client) {
-                    expect(error).to.not.be.ok();
-                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
-                    expect(client.clientSecret.length).to.be(256); // 32 hex chars (8 * 256 bits)
-                    expect(data.Config.Env).to.contain('CLOUDRON_OAUTH_CLIENT_ID=' + client.id);
-                    expect(data.Config.Env).to.contain('CLOUDRON_OAUTH_CLIENT_SECRET=' + client.clientSecret);
-                    done();
-                });
-            });
-        });
-
         it('installation - app can populate addons', function (done) {
             superagent.get(`http://localhost:${appEntry.httpPort}/populate_addons`).end(function (error, res) {
                 expect(!error).to.be.ok();
@@ -1568,6 +1554,34 @@ describe('App API', function () {
                     });
             });
         });
+
+        it('can restart app', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/restart')
+                .query({ access_token: token })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(202);
+                    taskId = res.body.taskId;
+                    done();
+                });
+        });
+
+        it('wait for app to restart', function (done) {
+            waitForTask(taskId, function () { setTimeout(done, 12000); }); // give app 12 seconds (to die and start)
+        });
+
+        it('did restart the app', function (done) {
+            apps.get(APP_ID, function (error, app) {
+                if (error) return done(error);
+
+                superagent.get('http://localhost:' + app.httpPort + APP_MANIFEST.healthCheckPath)
+                    .end(function (err, res) {
+                        if (res && res.statusCode === 200) return done();
+                        done(new Error('app is not running'));
+                    });
+            });
+        });
+
+
     });
 
     describe('uninstall', function () {

src/routes/test/backups-test.js

@@ -58,7 +58,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', DOMAIN_0.domain, [ ] /* portBindings */, { installationState: 'installed', runState: 'running' }, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', DOMAIN_0.domain, [ ] /* portBindings */, { installationState: 'installed', runState: 'running', mailboxDomain: DOMAIN_0.domain }, callback);
         },
 
         function createSettings(callback) {

src/routes/test/clients-test.js

@@ -1,546 +0,0 @@
-'use strict';
-
-/* global it:false */
-/* global describe:false */
-/* global before:false */
-/* global after:false */
-
-var accesscontrol = require('../../accesscontrol.js'),
-    async = require('async'),
-    constants = require('../../constants.js'),
-    clients = require('../../clients.js'),
-    database = require('../../database.js'),
-    oauth2 = require('../oauth2.js'),
-    expect = require('expect.js'),
-    uuid = require('uuid'),
-    hat = require('../../hat.js'),
-    superagent = require('superagent'),
-    server = require('../../server.js');
-
-var SERVER_URL = 'http://localhost:' + constants.PORT;
-
-var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
-var token = null;
-
-function setup(done) {
-    async.series([
-        server.start,
-        database._clear,
-
-        function (callback) {
-            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
-                .query({ setupToken: 'somesetuptoken' })
-                .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
-                .end(function (error, result) {
-                    expect(result).to.be.ok();
-                    expect(result.statusCode).to.equal(201);
-
-                    // stash token for further use
-                    token = result.body.token;
-
-                    callback();
-                });
-        }
-    ], done);
-}
-
-function cleanup(done) {
-    database._clear(function (error) {
-        expect(error).to.not.be.ok();
-
-        server.stop(done);
-    });
-}
-
-describe('OAuth Clients API', function () {
-    describe('add', function () {
-        before(setup),
-        after(cleanup);
-
-        it('fails without token', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails without appId', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with empty appId', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: '', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails without scope', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: 'http://foobar.com' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with empty scope', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: '' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails without redirectURI', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with empty redirectURI', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: '', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with malformed redirectURI', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: 'foobar', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with invalid name', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: '$"$%^45asdfasdfadf.adf.', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('succeeds with dash', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'fo-1234-bar', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(201);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(201);
-                    expect(result.body.id).to.be.a('string');
-                    expect(result.body.appId).to.be.a('string');
-                    expect(result.body.redirectURI).to.be.a('string');
-                    expect(result.body.clientSecret).to.be.a('string');
-                    expect(result.body.scope).to.be.a('string');
-                    expect(result.body.type).to.equal(clients.TYPE_EXTERNAL);
-
-                    done();
-                });
-        });
-    });
-
-    describe('get', function () {
-        var CLIENT_0 = {
-            id: '',
-            appId: 'someAppId-0',
-            redirectURI: 'http://some.callback0',
-            scope: accesscontrol.SCOPE_PROFILE
-        };
-
-        before(function (done) {
-            async.series([
-                setup,
-
-                function (callback) {
-                    superagent.post(SERVER_URL + '/api/v1/clients')
-                        .query({ access_token: token })
-                        .send({ appId: CLIENT_0.appId, redirectURI: CLIENT_0.redirectURI, scope: CLIENT_0.scope })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(201);
-
-                            CLIENT_0 = result.body;
-
-                            callback();
-                        });
-                }
-            ], done);
-        });
-
-        after(cleanup);
-
-        it('fails without token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-
-        it('fails with unknown id', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id.toUpperCase())
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-                    expect(result.body).to.eql(CLIENT_0);
-                    done();
-                });
-        });
-    });
-
-    describe('del', function () {
-        var CLIENT_0 = {
-            id: '',
-            appId: 'someAppId-0',
-            redirectURI: 'http://some.callback0',
-            scope: accesscontrol.SCOPE_PROFILE
-        };
-
-        var CLIENT_1 = {
-            id: '',
-            appId: 'someAppId-1',
-            redirectURI: 'http://some.callback1',
-            scope: accesscontrol.SCOPE_PROFILE,
-            type: clients.TYPE_OAUTH
-        };
-
-        before(function (done) {
-            async.series([
-                setup,
-
-                function (callback) {
-                    superagent.post(SERVER_URL + '/api/v1/clients')
-                        .query({ access_token: token })
-                        .send({ appId: CLIENT_0.appId, redirectURI: CLIENT_0.redirectURI, scope: CLIENT_0.scope })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(201);
-
-                            CLIENT_0 = result.body;
-
-                            callback();
-                        });
-                }
-            ], done);
-        });
-
-        after(cleanup);
-
-        it('fails without token', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-
-        it('fails with unknown id', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id.toUpperCase())
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(204);
-
-                    superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                        .query({ access_token: token })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(404);
-
-                            done();
-                        });
-                });
-        });
-
-        it('fails for cid-webadmin', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(405);
-
-                    superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin')
-                        .query({ access_token: token })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(200);
-
-                            done();
-                        });
-                });
-        });
-
-        it('fails for addon auth client', function (done) {
-            clients.add(CLIENT_1.appId, CLIENT_1.type, CLIENT_1.redirectURI, CLIENT_1.scope, function (error, result) {
-                expect(error).to.equal(null);
-
-                CLIENT_1.id = result.id;
-
-                superagent.del(SERVER_URL + '/api/v1/clients/' + CLIENT_1.id)
-                    .query({ access_token: token })
-                    .end(function (error, result) {
-                        expect(result.statusCode).to.equal(405);
-
-                        superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_1.id)
-                            .query({ access_token: token })
-                            .end(function (error, result) {
-                                expect(result.statusCode).to.equal(200);
-
-                                done();
-                            });
-                    });
-            });
-        });
-    });
-});
-
-describe('Clients', function () {
-    var USER_0 = {
-        userId: uuid.v4(),
-        username: 'someusername',
-        password: 'Strong#$%2345',
-        email: 'some@email.com',
-        admin: true,
-        salt: 'somesalt',
-        createdAt: (new Date()).toISOString(),
-        modifiedAt: (new Date()).toISOString(),
-        resetToken: hat(256)
-    };
-
-    // make csrf always succeed for testing
-    oauth2.csrf = function () {
-        return function (req, res, next) {
-            req.csrfToken = function () { return hat(256); };
-            next();
-        };
-    };
-
-    function setup2(done) {
-        async.series([
-            setup,
-
-            function (callback) {
-                superagent.get(SERVER_URL + '/api/v1/profile')
-                    .query({ access_token: token })
-                    .end(function (error, result) {
-                        expect(result).to.be.ok();
-                        expect(result.statusCode).to.eql(200);
-
-                        USER_0.id = result.body.id;
-
-                        callback();
-                    });
-            }
-        ], done);
-    }
-
-    describe('get', function () {
-        before(setup2);
-        after(cleanup);
-
-        it('fails due to missing token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to empty token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: '' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to wrong token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token.toUpperCase() })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-
-                    expect(result.body.clients.length).to.eql(3);
-
-                    done();
-                });
-        });
-    });
-
-    describe('get tokens by client', function () {
-        before(setup2);
-        after(cleanup);
-
-        it('fails due to missing token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to empty token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: '' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to wrong token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: token.toUpperCase() })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to unkown client', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/CID-WEBADMIN/tokens')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-
-                    expect(result.body.tokens.length).to.eql(1);
-                    expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
-
-                    done();
-                });
-        });
-    });
-
-    describe('delete tokens by client', function () {
-        before(setup2);
-        after(cleanup);
-
-        it('fails due to missing token', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to empty token', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: '' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to wrong token', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: token.toUpperCase() })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to unkown client', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/CID-WEBADMIN/tokens')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-
-                    expect(result.body.tokens.length).to.eql(1);
-                    expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
-
-                    superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                        .query({ access_token: token })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(204);
-
-                            // further calls with this token should not work
-                            superagent.get(SERVER_URL + '/api/v1/profile')
-                                .query({ access_token: token })
-                                .end(function (error, result) {
-                                    expect(result.statusCode).to.equal(401);
-                                    done();
-                                });
-                        });
-                });
-        });
-    });
-});

src/routes/test/cloudron-test.js

@@ -13,8 +13,9 @@ let async = require('async'),
     http = require('http'),
     nock = require('nock'),
     os = require('os'),
-    superagent = require('superagent'),
     server = require('../../server.js'),
+    speakeasy = require('speakeasy'),
+    superagent = require('superagent'),
     settings = require('../../settings.js'),
     tokendb = require('../../tokendb.js');
 
@@ -186,23 +187,239 @@ describe('Cloudron', function () {
                     expect(result.body.webServerOrigin).to.eql('https://cloudron.io');
                     expect(result.body.adminFqdn).to.eql(settings.adminFqdn());
                     expect(result.body.version).to.eql(constants.VERSION);
-                    expect(result.body.memory).to.eql(os.totalmem());
                     expect(result.body.cloudronName).to.be.a('string');
 
                     done();
                 });
         });
 
-        it('fails (non-admin)', function (done) {
+        it('succeeds (non-admin)', function (done) {
             superagent.get(SERVER_URL + '/api/v1/config')
                 .query({ access_token: token_1 })
                 .end(function (error, result) {
-                    expect(result.statusCode).to.equal(403);
+                    expect(result.statusCode).to.equal(200);
+                    expect(result.body.apiServerOrigin).to.eql('http://localhost:6060');
+                    expect(result.body.webServerOrigin).to.eql('https://cloudron.io');
+                    expect(result.body.adminFqdn).to.eql(settings.adminFqdn());
+                    expect(result.body.version).to.eql(constants.VERSION);
+                    expect(result.body.cloudronName).to.be.a('string');
                     done();
                 });
         });
     });
 
+    describe('login', function () {
+        before(function (done) {
+            async.series([
+                setup,
+                function (callback) {
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                        .query({ setupToken: 'somesetuptoken' })
+                        .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                        .end(function (error, result) {
+                            expect(result).to.be.ok();
+
+                            callback();
+                        });
+                },
+            ], done);
+        });
+
+        after(cleanup);
+
+        it('fails without body', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails without username', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails without password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails with empty username', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: '', password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails with empty password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME, password: '' })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails with unknown username', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME + USERNAME, password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(401);
+                    done();
+                });
+        });
+
+        it('fails with unknown email', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME + EMAIL, password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(401);
+                    done();
+                });
+        });
+
+        it('fails with wrong password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME, password: PASSWORD.toUpperCase() })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(401);
+                    done();
+                });
+        });
+
+        it('with username succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME, password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
+                    expect(result.body.accessToken).to.be.a('string');
+                    done();
+                });
+        });
+
+        it('with uppercase username succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
+                    expect(result.body.accessToken).to.be.a('string');
+                    done();
+                });
+        });
+
+        it('with email succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: EMAIL, password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
+                    expect(result.body.accessToken).to.be.a('string');
+                    done();
+                });
+        });
+
+        it('with uppercase email succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: EMAIL.toUpperCase(), password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
+                    expect(result.body.accessToken).to.be.a('string');
+                    done();
+                });
+        });
+    });
+
+    describe('2fa login', function () {
+        var secret, accessToken;
+
+        before(function (done) {
+            async.series([
+                setup,
+                function (callback) {
+                    superagent.post(`${SERVER_URL}/api/v1/cloudron/activate`).query({ setupToken: 'somesetuptoken' }).send({ username: USERNAME, password: PASSWORD, email: EMAIL }).end(function (error) {
+                        callback(error);
+                    });
+                },
+                function (callback) {
+                    superagent.post(`${SERVER_URL}/api/v1/cloudron/login`).send({ username: USERNAME, password: PASSWORD }).end(function (error, result) {
+                        accessToken = result.body.accessToken;
+                        callback(error);
+                    });
+                },
+                function (callback) {
+                    superagent.post(`${SERVER_URL}/api/v1/profile/twofactorauthentication`).query({ access_token: accessToken }).end(function (error, result) {
+                        secret = result.body.secret;
+                        callback(error);
+                    });
+                },
+                function (callback) {
+                    var totpToken = speakeasy.totp({
+                        secret: secret,
+                        encoding: 'base32'
+                    });
+
+                    superagent.post(`${SERVER_URL}/api/v1/profile/twofactorauthentication/enable`).query({ access_token: accessToken }).send({ totpToken: totpToken }).end(function (error) {
+                        callback(error);
+                    });
+                }
+            ], done);
+        });
+
+        after(function (done) {
+            async.series([
+                function (callback) {
+                    superagent.post(`${SERVER_URL}/api/v1/profile/twofactorauthentication/disable`).query({ access_token: accessToken }).send({ password: PASSWORD }).end(function (error) {
+                        callback(error);
+                    });
+                },
+                cleanup
+            ], done);
+        });
+
+        it('fails due to missing token', function (done) {
+            superagent.post(`${SERVER_URL}/api/v1/cloudron/login`).send({ username: USERNAME, password: PASSWORD }).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails due to wrong token', function (done) {
+            superagent.post(`${SERVER_URL}/api/v1/cloudron/login`).send({ username: USERNAME, password: PASSWORD }).send({ totpToken: 'wrongtoken' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            var totpToken = speakeasy.totp({
+                secret: secret,
+                encoding: 'base32'
+            });
+
+            superagent.post(`${SERVER_URL}/api/v1/cloudron/login`).send({ username: USERNAME, password: PASSWORD }).send({ totpToken: totpToken }).end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(200);
+                expect(result.body).to.be.an(Object);
+                expect(result.body.accessToken).to.be.a('string');
+                done();
+            });
+        });
+    });
+
     describe('logs', function () {
         before(function (done) {
             async.series([
@@ -271,4 +488,75 @@ describe('Cloudron', function () {
             req.on('error', done);
         });
     });
+
+    describe('misc routes', function () {
+        before(function (done) {
+            async.series([
+                setup,
+
+                function (callback) {
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                        .query({ setupToken: 'somesetuptoken' })
+                        .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                        .end(function (error, result) {
+                            expect(result).to.be.ok();
+
+                            // stash token for further use
+                            token = result.body.token;
+
+                            callback();
+                        });
+                },
+
+                function (callback) {
+                    superagent.post(SERVER_URL + '/api/v1/users')
+                        .query({ access_token: token })
+                        .send({ username: USERNAME_1, email: EMAIL_1, invite: false })
+                        .end(function (error, result) {
+                            expect(result).to.be.ok();
+                            expect(result.statusCode).to.eql(201);
+
+                            token_1 = hat(8 * 32);
+                            userId_1 = result.body.id;
+
+                            // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+                            tokendb.add({ id: 'tid-1', accessToken: token_1, identifier: userId_1, clientId: 'test-client-id', expires: Date.now() + 100000, scope: 'cloudron', name: '' }, callback);
+                        });
+                }
+            ], done);
+        });
+
+        after(cleanup);
+
+        describe('memory', function () {
+            it('cannot get without token', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/memory')
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(401);
+                        done();
+                    });
+            });
+
+            it('succeeds (admin)', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/memory')
+                    .query({ access_token: token })
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(200);
+                        expect(result.body.memory).to.eql(os.totalmem());
+                        expect(result.body.swap).to.be.a('number');
+
+                        done();
+                    });
+            });
+
+            it('fails (non-admin)', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/memory')
+                    .query({ access_token: token_1 })
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(403);
+                        done();
+                    });
+            });
+        });
+    });
 });