Fix spamd logs

this lets us put images in app_not_responding.html

CHANGES

@@ -1747,3 +1747,47 @@
 * Fix various repair workflows
 * acme2: Implement post-as-get
 
+[4.4.1]
+* ami: fix AWS provider validation
+
+[4.4.2]
+* Fix crash when reporting that DKIM is not setup correctly
+* Stopped apps cannot be updated or auto-updated
+* eventlog: track support ticket creation and remote support status
+
+[4.4.3]
+* Add restart button in recovery section
+* Fix issue where memory usage was not computed correctly
+* cloudflare: support API tokens
+
+[4.4.4]
+* Fix bug where restart button in terminal was not working
+* Add search field in apps view
+* Make app view tags and domain filter persistent
+* Add timezone UI
+
+[4.4.5]
+* Fix user listing regression in group edit dialog
+* Do not show error page for 503
+* Add mail list and mail box update events
+* Certs of stopped apps are not renewed anymore
+* Fix broken memory sliders in the services UI
+* Set CPU Shares
+* Update nodejs to 12.14.1
+* Update MySQL addon packet size to 64M
+
+[5.0.0]
+* Show backup disk usage in graphs
+* Add per-user app passwords
+* Make app not responding page customizable
+* Make footer customizable
+* Add UI to import backups
+* Display timestamps in browser timezone in the UI
+* Mail eventlog and usage
+* Add user roles - owner, admin, user manager and user
+* Setup logrotate configs for collectd since upstream does not set it up
+* mail: Add X-Envelope-To and X-Envelope-From headers for incoming mails
+* linode: add object storage backend
+* restore: carefully replace backup config
+* spam: add default corpus and global db
+

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2019 Cloudron UG
+Copyright (c) 2020 Cloudron UG
 
 With regard to the Cloudron Software:
 

baseimage/initializeBaseUbuntuImage.sh

@@ -62,10 +62,10 @@ apt-get -o Dpkg::Options::="--force-confold" install -y sudo
 cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
 echo "==> Installing node.js"
-mkdir -p /usr/local/node-10.15.1
-curl -sL https://nodejs.org/dist/v10.15.1/node-v10.15.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.15.1
-ln -sf /usr/local/node-10.15.1/bin/node /usr/bin/node
-ln -sf /usr/local/node-10.15.1/bin/npm /usr/bin/npm
+mkdir -p /usr/local/node-10.18.1
+curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
+ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
+ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
 apt-get install -y python   # Install python which is required for npm rebuild
 [[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
@@ -123,6 +123,13 @@ timedatectl set-ntp 1
 # mysql follows the system timezone
 timedatectl set-timezone UTC
 
+echo "==> Adding sshd configuration warning"
+sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://cloudron.io/documentation/security/#securing-ssh-access' -i /etc/ssh/sshd_config
+
+# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
+echo "==> Disabling motd news"
+sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
+
 # Disable bind for good measure (on online.net, kimsufi servers these are pre-installed and conflicts with unbound)
 systemctl stop bind9 || true
 systemctl disable bind9 || true

migrations/20200101235723-domains-set-cloudflare-tokenType.js

@@ -0,0 +1,22 @@
+'use strict';
+
+let async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('SELECT * FROM domains', function (error, domains) {
+        if (error) return callback(error);
+
+        async.eachSeries(domains, function (domain, iteratorCallback) {
+            if (domain.provider !== 'cloudflare') return iteratorCallback();
+
+            let config = JSON.parse(domain.configJson);
+            config.tokenType = 'GlobalApiKey';
+
+            db.runSql('UPDATE domains SET configJson = ? WHERE domain = ?', [ JSON.stringify(config), domain.domain ], iteratorCallback);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20200129052822-apps-add-cpuShares.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN cpuShares INTEGER DEFAULT 512', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN cpuShares', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200131232251-appPasswords-create-table.js

@@ -0,0 +1,26 @@
+'use strict';
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE appPasswords(' +
+                'id VARCHAR(128) NOT NULL UNIQUE,' +
+				'name VARCHAR(128) NOT NULL,' +
+                'userId VARCHAR(128) NOT NULL,' +
+                'identifier VARCHAR(128) NOT NULL,' +
+                'hashedPassword VARCHAR(1024) NOT NULL,' +
+				'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
+                'FOREIGN KEY(userId) REFERENCES users(id),' +
+                'UNIQUE (name, userId),' +
+				'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE appPasswords', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200205172146-remove-authcodedb.js

@@ -0,0 +1,22 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DROP TABLE authcodes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    var cmd = `CREATE TABLE IF NOT EXISTS authcodes(
+        authCode VARCHAR(128) NOT NULL UNIQUE,
+        userId VARCHAR(128) NOT NULL,
+        clientId VARCHAR(128) NOT NULL,
+        expiresAt BIGINT NOT NULL,
+        PRIMARY KEY(authCode)) CHARACTER SET utf8 COLLATE utf8_bin`;
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200206154808-remove-clients.js

@@ -0,0 +1,24 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DROP TABLE clients', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    var cmd = `CREATE TABLE IF NOT EXISTS clients(
+      id VARCHAR(128) NOT NULL UNIQUE,
+      appId VARCHAR(128) NOT NULL,
+      type VARCHAR(16) NOT NULL,
+      clientSecret VARCHAR(512) NOT NULL,
+      redirectURI VARCHAR(512) NOT NULL,
+      scope VARCHAR(512) NOT NULL,
+      PRIMARY KEY(id)) CHARACTER SET utf8 COLLATE utf8_bin`;
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20200214051201-domains-drop-locked.js

@@ -0,0 +1,17 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE domains DROP COLUMN locked', function (error) {
+        if (error) return callback(error);
+
+        callback();
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE domains ADD COLUMN locked BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20200221211850-users-add-role.js

@@ -0,0 +1,40 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        db.runSql.bind(db, 'ALTER TABLE users ADD COLUMN role VARCHAR(32)'),
+        function migrateAdminFlag(done) {
+            db.all('SELECT * FROM users ORDER BY createdAt', function (error, results) {
+                if (error) return done(error);
+                let ownerFound = false;
+
+                async.eachSeries(results, function (user, next) {
+                    let role;
+                    if (!ownerFound && user.admin) {
+                        role = 'owner';
+                        ownerFound = true;
+                        console.log(`Designating ${user.username} ${user.email} ${user.id} as the owner of this cloudron`);
+                    } else {
+                        role = user.admin ? 'admin' : 'user';
+                    }
+                    db.runSql('UPDATE users SET role=? WHERE id=?', [ role, user.id ], next);
+                }, done);
+            });
+        },
+        db.runSql.bind(db, 'ALTER TABLE users DROP COLUMN admin'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY role VARCHAR(32) NOT NULL'),
+        db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN role', function (error) {
+        if (error) console.error(error);
+
+        callback(error);
+    });
+};
+

migrations/schema.sql

@@ -26,8 +26,8 @@ CREATE TABLE IF NOT EXISTS users(
     fallbackEmail VARCHAR(512) DEFAULT "",
     twoFactorAuthenticationSecret VARCHAR(128) DEFAULT "",
     twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
-    admin BOOLEAN DEFAULT false,
     source VARCHAR(128) DEFAULT "",
+    role VARCHAR(32),
 
     PRIMARY KEY(id));
 
@@ -52,15 +52,6 @@ CREATE TABLE IF NOT EXISTS tokens(
     expires BIGINT NOT NULL, // FIXME: make this a timestamp
     PRIMARY KEY(accessToken));
 
-CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(128) NOT NULL UNIQUE, // prefixed with cid- to identify token easily in auth routes
-    appId VARCHAR(128) NOT NULL,     // name of the client (for external apps) or id of app (for built-in apps)
-    type VARCHAR(16) NOT NULL,
-    clientSecret VARCHAR(512) NOT NULL,
-    redirectURI VARCHAR(512) NOT NULL,
-    scope VARCHAR(512) NOT NULL,
-    PRIMARY KEY(id));
-
 CREATE TABLE IF NOT EXISTS apps(
     id VARCHAR(128) NOT NULL UNIQUE,
     appStoreId VARCHAR(128) NOT NULL, // empty for custom apps
@@ -78,6 +69,7 @@ CREATE TABLE IF NOT EXISTS apps(
     updateTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,   // when the last app update was done
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, // when this db record was updated (useful for UI caching)
     memoryLimit BIGINT DEFAULT 0,
+    cpuShares INTEGER DEFAULT 512,
     xFrameOptions VARCHAR(512),
     sso BOOLEAN DEFAULT 1, // whether user chose to enable SSO
     debugModeJson TEXT, // options for development mode
@@ -104,13 +96,6 @@ CREATE TABLE IF NOT EXISTS appPortBindings(
     FOREIGN KEY(appId) REFERENCES apps(id),
     PRIMARY KEY(hostPort));
 
-CREATE TABLE IF NOT EXISTS authcodes(
-    authCode VARCHAR(128) NOT NULL UNIQUE,
-    userId VARCHAR(128) NOT NULL,
-    clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL, // ## FIXME: make this a timestamp
-    PRIMARY KEY(authCode));
-
 CREATE TABLE IF NOT EXISTS settings(
     name VARCHAR(128) NOT NULL UNIQUE,
     value TEXT,
@@ -157,7 +142,6 @@ CREATE TABLE IF NOT EXISTS domains(
     provider VARCHAR(16) NOT NULL,
     configJson TEXT, /* JSON containing the dns backend provider config */
     tlsConfigJson TEXT, /* JSON containing the tls provider config */
-    locked BOOLEAN,
 
     PRIMARY KEY (domain))
 
@@ -231,4 +215,16 @@ CREATE TABLE IF NOT EXISTS notifications(
     PRIMARY KEY (id)
 );
 
+CREATE TABLE IF NOT EXISTS appPasswords(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    name VARCHAR(128) NOT NULL,
+    userId VARCHAR(128) NOT NULL,
+    identifier VARCHAR(128) NOT NULL, // resourceId: app id or mail or webadmin
+    hashedPassword VARCHAR(1024) NOT NULL,
+    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY(userId) REFERENCES users(id),
+    UNIQUE (name, userId),
+    PRIMARY KEY (id)
+);
+
 CHARACTER SET utf8 COLLATE utf8_bin;

package.json

@@ -17,71 +17,59 @@
     "@google-cloud/dns": "^1.1.0",
     "@google-cloud/storage": "^2.5.0",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^2.6.2",
-    "aws-sdk": "^2.476.0",
+    "async": "^2.6.3",
+    "aws-sdk": "^2.610.0",
     "body-parser": "^1.19.0",
     "cloudron-manifestformat": "^4.0.0",
     "connect": "^3.7.0",
-    "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "^1.2.1",
+    "connect-lastmile": "^1.2.2",
     "connect-timeout": "^1.9.0",
-    "cookie-parser": "^1.4.4",
-    "cookie-session": "^1.3.3",
-    "cron": "^1.7.1",
-    "csurf": "^1.10.0",
+    "cookie-session": "^1.4.0",
+    "cron": "^1.8.2",
     "db-migrate": "^0.11.6",
     "db-migrate-mysql": "^1.1.10",
     "debug": "^4.1.1",
     "dockerode": "^2.5.8",
     "ejs": "^2.6.1",
-    "ejs-cli": "^2.0.1",
+    "ejs-cli": "^2.1.1",
     "express": "^4.17.1",
-    "express-session": "^1.16.2",
     "js-yaml": "^3.13.1",
     "json": "^9.0.6",
     "ldapjs": "^1.0.2",
-    "lodash": "^4.17.11",
+    "lodash": "^4.17.15",
     "lodash.chunk": "^4.2.0",
     "mime": "^2.4.4",
-    "moment-timezone": "^0.5.25",
+    "moment-timezone": "^0.5.27",
     "morgan": "^1.9.1",
     "multiparty": "^4.2.1",
-    "mysql": "^2.17.1",
-    "nodemailer": "^6.2.1",
+    "mysql": "^2.18.1",
+    "nodemailer": "^6.4.2",
     "nodemailer-smtp-transport": "^2.7.4",
-    "oauth2orize": "^1.11.0",
     "once": "^1.4.0",
     "parse-links": "^0.1.0",
-    "passport": "^0.4.0",
-    "passport-http": "^0.3.0",
-    "passport-http-bearer": "^1.0.1",
-    "passport-local": "^1.0.0",
-    "passport-oauth2-client-password": "^0.1.2",
     "pretty-bytes": "^5.3.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
-    "qrcode": "^1.3.3",
-    "readdirp": "^3.0.2",
+    "qrcode": "^1.4.4",
+    "readdirp": "^3.3.0",
     "request": "^2.88.0",
     "rimraf": "^2.6.3",
     "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^0.7.1",
+    "safetydance": "^1.0.0",
     "semver": "^6.1.1",
-    "session-file-store": "^1.3.1",
-    "showdown": "^1.9.0",
+    "showdown": "^1.9.1",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^5.0.9",
+    "superagent": "^5.2.1",
     "supererror": "^0.7.2",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.1.0",
     "tldjs": "^2.3.1",
-    "underscore": "^1.9.1",
-    "uuid": "^3.3.2",
-    "valid-url": "^1.0.9",
+    "underscore": "^1.9.2",
+    "uuid": "^3.4.0",
     "validator": "^11.0.0",
-    "ws": "^7.0.0",
-    "xml2js": "^0.4.19"
+    "ws": "^7.2.1",
+    "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",

scripts/cloudron-setup

@@ -92,13 +92,14 @@ fi
 echo "Running cloudron-setup with args : $@" > "${LOG_FILE}"
 
 # validate arguments in the absence of data
-readonly AVAILABLE_PROVIDERS="azure, caas, cloudscale, contabo, digitalocean, ec2, exoscale, galaxygate, gce, hetzner, interox, lightsail, linode, netcup, ovh, rosehosting, scaleway, skysilk, time4vps, upcloud, vultr or generic"
+readonly AVAILABLE_PROVIDERS="azure, caas, cloudscale, contabo, digitalocean, ec2, exoscale, gce, hetzner, interox, lightsail, linode, netcup, ovh, rosehosting, scaleway, skysilk, time4vps, upcloud, vultr or generic"
 if [[ -z "${provider}" ]]; then
     echo "--provider is required ($AVAILABLE_PROVIDERS)"
     exit 1
 elif [[  \
             "${provider}" != "ami" && \
             "${provider}" != "azure" && \
+            "${provider}" != "azure-image" && \
             "${provider}" != "caas" && \
             "${provider}" != "cloudscale" && \
             "${provider}" != "contabo" && \
@@ -106,13 +107,13 @@ elif [[  \
             "${provider}" != "digitalocean-mp" && \
             "${provider}" != "ec2" && \
             "${provider}" != "exoscale" && \
-            "${provider}" != "galaxygate" && \
             "${provider}" != "gce" && \
             "${provider}" != "hetzner" && \
             "${provider}" != "interox" && \
             "${provider}" != "interox-image" && \
             "${provider}" != "lightsail" && \
             "${provider}" != "linode" && \
+            "${provider}" != "linode-oneclick" && \
             "${provider}" != "linode-stackscript" && \
             "${provider}" != "netcup" && \
             "${provider}" != "netcup-image" && \
@@ -202,21 +203,11 @@ if [[ "${initBaseImage}" == "true" ]]; then
     echo ""
 fi
 
-# NOTE: this install script only supports 3.x and above
+# NOTE: this install script only supports 4.2 and above
 echo "=> Installing version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
-# this file is used >= 4.2
 echo "${provider}" > /etc/cloudron/PROVIDER
 
-# this file is unused <= 4.2 and exists to make legacy installations work. the start script will remove this file anyway
-cat > "/etc/cloudron/cloudron.conf" <<CONF_END
-{
-    "apiServerOrigin": "${apiServerOrigin}",
-    "webServerOrigin": "${webServerOrigin}",
-    "provider": "${provider}"
-}
-CONF_END
-
 [[ -n "${license}" ]] && echo -n "$license" > /etc/cloudron/LICENSE
 
 if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
@@ -224,7 +215,6 @@ if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
     exit 1
 fi
 
-# only needed for >= 4.2
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('api_server_origin', '${apiServerOrigin}');" 2>/dev/null
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('web_server_origin', '${webServerOrigin}');" 2>/dev/null
 

scripts/cloudron-support

@@ -34,7 +34,7 @@ while true; do
     --help) echo -e "${HELP_MESSAGE}"; exit 0;;
     --enable-ssh) enableSSH="true"; shift;;
     --admin-login)
-        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE admin=1 LIMIT 1" 2>/dev/null)
+        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
         printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > /tmp/cloudron_ghost.json
         echo "Login as ${admin_username} / ${admin_password} . Remove /tmp/cloudron_ghost.json when done."
@@ -80,13 +80,13 @@ echo -e $LINE"Filesystem stats"$LINE >> $OUT
 df -h &>> $OUT
 
 echo -e $LINE"Appsdata stats"$LINE >> $OUT
-du -hcsL /home/yellowtent/appsdata/* &>> $OUT
+du -hcsL /home/yellowtent/appsdata/* &>> $OUT || true
 
 echo -e $LINE"Boxdata stats"$LINE >> $OUT
 du -hcsL /home/yellowtent/boxdata/* &>> $OUT
 
 echo -e $LINE"Backup stats (possibly misleading)"$LINE >> $OUT
-du -hcsL /var/backups/* &>> $OUT
+du -hcsL /var/backups/* &>> $OUT || true
 
 echo -e $LINE"System daemon status"$LINE >> $OUT
 systemctl status --lines=100 cloudron.target box mysql unbound cloudron-syslog nginx collectd docker &>> $OUT

scripts/createReleaseTarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v10.15.1" ]]; then
-    echo "This script requires node 10.15.1"
+if [[ "$(node --version)" != "v10.18.1" ]]; then
+    echo "This script requires node 10.18.1"
     exit 1
 fi
 

scripts/installer.sh

@@ -57,12 +57,12 @@ if [[ $(docker version --format {{.Client.Version}}) != "18.09.2" ]]; then
 fi
 
 echo "==> installer: updating node"
-if [[ "$(node --version)" != "v10.15.1" ]]; then
-    mkdir -p /usr/local/node-10.15.1
-    $curl -sL https://nodejs.org/dist/v10.15.1/node-v10.15.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.15.1
-    ln -sf /usr/local/node-10.15.1/bin/node /usr/bin/node
-    ln -sf /usr/local/node-10.15.1/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-8.11.2 /usr/local/node-8.9.3
+if [[ "$(node --version)" != "v10.18.1" ]]; then
+    mkdir -p /usr/local/node-10.18.1
+    $curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
+    ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
+    ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
+    rm -rf /usr/local/node-10.15.1
 fi
 
 # this is here (and not in updater.js) because rebuild requires the above node

setup/start.sh

@@ -47,7 +47,8 @@ mkdir -p "${PLATFORM_DATA_DIR}/backup"
 mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
          "${PLATFORM_DATA_DIR}/logs/updater" \
          "${PLATFORM_DATA_DIR}/logs/tasks" \
-         "${PLATFORM_DATA_DIR}/logs/crash"
+         "${PLATFORM_DATA_DIR}/logs/crash" \
+         "${PLATFORM_DATA_DIR}/logs/collectd"
 mkdir -p "${PLATFORM_DATA_DIR}/update"
 
 mkdir -p "${BOX_DATA_DIR}/appicons"
@@ -114,7 +115,7 @@ rm -f /etc/sudoers.d/${USER}
 cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER}
 
 echo "==> Configuring collectd"
-rm -rf /etc/collectd
+rm -rf /etc/collectd /var/log/collectd.log
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
 systemctl restart collectd

setup/start/collectd/collectd.conf

@@ -57,7 +57,7 @@ LoadPlugin logfile
 
 <Plugin logfile>
    LogLevel "info"
-   File "/var/log/collectd.log"
+   File "/home/yellowtent/platformdata/logs/collectd/collectd.log"
    Timestamp true
    PrintSeverity false
 </Plugin>

setup/start/custom.yml

@@ -37,4 +37,4 @@
 #   notifyCloudronAdmins: false
 #
 # footer:
-#   body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
+#   body: '&copy; 2020 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'

setup/start/logrotate/platform

@@ -9,6 +9,7 @@
 /home/yellowtent/platformdata/logs/sftp/*.log
 /home/yellowtent/platformdata/logs/redis-*/*.log
 /home/yellowtent/platformdata/logs/crash/*.log
+/home/yellowtent/platformdata/logs/collectd/*.log
 /home/yellowtent/platformdata/logs/updater/*.log {
     # only keep one rotated file, we currently do not send that over the api
     rotate 1

src/accesscontrol.js

@@ -1,140 +1,29 @@
 'use strict';
 
 exports = module.exports = {
-    SCOPE_APPS_READ: 'apps:read',
-    SCOPE_APPS_MANAGE: 'apps:manage',
-    SCOPE_APPSTORE: 'appstore',
-    SCOPE_CLIENTS: 'clients',
-    SCOPE_CLOUDRON: 'cloudron',
-    SCOPE_DOMAINS_READ: 'domains:read',
-    SCOPE_DOMAINS_MANAGE: 'domains:manage',
-    SCOPE_MAIL: 'mail',
-    SCOPE_PROFILE: 'profile',
-    SCOPE_SETTINGS: 'settings',
-    SCOPE_SUBSCRIPTION: 'subscription',
-    SCOPE_USERS_READ: 'users:read',
-    SCOPE_USERS_MANAGE: 'users:manage',
-    VALID_SCOPES: [ 'apps', 'appstore', 'clients', 'cloudron', 'domains', 'mail', 'profile', 'settings', 'subscription', 'users' ], // keep this sorted
-
-    SCOPE_ANY: '*',
-
-    validateScopeString: validateScopeString,
-    hasScopes: hasScopes,
-    canonicalScopeString: canonicalScopeString,
-    intersectScopes: intersectScopes,
-    validateToken: validateToken,
-    scopesForUser: scopesForUser
+    verifyToken: verifyToken
 };
 
 var assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    debug = require('debug')('box:accesscontrol'),
     tokendb = require('./tokendb.js'),
-    users = require('./users.js'),
-    _ = require('underscore');
+    users = require('./users.js');
 
-// returns scopes that does not have wildcards and is sorted
-function canonicalScopeString(scope) {
-    if (scope === exports.SCOPE_ANY) return exports.VALID_SCOPES.join(',');
-
-    return scope.split(',').sort().join(',');
-}
-
-function intersectScopes(allowedScopes, wantedScopes) {
-    assert(Array.isArray(allowedScopes), 'Expecting sorted array');
-    assert(Array.isArray(wantedScopes), 'Expecting sorted array');
-
-    if (_.isEqual(allowedScopes, wantedScopes)) return allowedScopes; // quick path
-
-    let wantedScopesMap = new Map();
-    let results = [];
-
-    // make a map of scope -> [ subscopes ]
-    for (let w of wantedScopes) {
-        let parts = w.split(':');
-        let subscopes = wantedScopesMap.get(parts[0]) || new Set();
-        subscopes.add(parts[1] || '*');
-        wantedScopesMap.set(parts[0], subscopes);
-    }
-
-    for (let a of allowedScopes) {
-        let parts = a.split(':');
-        let as = parts[1] || '*';
-
-        let subscopes = wantedScopesMap.get(parts[0]);
-        if (!subscopes) continue;
-
-        if (subscopes.has('*') || subscopes.has(as)) {
-            results.push(a);
-        } else if (as === '*') {
-            results = results.concat(Array.from(subscopes).map(function (ss) { return `${a}:${ss}`; }));
-        }
-    }
-
-    return results;
-}
-
-function validateScopeString(scope) {
-    assert.strictEqual(typeof scope, 'string');
-
-    if (scope === '') return new BoxError(BoxError.BAD_FIELD, 'Empty scope not allowed', { field: 'scope' });
-
-    // NOTE: this function intentionally does not allow '*'. This is only allowed in the db to allow
-    // us not write a migration script every time we add a new scope
-    var allValid = scope.split(',').every(function (s) { return exports.VALID_SCOPES.indexOf(s.split(':')[0]) !== -1; });
-    if (!allValid) return new BoxError(BoxError.BAD_FIELD, 'Invalid scope. Available scopes are ' + exports.VALID_SCOPES.join(', '), { field: 'scope' });
-    return null;
-}
-
-// tests if all requiredScopes are attached to the request
-function hasScopes(authorizedScopes, requiredScopes) {
-    assert(Array.isArray(authorizedScopes), 'Expecting array');
-    assert(Array.isArray(requiredScopes), 'Expecting array');
-
-    if (authorizedScopes.indexOf(exports.SCOPE_ANY) !== -1) return null;
-
-    for (var i = 0; i < requiredScopes.length; ++i) {
-        const scopeParts = requiredScopes[i].split(':');
-
-        // this allows apps:write if the token has a higher apps scope
-        if (authorizedScopes.indexOf(requiredScopes[i]) === -1 && authorizedScopes.indexOf(scopeParts[0]) === -1) {
-            debug('scope: missing scope "%s".', requiredScopes[i]);
-            return new BoxError(BoxError.NOT_FOUND, 'Missing required scope "' + requiredScopes[i] + '"');
-        }
-    }
-
-    return null;
-}
-
-function scopesForUser(user, callback) {
-    assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (user.admin) return callback(null, exports.VALID_SCOPES);
-
-    callback(null, [ 'profile', 'apps:read' ]);
-}
-
-function validateToken(accessToken, callback) {
+function verifyToken(accessToken, callback) {
     assert.strictEqual(typeof accessToken, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     tokendb.getByAccessToken(accessToken, function (error, token) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
-        if (error) return callback(error); // this triggers 'internal error' in passport
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+        if (error) return callback(error);
 
         users.get(token.identifier, function (error, user) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
+            if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
             if (error) return callback(error);
 
-            if (!user.active) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
+            if (!user.active) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
 
-            scopesForUser(user, function (error, userScopes) {
-                if (error) return callback(error);
-
-                const authorizedScopes = intersectScopes(userScopes, token.scope.split(','));
-                callback(null, user, { authorizedScopes }); // ends up in req.authInfo
-            });
+            callback(null, user);
         });
     });
 }

src/addons.js

@@ -20,6 +20,8 @@ exports = module.exports = {
     getMountsSync: getMountsSync,
     getContainerNamesSync: getContainerNamesSync,
 
+    getServiceDetails: getServiceDetails,
+
     // exported for testing
     _setupOauth: setupOauth,
     _teardownOauth: teardownOauth,
@@ -29,13 +31,11 @@ exports = module.exports = {
     SERVICE_STATUS_STOPPED: 'stopped'
 };
 
-var accesscontrol = require('./accesscontrol.js'),
-    appdb = require('./appdb.js'),
+var appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
-    clients = require('./clients.js'),
     constants = require('./constants.js'),
     crypto = require('crypto'),
     debug = require('debug')('box:addons'),
@@ -333,6 +333,9 @@ function getService(serviceName, callback) {
     var tmp = {
         name: serviceName,
         status: null,
+        memoryUsed: 0,
+        memoryPercent: 0,
+        error: null,
         config: {
             // If a property is not set then we cannot change it through the api, see below
             // memory: 0,
@@ -621,7 +624,6 @@ function updateServiceConfig(platformConfig, callback) {
 
     debug('updateServiceConfig: %j', platformConfig);
 
-    // TODO: this should possibly also rollback memory to default
     async.eachSeries([ 'mysql', 'postgresql', 'mail', 'mongodb', 'graphite' ], function iterator(serviceName, iteratorCallback) {
         const containerConfig = platformConfig[serviceName];
         let memory, memorySwap;
@@ -775,29 +777,11 @@ function setupOauth(app, options, callback) {
 
     if (!app.sso) return callback(null);
 
-    var appId = app.id;
-    var redirectURI = 'https://' + app.fqdn;
-    var scope = accesscontrol.SCOPE_PROFILE;
+    const env = [];
 
-    clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds
-        if (error && error.reason !== BoxError.NOT_FOUND) return callback(error);
+    debugApp(app, 'Setting oauth addon config to %j', env);
 
-        clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) {
-            if (error) return callback(error);
-
-            const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';
-
-            var env = [
-                { name: `${envPrefix}OAUTH_CLIENT_ID`, value: result.id },
-                { name: `${envPrefix}OAUTH_CLIENT_SECRET`, value: result.clientSecret },
-                { name: `${envPrefix}OAUTH_ORIGIN`, value: settings.adminOrigin() }
-            ];
-
-            debugApp(app, 'Setting oauth addon config to %j', env);
-
-            appdb.setAddonConfig(appId, 'oauth', env, callback);
-        });
-    });
+    appdb.setAddonConfig(app.id, 'oauth', env, callback);
 }
 
 function teardownOauth(app, options, callback) {
@@ -807,11 +791,7 @@ function teardownOauth(app, options, callback) {
 
     debugApp(app, 'teardownOauth');
 
-    clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) {
-        if (error && error.reason !== BoxError.NOT_FOUND) debug(error);
-
-        appdb.unsetAddonConfig(app.id, 'oauth', callback);
-    });
+    appdb.unsetAddonConfig(app.id, 'oauth', callback);
 }
 
 function setupEmail(app, options, callback) {

src/appdb.js

@@ -40,7 +40,7 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'subdomains.subdomain AS location', 'subdomains.domain',
-    'apps.accessRestrictionJson', 'apps.memoryLimit',
+    'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup',
     'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
@@ -242,6 +242,7 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
     const accessRestriction = data.accessRestriction || null;
     const accessRestrictionJson = JSON.stringify(accessRestriction);
     const memoryLimit = data.memoryLimit || 0;
+    const cpuShares = data.cpuShares || 512;
     const installationState = data.installationState;
     const runState = data.runState;
     const sso = 'sso' in data ? data.sso : null;
@@ -256,10 +257,10 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
     var queries = [];
 
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, '
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
             + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit,
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
             sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson ]
     });
 
@@ -348,12 +349,13 @@ function del(id, callback) {
         { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
         { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
     ];
 
     database.transaction(queries, function (error, results) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[3].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+        if (results[4].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
 
         callback(null);
     });

src/apphealthmonitor.js

@@ -186,9 +186,10 @@ function processApp(callback) {
         async.each(result, checkAppHealth, function (error) {
             if (error) console.error(error);
 
-            var alive = result
+            const alive = result
                 .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; })
-                .map(function (a) { return (a.location || 'naked_domain') + '|' + (a.manifest.id || 'customapp'); }).join(', ');
+                .map(a => a.fqdn)
+                .join(', ');
 
             debug('apps alive: [%s]', alive);
 

src/apps.js

@@ -19,6 +19,7 @@ exports = module.exports = {
     setIcon: setIcon,
     setTags: setTags,
     setMemoryLimit: setMemoryLimit,
+    setCpuShares: setCpuShares,
     setAutomaticBackup: setAutomaticBackup,
     setAutomaticUpdate: setAutomaticUpdate,
     setReverseProxyConfig: setReverseProxyConfig,
@@ -43,6 +44,7 @@ exports = module.exports = {
 
     start: start,
     stop: stop,
+    restart: restart,
 
     exec: exec,
 
@@ -79,6 +81,7 @@ exports = module.exports = {
     ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app. this is state because it blocks other operations
     ISTATE_PENDING_START: 'pending_start',
     ISTATE_PENDING_STOP: 'pending_stop',
+    ISTATE_PENDING_RESTART: 'pending_restart',
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
@@ -129,6 +132,7 @@ var appdb = require('./appdb.js'),
     tasks = require('./tasks.js'),
     TransformStream = require('stream').Transform,
     updateChecker = require('./updatechecker.js'),
+    users = require('./users.js'),
     util = require('util'),
     uuid = require('uuid'),
     validator = require('validator'),
@@ -158,7 +162,6 @@ function validatePortBindings(portBindings, manifest) {
         993, /* imaps */
         2003, /* graphite (lo) */
         2004, /* graphite (lo) */
-        2020, /* mail server */
         2514, /* cloudron-syslog (lo) */
         constants.PORT, /* app server (lo) */
         constants.SYSADMIN_PORT, /* sysadmin app server (lo) */
@@ -247,6 +250,14 @@ function validateMemoryLimit(manifest, memoryLimit) {
     return null;
 }
 
+function validateCpuShares(cpuShares) {
+    assert.strictEqual(typeof cpuShares, 'number');
+
+    if (cpuShares < 2 || cpuShares > 1024) return new BoxError(BoxError.BAD_FIELD, 'cpuShares has to be between 2 and 1024');
+
+    return null;
+}
+
 function validateDebugMode(debugMode) {
     assert.strictEqual(typeof debugMode, 'object');
 
@@ -316,23 +327,25 @@ function validateEnv(env) {
 function validateDataDir(dataDir) {
     if (dataDir === null) return null;
 
-    if (path.resolve(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, 'dataDir must be an absolute path', { field: 'dataDir' });
+    if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`, { field: 'dataDir' });
+    if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`, { field: 'dataDir' });
+    if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`, { field: 'dataDir' });
 
     // nfs shares will have the directory mounted already
     let stat = safe.fs.lstatSync(dataDir);
     if (stat) {
-        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} is not a directory`, { field: 'dataDir' });
+        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`, { field: 'dataDir' });
         let entries = safe.fs.readdirSync(dataDir);
-        if (!entries) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} could not be listed`, { field: 'dataDir' });
-        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} is not empty`, { field: 'dataDir' });
+        if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`, { field: 'dataDir' });
+        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`, { field: 'dataDir' });
     }
 
     // backup logic relies on paths not overlapping (because it recurses)
-    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} cannot be inside apps data`, { field: 'dataDir' });
+    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`, { field: 'dataDir' });
 
     // if we made it this far, it cannot start with any of these realistically
     const fhs = [ '/bin', '/boot', '/etc', '/lib', '/lib32', '/lib64', '/proc', '/run', '/sbin', '/tmp', '/usr' ];
-    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `dataDir ${dataDir} cannot be placed inside this location`, { field: 'dataDir' });
+    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`, { field: 'dataDir' });
 
     return null;
 }
@@ -381,7 +394,7 @@ function removeInternalFields(app) {
     return _.pick(app,
         'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId',
         'location', 'domain', 'fqdn', 'mailboxName', 'mailboxDomain',
-        'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit',
+        'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
         'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir');
 }
@@ -443,7 +456,7 @@ function hasAccessTo(app, user, callback) {
     // check user access
     if (app.accessRestriction.users.some(function (e) { return e === user.id; })) return callback(null, true);
 
-    if (user.admin) return callback(null, true); // admins can always access any app
+    if (users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) return callback(null, true); // admins can always access any app
 
     if (!app.accessRestriction.groups) return callback(null, false);
 
@@ -652,7 +665,7 @@ function checkAppState(app, state) {
         if (app.error.installationState === state) return null;
 
         // allow uninstall from any state
-        if (state !== exports.ISTATE_PENDING_UNINSTALL) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
+        if (state !== exports.ISTATE_PENDING_UNINSTALL && state !== exports.ISTATE_PENDING_RESTORE) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
     }
 
     return null;
@@ -931,6 +944,35 @@ function setMemoryLimit(appId, memoryLimit, auditSource, callback) {
     });
 }
 
+function setCpuShares(appId, cpuShares, auditSource, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof cpuShares, 'number');
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(appId, function (error, app) {
+        if (error) return callback(error);
+
+        error = checkAppState(app, exports.ISTATE_PENDING_RESIZE);
+        if (error) return callback(error);
+
+        error = validateCpuShares(cpuShares);
+        if (error) return callback(error);
+
+        const task = {
+            args: {},
+            values: { cpuShares }
+        };
+        addTask(appId, exports.ISTATE_PENDING_RESIZE, task, function (error, result) {
+            if (error) return callback(error);
+
+            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId, app: app, cpuShares, taskId: result.taskId });
+
+            callback(null, { taskId: result.taskId });
+        });
+    });
+}
+
 function setEnvironment(appId, env, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof env, 'object');
@@ -1229,6 +1271,8 @@ function update(appId, data, auditSource, callback) {
         error = checkAppState(app, exports.ISTATE_PENDING_UPDATE);
         if (error) return callback(error);
 
+        if (app.runState === exports.RSTATE_STOPPED) return callback(new BoxError(BoxError.BAD_STATE, 'Stopped apps cannot be updated'));
+
         downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
             if (error) return callback(error);
 
@@ -1349,6 +1393,7 @@ function getLogs(appId, options, callback) {
 }
 
 // does a re-configure when called from most states. for install/clone errors, it re-installs with an optional manifest
+// re-configure can take a dockerImage but not a manifest because re-configure does not clean up addons
 function repair(appId, data, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof data, 'object'); // { manifest }
@@ -1360,7 +1405,7 @@ function repair(appId, data, auditSource, callback) {
     get(appId, function (error, app) {
         if (error) return callback(error);
 
-        const errorState = (app.error && app.error.installationState) || exports.ISTATE_PENDING_CONFIGURE;
+        let errorState = (app.error && app.error.installationState) || exports.ISTATE_PENDING_CONFIGURE;
 
         const task = {
             args: {},
@@ -1381,6 +1426,12 @@ function repair(appId, data, auditSource, callback) {
                 task.values.manifest = data.manifest;
                 task.args.oldManifest = app.manifest;
             }
+        } else {
+            errorState = exports.ISTATE_PENDING_CONFIGURE;
+            if (data.dockerImage) {
+                let newManifest = _.extend({}, app.manifest, { dockerImage: data.dockerImage });
+                task.values.manifest = newManifest;
+            }
         }
 
         addTask(appId, errorState, task, function (error, result) {
@@ -1545,7 +1596,7 @@ function clone(appId, data, user, auditSource, callback) {
             error = validatePortBindings(portBindings, manifest);
             if (error) return callback(error);
 
-            const mailboxName = mailboxNameForLocation(location, manifest);
+            const mailboxName = app.mailboxName.endsWith('.app') ? mailboxNameForLocation(location, manifest) : app.mailboxName;
             const locations = [{subdomain: location, domain}];
             validateLocations(locations, function (error, domainObjectMap) {
                 if (error) return callback(error);
@@ -1676,6 +1727,30 @@ function stop(appId, callback) {
     });
 }
 
+function restart(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will restart app with id:%s', appId);
+
+    get(appId, function (error, app) {
+        if (error) return callback(error);
+
+        error = checkAppState(app, exports.ISTATE_PENDING_RESTART);
+        if (error) return callback(error);
+
+        const task = {
+            args: {},
+            values: { runState: exports.RSTATE_RUNNING }
+        };
+        addTask(appId, exports.ISTATE_PENDING_RESTART, task, function (error, result) {
+            if (error) return callback(error);
+
+            callback(null, { taskId: result.taskId });
+        });
+    });
+}
+
 function checkManifestConstraints(manifest) {
     assert(manifest && typeof manifest === 'object');
 
@@ -1749,6 +1824,8 @@ function canAutoupdateApp(app, newManifest) {
     if (!app.enableAutomaticUpdate) return false;
     if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(newManifest.version))) return false; // major changes are blocking
 
+    if (app.runState === exports.RSTATE_STOPPED) return false; // stopped apps won't run migration scripts and shouldn't be updated
+
     const newTcpPorts = newManifest.tcpPorts || { };
     const newUdpPorts = newManifest.udpPorts || { };
     const portBindings = app.portBindings; // this is never null

src/appstore.js

@@ -1,16 +1,22 @@
 'use strict';
 
 exports = module.exports = {
+    getFeatures: getFeatures,
+
     getApps: getApps,
     getApp: getApp,
     getAppVersion: getAppVersion,
 
+    trackBeginSetup: trackBeginSetup,
+    trackFinishedSetup: trackFinishedSetup,
+
     registerWithLoginCredentials: registerWithLoginCredentials,
     registerWithLicense: registerWithLicense,
 
     purchaseApp: purchaseApp,
     unpurchaseApp: unpurchaseApp,
 
+    getUserToken: getUserToken,
     getSubscription: getSubscription,
     isFreePlan: isFreePlan,
 
@@ -27,13 +33,13 @@ var apps = require('./apps.js'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    custom = require('./custom.js'),
     debug = require('debug')('box:appstore'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     groups = require('./groups.js'),
     mail = require('./mail.js'),
     os = require('os'),
+    paths = require('./paths.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     settings = require('./settings.js'),
@@ -43,6 +49,38 @@ var apps = require('./apps.js'),
 
 const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
+// These are the default options and will be adjusted once a subscription state is obtained
+// Keep in sync with appstore/routes/cloudrons.js
+let gFeatures = {
+    userMaxCount: null,
+    externalLdap: true,
+    eventLog: true,
+    privateDockerRegistry: true,
+    branding: true,
+    userManager: true,
+    multiAdmin: true,
+    support: true
+};
+
+// attempt to load feature cache in case appstore would be down
+let tmp = safe.JSON.parse(safe.fs.readFileSync(paths.FEATURES_INFO_FILE, 'utf8'));
+if (tmp) gFeatures = tmp;
+
+function getFeatures() {
+    return gFeatures;
+}
+
+function isAppAllowed(appstoreId, listingConfig) {
+    assert.strictEqual(typeof listingConfig, 'object');
+    assert.strictEqual(typeof appstoreId, 'string');
+
+    if (listingConfig.blacklist && listingConfig.blacklist.includes(appstoreId)) return false;
+
+    if (listingConfig.whitelist) return listingConfig.whitelist.includes(appstoreId);
+
+    return true;
+}
+
 function getCloudronToken(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -96,6 +134,23 @@ function registerUser(email, password, callback) {
     });
 }
 
+function getUserToken(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
+
+        const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
+
+        superagent.post(url).send({}).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${result.status}`));
+
+            callback(null, result.body.accessToken);
+        });
+    });
+}
+
 function getSubscription(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -110,7 +165,11 @@ function getSubscription(callback) {
             if (result.statusCode === 502) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`));
             if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`));
 
-            callback(null, result.body); // { email, subscription }
+            // update the features cache
+            gFeatures = result.body.features;
+            safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
+
+            callback(null, result.body);
         });
     });
 }
@@ -375,6 +434,35 @@ function registerCloudron(data, callback) {
     });
 }
 
+// This works without a Cloudron token as this Cloudron was not yet registered
+let gBeginSetupAlreadyTracked = false;
+function trackBeginSetup(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
+    // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
+    if (gBeginSetupAlreadyTracked) return;
+    gBeginSetupAlreadyTracked = true;
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
+
+    superagent.post(url).send({ provider }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
+// This works without a Cloudron token as this Cloudron was not yet registered
+function trackFinishedSetup(domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;
+
+    superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
 function registerWithLicense(license, domain, callback) {
     assert.strictEqual(typeof license, 'string');
     assert.strictEqual(typeof domain, 'string');
@@ -409,19 +497,20 @@ function registerWithLoginCredentials(options, callback) {
             login(options.email, options.password, options.totpToken || '', function (error, result) {
                 if (error) return callback(error);
 
-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, provider: settings.provider(), version: constants.VERSION }, callback);
+                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, provider: settings.provider(), version: constants.VERSION, purpose: options.purpose || '' }, callback);
             });
         });
     });
 }
 
-function createTicket(info, callback) {
+function createTicket(info, auditSource, callback) {
     assert.strictEqual(typeof info, 'object');
     assert.strictEqual(typeof info.email, 'string');
     assert.strictEqual(typeof info.displayName, 'string');
     assert.strictEqual(typeof info.type, 'string');
     assert.strictEqual(typeof info.subject, 'string');
     assert.strictEqual(typeof info.description, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     function collectAppInfoIfNeeded(callback) {
@@ -438,7 +527,7 @@ function createTicket(info, callback) {
 
             let url = settings.apiServerOrigin() + '/api/v1/ticket';
 
-            info.supportEmail = custom.spec().support.email; // destination address for tickets
+            info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
 
             superagent.post(url).query({ accessToken: token }).send(info).timeout(10 * 1000).end(function (error, result) {
                 if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
@@ -446,7 +535,9 @@ function createTicket(info, callback) {
                 if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
                 if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-                callback(null);
+                eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+
+                callback(null, { message: `An email for sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
             });
         });
     });
@@ -469,7 +560,13 @@ function getApps(callback) {
                 if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
                 if (!result.body.apps) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-                callback(null, result.body.apps);
+                settings.getAppstoreListingConfig(function (error, listingConfig) {
+                    if (error) return callback(error);
+
+                    const filteredApps = result.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
+
+                    callback(null, filteredApps);
+                });
             });
         });
     });
@@ -480,20 +577,26 @@ function getAppVersion(appId, version, callback) {
     assert.strictEqual(typeof version, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
+    settings.getAppstoreListingConfig(function (error, listingConfig) {
         if (error) return callback(error);
 
-        let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
-        if (version !== 'latest') url += `/versions/${version}`;
+        if (!isAppAllowed(appId, listingConfig)) return callback(new BoxError(BoxError.FEATURE_DISABLED));
 
-        superagent.get(url).query({ accessToken: token }).timeout(10 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
+        getCloudronToken(function (error, token) {
+            if (error) return callback(error);
 
-            callback(null, result.body);
+            let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
+            if (version !== 'latest') url += `/versions/${version}`;
+
+            superagent.get(url).query({ accessToken: token }).timeout(10 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
+
+                callback(null, result.body);
+            });
         });
     });
 }

src/apptask.js

@@ -27,6 +27,7 @@ var addons = require('./addons.js'),
     auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
+    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:apptask'),
     df = require('@sindresorhus/df'),
@@ -51,8 +52,7 @@ var addons = require('./addons.js'),
     util = require('util'),
     _ = require('underscore');
 
-const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
-    CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh'),
+const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/app.ejs', { encoding: 'utf8' }),
     MV_VOLUME_CMD = path.join(__dirname, 'scripts/mvvolume.sh'),
     LOGROTATE_CONFIG_EJS = fs.readFileSync(__dirname + '/logrotate.ejs', { encoding: 'utf8' }),
     CONFIGURE_LOGROTATE_CMD = path.join(__dirname, 'scripts/configurelogrotate.sh');
@@ -229,29 +229,14 @@ function addCollectdProfile(app, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir: apps.getDataDir(app, app.dataDir) });
-    fs.writeFile(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), collectdConf, function (error) {
-        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
-
-        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', app.id ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
-
-            callback(null);
-        });
-    });
+    collectd.addProfile(app.id, collectdConf, callback);
 }
 
 function removeCollectdProfile(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), function (error) {
-        if (error && error.code !== 'ENOENT') debugApp(app, 'Error removing collectd profile', error);
-        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', app.id ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
-
-            callback(null);
-        });
-    });
+    collectd.removeProfile(app.id, callback);
 }
 
 function addLogrotateConfig(app, callback) {
@@ -825,7 +810,7 @@ function update(app, args, progressCallback, callback) {
     async.series([
         // this protects against the theoretical possibility of an app being marked for update from
         // a previous version of box code
-        progressCallback.bind(null, { percent: 0, message: 'Verify manifest' }),
+        progressCallback.bind(null, { percent: 5, message: 'Verify manifest' }),
         verifyManifest.bind(null, updateConfig.manifest),
 
         function (next) {
@@ -920,6 +905,10 @@ function start(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 20, message: 'Starting container' }),
         docker.startContainer.bind(null, app.id),
 
+        // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
+        progressCallback.bind(null, { percent: 60, message: 'Configuring reverse proxy' }),
+        configureReverseProxy.bind(null, app),
+
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
@@ -952,6 +941,27 @@ function stop(app, args, progressCallback, callback) {
     });
 }
 
+function restart(app, args, progressCallback, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        progressCallback.bind(null, { percent: 20, message: 'Restarting container' }),
+        docker.restartContainer.bind(null, app.id),
+
+        progressCallback.bind(null, { percent: 100, message: 'Done' }),
+        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error starting app: %s', error);
+            return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
+        }
+        callback(null);
+    });
+}
+
 function uninstall(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
@@ -1029,6 +1039,8 @@ function run(appId, args, progressCallback, callback) {
             return start(app, args, progressCallback, callback);
         case apps.ISTATE_PENDING_STOP:
             return stop(app, args, progressCallback, callback);
+        case apps.ISTATE_PENDING_RESTART:
+            return restart(app, args, progressCallback, callback);
         default:
             debugApp(app, 'apptask launched with invalid command');
             return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState));

src/apptaskmanager.js

@@ -48,7 +48,7 @@ function scheduleTask(appId, taskId, callback) {
 
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 0, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
+        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
         gPendingTasks.push({ appId, taskId, callback });
         return;
     }
@@ -57,7 +57,7 @@ function scheduleTask(appId, taskId, callback) {
 
     if (lockError) {
         debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 0, message: waitText(lockError.operation) }, NOOP_CALLBACK);
+        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
         gPendingTasks.push({ appId, taskId, callback });
         return;
     }

src/authcodedb.js

@@ -1,78 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-exports = module.exports = {
-    get: get,
-    add: add,
-    del: del,
-    delExpired: delExpired,
-
-    _clear: clear
-};
-
-var assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js');
-
-var AUTHCODES_FIELDS = [ 'authCode', 'userId', 'clientId', 'expiresAt' ].join(',');
-
-function get(authCode, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + AUTHCODES_FIELDS + ' FROM authcodes WHERE authCode = ? AND expiresAt > ?', [ authCode, Date.now() ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Authcode not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function add(authCode, clientId, userId, expiresAt, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof expiresAt, 'number');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('INSERT INTO authcodes (authCode, clientId, userId, expiresAt) VALUES (?, ?, ?, ?)',
-        [ authCode, clientId, userId, expiresAt ], function (error, result) {
-            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-            if (error || result.affectedRows !== 1) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            callback(null);
-        });
-}
-
-function del(authCode, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes WHERE authCode = ?', [ authCode ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Authcode not found'));
-
-        callback(null);
-    });
-}
-
-function delExpired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes WHERE expiresAt <= ?', [ Date.now() ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        return callback(null, result.affectedRows);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes', function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-

src/backups.js

@@ -30,6 +30,8 @@ exports = module.exports = {
 
     checkConfiguration: checkConfiguration,
 
+    configureCollectd: configureCollectd,
+
     SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8),
 
     // for testing
@@ -44,12 +46,14 @@ var addons = require('./addons.js'),
     assert = require('assert'),
     backupdb = require('./backupdb.js'),
     BoxError = require('./boxerror.js'),
+    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     crypto = require('crypto'),
     database = require('./database.js'),
     DataLayout = require('./datalayout.js'),
     debug = require('debug')('box:backups'),
     df = require('@sindresorhus/df'),
+    ejs = require('ejs'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
     locker = require('./locker.js'),
@@ -69,6 +73,7 @@ var addons = require('./addons.js'),
     zlib = require('zlib');
 
 const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');
+const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/cloudron-backup.ejs', { encoding: 'utf8' });
 
 function debugApp(app) {
     assert(typeof app === 'object');
@@ -88,6 +93,7 @@ function api(provider) {
     case 'exoscale-sos': return require('./storage/s3.js');
     case 'wasabi': return require('./storage/s3.js');
     case 'scaleway-objectstorage': return require('./storage/s3.js');
+    case 'linode-objectstorage': return require('./storage/s3.js');
     case 'noop': return require('./storage/noop.js');
     default: return null;
     }
@@ -722,8 +728,8 @@ function runBackupUpload(uploadConfig, progressCallback, callback) {
         }
 
         callback();
-    }).on('message', function (progress) { // script sends either 'message' or 'result' property
-        if (!progress.result) return progressCallback({ message: `${progress.message} (${progressTag})` });
+    }).on('message', function (progress) { // this is { message } or { result }
+        if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
         debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
         result = progress.result;
     });
@@ -1302,3 +1308,15 @@ function checkConfiguration(callback) {
         callback(null, message);
     });
 }
+
+function configureCollectd(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (backupConfig.provider === 'filesystem') {
+        const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { backupDir: backupConfig.backupFolder });
+        collectd.addProfile('cloudron-backup', collectdConf, callback);
+    } else {
+        collectd.removeProfile('cloudron-backup', callback);
+    }
+}

src/boxerror.js

@@ -45,6 +45,7 @@ BoxError.DATABASE_ERROR = 'Database Error';
 BoxError.DNS_ERROR = 'DNS Error';
 BoxError.DOCKER_ERROR = 'Docker Error';
 BoxError.EXTERNAL_ERROR = 'External Error'; // use this for external API errors
+BoxError.FEATURE_DISABLED = 'Feature Disabled';
 BoxError.FS_ERROR = 'FileSystem Error';
 BoxError.INACTIVE = 'Inactive';
 BoxError.INTERNAL_ERROR = 'Internal Error';
@@ -77,6 +78,8 @@ BoxError.toHttpError = function (error) {
         return new HttpError(402, error);
     case BoxError.NOT_FOUND:
         return new HttpError(404, error);
+    case BoxError.FEATURE_DISABLED:
+        return new HttpError(405, error);
     case BoxError.ALREADY_EXISTS:
     case BoxError.BAD_STATE:
     case BoxError.CONFLICT:

src/cert/acme2.js

@@ -285,7 +285,7 @@ Acme2.prototype.waitForChallenge = function (challenge, callback) {
                 return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
             }
 
-            debug('waitForChallenge: status is "%s %j', result.body.status, result.body);
+            debug('waitForChallenge: status is "%s" %j', result.body.status, result.body);
 
             if (result.body.status === 'pending') return retryCallback(new BoxError(BoxError.TRY_AGAIN));
             else if (result.body.status === 'valid') return retryCallback();

src/clientdb.js

@@ -1,179 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get: get,
-    getAll: getAll,
-    getAllWithTokenCount: getAllWithTokenCount,
-    getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
-    add: add,
-    del: del,
-    getByAppId: getByAppId,
-    getByAppIdAndType: getByAppIdAndType,
-
-    upsert: upsert,
-
-    delByAppId: delByAppId,
-    delByAppIdAndType: delByAppIdAndType,
-
-    _clear: clear
-};
-
-var assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js');
-
-var CLIENTS_FIELDS = [ 'id', 'appId', 'type', 'clientSecret', 'redirectURI', 'scope' ].join(',');
-var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.type', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE id = ?', [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, `Client not found: ${id}`));
-
-        callback(null, result[0]);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients ORDER BY appId', function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAllWithTokenCount(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAllWithTokenCountByIdentifier(identifier, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? LIMIT 1', [ appId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function getByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? AND type = ? LIMIT 1', [ appId, type ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null, result[0]);
-    });
-}
-
-function add(id, appId, type, clientSecret, redirectURI, scope, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof clientSecret, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
-
-    database.query('INSERT INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-        if (error || result.affectedRows === 0) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function upsert(id, appId, type, clientSecret, redirectURI, scope, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof clientSecret, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
-
-    database.query('REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-        if (error || result.affectedRows === 0) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients WHERE id = ?', [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, `Client not found: ${id}`));
-
-        callback(null);
-    });
-}
-
-function delByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients WHERE appId=?', [ appId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null);
-    });
-}
-
-function delByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients WHERE appId=? AND type=?', [ appId, type ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Client not found'));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM clients', function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}

src/clients.js

@@ -1,329 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    add: add,
-    get: get,
-    del: del,
-    getAll: getAll,
-    getByAppIdAndType: getByAppIdAndType,
-    getTokensByUserId: getTokensByUserId,
-    delTokensByUserId: delTokensByUserId,
-    delByAppIdAndType: delByAppIdAndType,
-    addTokenByUserId: addTokenByUserId,
-    delToken: delToken,
-
-    issueDeveloperToken: issueDeveloperToken,
-
-    addDefaultClients: addDefaultClients,
-
-    removeTokenPrivateFields: removeTokenPrivateFields,
-
-    // client ids. we categorize them so we can have different restrictions based on the client
-    ID_WEBADMIN: 'cid-webadmin',    // dashboard oauth
-    ID_SDK: 'cid-sdk',              // created by user via dashboard
-    ID_CLI: 'cid-cli',              // created via cli tool
-
-    // client type enums
-    TYPE_EXTERNAL: 'external',
-    TYPE_BUILT_IN: 'built-in',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_PROXY: 'addon-proxy'
-};
-
-var apps = require('./apps.js'),
-    assert = require('assert'),
-    async = require('async'),
-    BoxError = require('./boxerror.js'),
-    clientdb = require('./clientdb.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:clients'),
-    eventlog = require('./eventlog.js'),
-    hat = require('./hat.js'),
-    accesscontrol = require('./accesscontrol.js'),
-    tokendb = require('./tokendb.js'),
-    users = require('./users.js'),
-    uuid = require('uuid'),
-    _ = require('underscore');
-
-function validateClientName(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 character', { field: 'name' });
-    if (name.length > 128) return new BoxError(BoxError.BAD_FIELD, 'name too long', { field: 'name' });
-
-    if (/[^a-zA-Z0-9-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'name can only contain alphanumerals and dash', { field: 'name' });
-
-    return null;
-}
-
-function validateTokenName(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (name.length > 64) return new BoxError(BoxError.BAD_FIELD, 'name too long', { field: 'name' });
-
-    return null;
-}
-
-function add(appId, type, redirectURI, scope, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = accesscontrol.validateScopeString(scope);
-    if (error) return callback(error);
-
-    error = validateClientName(appId);
-    if (error) return callback(error);
-
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(8 * 128);
-
-    clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
-        if (error) return callback(error);
-
-        var client = {
-            id: id,
-            appId: appId,
-            type: type,
-            clientSecret: clientSecret,
-            redirectURI: redirectURI,
-            scope: scope
-        };
-
-        callback(null, client);
-    });
-}
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.get(id, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.del(id, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.getAll(function (error, results) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, []);
-        if (error) return callback(error);
-
-        var tmp = [];
-        async.each(results, function (record, callback) {
-            if (record.type === exports.TYPE_EXTERNAL || record.type === exports.TYPE_BUILT_IN) {
-                // the appId in this case holds the name
-                record.name = record.appId;
-
-                tmp.push(record);
-
-                return callback(null);
-            }
-
-            apps.get(record.appId, function (error, result) {
-                if (error) {
-                    console.error('Failed to get app details for oauth client', record.appId, error);
-                    return callback(null);  // ignore error so we continue listing clients
-                }
-
-                if (record.type === exports.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === exports.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-
-                record.domain = result.fqdn;
-
-                tmp.push(record);
-
-                callback(null);
-            });
-        }, function (error) {
-            if (error) return callback(error);
-            callback(null, tmp);
-        });
-    });
-}
-
-function getByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.getByAppIdAndType(appId, type, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function getTokensByUserId(clientId, userId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.getByIdentifierAndClientId(userId, clientId, function (error, result) {
-        if (error && error.reason === BoxError.NOT_FOUND) {
-            // this can mean either that there are no tokens or the clientId is actually unknown
-            get(clientId, function (error/*, result*/) {
-                if (error) return callback(error);
-                callback(null, []);
-            });
-            return;
-        }
-        if (error) return callback(error);
-        callback(null, result || []);
-    });
-}
-
-function delTokensByUserId(clientId, userId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.delByIdentifierAndClientId(userId, clientId, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) {
-            // this can mean either that there are no tokens or the clientId is actually unknown
-            get(clientId, function (error/*, result*/) {
-                if (error) return callback(error);
-                callback(null);
-            });
-            return;
-        }
-        if (error) return callback(error);
-        callback(null);
-    });
-}
-
-function delByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    getByAppIdAndType(appId, type, function (error, result) {
-        if (error) return callback(error);
-
-        tokendb.delByClientId(result.id, function (error) {
-            if (error && error.reason !== BoxError.NOT_FOUND) return callback(error);
-
-            clientdb.delByAppIdAndType(appId, type, function (error) {
-                if (error) return callback(error);
-
-                callback(null);
-            });
-        });
-    });
-}
-
-function addTokenByUserId(clientId, userId, expiresAt, options, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof expiresAt, 'number');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const name = options.name || '';
-    let error = validateTokenName(name);
-    if (error) return callback(error);
-
-    get(clientId, function (error, result) {
-        if (error) return callback(error);
-
-        users.get(userId, function (error, user) {
-            if (error) return callback(error);
-
-            accesscontrol.scopesForUser(user, function (error, userScopes) {
-                if (error) return callback(error);
-
-                const scope = accesscontrol.canonicalScopeString(result.scope);
-                const authorizedScopes = accesscontrol.intersectScopes(userScopes, scope.split(','));
-
-                const token = {
-                    id: 'tid-' + uuid.v4(),
-                    accessToken: hat(8 * 32),
-                    identifier: userId,
-                    clientId: result.id,
-                    expires: expiresAt,
-                    scope: authorizedScopes.join(','),
-                    name: name
-                };
-
-                tokendb.add(token, function (error) {
-                    if (error) return callback(error);
-
-                    callback(null, {
-                        accessToken: token.accessToken,
-                        tokenScopes: authorizedScopes,
-                        identifier: userId,
-                        clientId: result.id,
-                        expires: expiresAt
-                    });
-                });
-            });
-        });
-    });
-}
-
-function issueDeveloperToken(userObject, auditSource, callback) {
-    assert.strictEqual(typeof userObject, 'object');
-    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const expiresAt = Date.now() + (30 * 24 * 60 * 60 * 1000); // cli tokens are valid for a month
-
-    addTokenByUserId(exports.ID_CLI, userObject.id, expiresAt, {}, function (error, result) {
-        if (error) return callback(error);
-
-        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: userObject.id, user: users.removePrivateFields(userObject) });
-
-        callback(null, result);
-    });
-}
-
-function delToken(clientId, tokenId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof tokenId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    get(clientId, function (error) {
-        if (error) return callback(error);
-
-        tokendb.del(tokenId, function (error) {
-            if (error) return callback(error);
-
-            callback(null);
-        });
-    });
-}
-
-function addDefaultClients(origin, callback) {
-    assert.strictEqual(typeof origin, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Adding default clients');
-
-    // The domain might have changed, therefor we have to update the record
-    // id, appId, type, clientSecret, redirectURI, scope
-    async.series([
-        clientdb.upsert.bind(null, exports.ID_WEBADMIN, 'Settings', 'built-in', 'secret-webadmin', origin, '*'),
-        clientdb.upsert.bind(null, exports.ID_SDK, 'SDK', 'built-in', 'secret-sdk', origin, '*'),
-        clientdb.upsert.bind(null, exports.ID_CLI, 'Cloudron Tool', 'built-in', 'secret-cli', origin, '*')
-    ], callback);
-}
-
-function removeTokenPrivateFields(token) {
-    return _.pick(token, 'id', 'identifier', 'clientId', 'scope', 'expires', 'name');
-}

src/cloudron.js

@@ -22,22 +22,20 @@ exports = module.exports = {
 };
 
 var apps = require('./apps.js'),
+    appstore = require('./appstore.js'),
     assert = require('assert'),
     async = require('async'),
     auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
-    clients = require('./clients.js'),
     constants = require('./constants.js'),
     cron = require('./cron.js'),
     debug = require('debug')('box:cloudron'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
-    custom = require('./custom.js'),
     fs = require('fs'),
     mail = require('./mail.js'),
     notifications = require('./notifications.js'),
-    os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
     platform = require('./platform.js'),
@@ -107,6 +105,12 @@ function runStartupTasks() {
     // configure nginx to be reachable by IP
     reverseProxy.writeDefaultConfig(NOOP_CALLBACK);
 
+    // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return console.error('Failed to read backup config.', error);
+        backups.configureCollectd(backupConfig, NOOP_CALLBACK);
+    });
+
     // always generate webadmin config since we have no versioning mechanism for the ejs
     if (settings.adminDomain()) reverseProxy.writeAdminConfig(settings.adminDomain(), NOOP_CALLBACK);
 
@@ -134,16 +138,20 @@ function getConfig(callback) {
             mailFqdn: settings.mailFqdn(),
             version: constants.VERSION,
             isDemo: settings.isDemo(),
-            memory: os.totalmem(),
             provider: settings.provider(),
             cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
-            uiSpec: custom.uiSpec()
+            footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
+            features: appstore.getFeatures()
         });
     });
 }
 
 function reboot(callback) {
-    shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
+    notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '', function (error) {
+        if (error) console.error('Failed to clear reboot notification.', error);
+
+        shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
+    });
 }
 
 function isRebootRequired(callback) {
@@ -154,14 +162,14 @@ function isRebootRequired(callback) {
 }
 
 // called from cron.js
-function runSystemChecks() {
+function runSystemChecks(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     async.parallel([
         checkBackupConfiguration,
         checkMailStatus,
         checkRebootRequired
-    ], function (error) {
-        debug('runSystemChecks: done', error);
-    });
+    ], callback);
 }
 
 function checkBackupConfiguration(callback) {
@@ -196,7 +204,7 @@ function checkRebootRequired(callback) {
     isRebootRequired(function (error, rebootRequired) {
         if (error) return callback(error);
 
-        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish security updates, a [reboot](/#/system) is necessary.' : '', callback);
+        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '', callback);
     });
 }
 
@@ -253,6 +261,8 @@ function prepareDashboardDomain(domain, auditSource, callback) {
 
     debug(`prepareDashboardDomain: ${domain}`);
 
+    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+
     domains.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
@@ -291,10 +301,7 @@ function setDashboardDomain(domain, auditSource, callback) {
 
             const fqdn = domains.fqdn(constants.ADMIN_LOCATION, domainObject);
 
-            async.series([
-                (done) => settings.setAdmin(domain, fqdn, done),
-                (done) => clients.addDefaultClients(settings.adminOrigin(), done)
-            ], function (error) {
+            settings.setAdmin(domain, fqdn, function (error) {
                 if (error) return callback(error);
 
                 eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain: domain, fqdn: fqdn });
@@ -313,6 +320,8 @@ function setDashboardAndMailDomain(domain, auditSource, callback) {
 
     debug(`setDashboardAndMailDomain: ${domain}`);
 
+    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+
     setDashboardDomain(domain, auditSource, function (error) {
         if (error) return callback(error);
 

src/collectd.js

@@ -0,0 +1,55 @@
+'use strict';
+
+exports = module.exports = {
+    addProfile,
+    removeProfile
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    debug = require('debug')('collectd'),
+    fs = require('fs'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js');
+
+const CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh');
+
+function addProfile(name, profile, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof profile, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const configFilePath = path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`);
+
+    // skip restarting collectd if the profile already exists with the same contents
+    const currentProfile = safe.fs.readFileSync(configFilePath, 'utf8') || '';
+    if (currentProfile === profile) return callback(null);
+
+    fs.writeFile(configFilePath, profile, function (error) {
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
+
+        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}, function (error) {
+            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
+
+            callback(null);
+        });
+    });
+
+}
+
+function removeProfile(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`), function (error) {
+        if (error && error.code !== 'ENOENT') debug('Error removing collectd profile', error);
+
+        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}, function (error) {
+            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
+
+            callback(null);
+        });
+    });
+}

src/collectd/cloudron-backup.ejs

@@ -0,0 +1,9 @@
+<Plugin python>
+  <Module du>
+    <Path>
+        Instance "cloudron-backup"
+        Dir "<%= backupDir %>"
+    </Path>
+  </Module>
+</Plugin>
+

src/constants.js

@@ -47,6 +47,10 @@ exports = module.exports = {
     CLOUDRON: CLOUDRON,
     TEST: TEST,
 
+    SUPPORT_EMAIL: 'support@cloudron.io',
+
+    FOOTER: '&copy; 2020  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
+
     VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '4.2.0-test'
 };
 

src/cron.js

@@ -12,6 +12,7 @@ var appHealthMonitor = require('./apphealthmonitor.js'),
     apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
+    async = require('async'),
     auditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     cloudron = require('./cloudron.js'),
@@ -59,150 +60,141 @@ var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 function startJobs(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    var randomHourMinute = Math.floor(60*Math.random());
+    const randomMinute = Math.floor(60*Math.random());
     gJobs.alive = new CronJob({
-        cronTime: '00 ' + randomHourMinute + ' * * * *', // every hour on a random minute
+        cronTime: '00 ' + randomMinute + ' * * * *', // every hour on a random minute
         onTick: appstore.sendAliveStatus,
         start: true
     });
 
+    gJobs.systemChecks = new CronJob({
+        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
+        onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.diskSpaceChecker = new CronJob({
+        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
+        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.boxUpdateCheckerJob = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkBoxUpdates(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.appUpdateChecker = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkAppUpdates(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.cleanupTokens = new CronJob({
+        cronTime: '00 */30 * * * *', // every 30 minutes
+        onTick: janitor.cleanupTokens,
+        start: true
+    });
+
+    gJobs.cleanupBackups = new CronJob({
+        cronTime: '00 45 1,3,5,23 * * *', // every 6 hours. try not to overlap with ensureBackup job
+        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.cleanupEventlog = new CronJob({
+        cronTime: '00 */30 * * * *', // every 30 minutes
+        onTick: eventlog.cleanup,
+        start: true
+    });
+
+    gJobs.dockerVolumeCleaner = new CronJob({
+        cronTime: '00 00 */12 * * *', // every 12 hours
+        onTick: janitor.cleanupDockerVolumes,
+        start: true
+    });
+
+    gJobs.schedulerSync = new CronJob({
+        cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
+        onTick: scheduler.sync,
+        start: true
+    });
+
+    gJobs.certificateRenew = new CronJob({
+        cronTime: '00 00 */12 * * *', // every 12 hours
+        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.appHealthMonitor = new CronJob({
+        cronTime: '*/10 * * * * *', // every 10 seconds
+        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
+        start: true
+    });
+
     settings.getAll(function (error, allSettings) {
         if (error) return callback(error);
 
-        recreateJobs(allSettings[settings.TIME_ZONE_KEY]);
-        appAutoupdatePatternChanged(allSettings[settings.APP_AUTOUPDATE_PATTERN_KEY]);
-        boxAutoupdatePatternChanged(allSettings[settings.BOX_AUTOUPDATE_PATTERN_KEY]);
+        const tz = allSettings[settings.TIME_ZONE_KEY];
+        backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
+        appAutoupdatePatternChanged(allSettings[settings.APP_AUTOUPDATE_PATTERN_KEY], tz);
+        boxAutoupdatePatternChanged(allSettings[settings.BOX_AUTOUPDATE_PATTERN_KEY], tz);
         dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
 
         callback();
     });
 }
 
+// eslint-disable-next-line no-unused-vars
 function handleSettingsChanged(key, value) {
     assert.strictEqual(typeof key, 'string');
-
     // value is a variant
+
     switch (key) {
-    case settings.TIME_ZONE_KEY: recreateJobs(value); break;
-    case settings.APP_AUTOUPDATE_PATTERN_KEY: appAutoupdatePatternChanged(value); break;
-    case settings.BOX_AUTOUPDATE_PATTERN_KEY: boxAutoupdatePatternChanged(value); break;
-    case settings.DYNAMIC_DNS_KEY: dynamicDnsChanged(value); break;
-    default: break;
+    case settings.TIME_ZONE_KEY:
+    case settings.BACKUP_CONFIG_KEY:
+    case settings.APP_AUTOUPDATE_PATTERN_KEY:
+    case settings.BOX_AUTOUPDATE_PATTERN_KEY:
+    case settings.DYNAMIC_DNS_KEY:
+        debug('handleSettingsChanged: recreating all jobs');
+        async.series([
+            stopJobs,
+            startJobs
+        ], NOOP_CALLBACK);
+        break;
+    default:
+        break;
     }
 }
 
-function recreateJobs(tz) {
+function backupConfigChanged(value, tz) {
+    assert.strictEqual(typeof value, 'object');
     assert.strictEqual(typeof tz, 'string');
 
-    debug('Creating jobs with timezone %s', tz);
+    debug(`backupConfigChanged: interval ${value.intervalSecs} (${tz})`);
 
     if (gJobs.backup) gJobs.backup.stop();
+    let pattern;
+    if (value.intervalSecs <= 6 * 60 * 60) {
+        pattern = '00 00 1,7,13,19 * * *'; // no option but to backup in the middle of the day
+    } else {
+        pattern = '00 00 1,3,5,23 * * *'; // avoid middle of the day backups
+    }
+
     gJobs.backup = new CronJob({
-        cronTime: '00 00 */6 * * *', // check every 6 hours
+        cronTime: pattern,
         onTick: backups.ensureBackup.bind(null, auditSource.CRON, NOOP_CALLBACK),
         start: true,
         timeZone: tz
     });
-
-    if (gJobs.systemChecks) gJobs.systemChecks.stop();
-    gJobs.systemChecks = new CronJob({
-        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: cloudron.runSystemChecks,
-        start: true,
-        runOnInit: true, // run system check immediately
-        timeZone: tz
-    });
-
-    if (gJobs.diskSpaceChecker) gJobs.diskSpaceChecker.stop();
-    gJobs.diskSpaceChecker = new CronJob({
-        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
-        start: true,
-        runOnInit: true, // run system check immediately
-        timeZone: tz
-    });
-
-    // randomized pattern per cloudron every hour
-    var randomMinute = Math.floor(60*Math.random());
-
-    if (gJobs.boxUpdateCheckerJob) gJobs.boxUpdateCheckerJob.stop();
-    gJobs.boxUpdateCheckerJob = new CronJob({
-        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
-        onTick: () => updateChecker.checkBoxUpdates(NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.appUpdateChecker) gJobs.appUpdateChecker.stop();
-    gJobs.appUpdateChecker = new CronJob({
-        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
-        onTick: () => updateChecker.checkAppUpdates(NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.cleanupTokens) gJobs.cleanupTokens.stop();
-    gJobs.cleanupTokens = new CronJob({
-        cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: janitor.cleanupTokens,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.cleanupBackups) gJobs.cleanupBackups.stop();
-    gJobs.cleanupBackups = new CronJob({
-        cronTime: '00 45 */6 * * *', // every 6 hours. try not to overlap with ensureBackup job
-        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.cleanupEventlog) gJobs.cleanupEventlog.stop();
-    gJobs.cleanupEventlog = new CronJob({
-        cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: eventlog.cleanup,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.dockerVolumeCleaner) gJobs.dockerVolumeCleaner.stop();
-    gJobs.dockerVolumeCleaner = new CronJob({
-        cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: janitor.cleanupDockerVolumes,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.schedulerSync) gJobs.schedulerSync.stop();
-    gJobs.schedulerSync = new CronJob({
-        cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
-        onTick: scheduler.sync,
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.certificateRenew) gJobs.certificateRenew.stop();
-    gJobs.certificateRenew = new CronJob({
-        cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
-
-    if (gJobs.appHealthMonitor) gJobs.appHealthMonitor.stop();
-    gJobs.appHealthMonitor = new CronJob({
-        cronTime: '*/10 * * * * *', // every 10 seconds
-        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
-        start: true,
-        timeZone: tz
-    });
 }
 
-function boxAutoupdatePatternChanged(pattern) {
+function boxAutoupdatePatternChanged(pattern, tz) {
     assert.strictEqual(typeof pattern, 'string');
-    assert(gJobs.boxUpdateCheckerJob);
+    assert.strictEqual(typeof tz, 'string');
 
-    debug('Box auto update pattern changed to %s', pattern);
+    debug(`boxAutoupdatePatternChanged: pattern - ${pattern} (${tz})`);
 
     if (gJobs.boxAutoUpdater) gJobs.boxAutoUpdater.stop();
 
@@ -220,15 +212,15 @@ function boxAutoupdatePatternChanged(pattern) {
             }
         },
         start: true,
-        timeZone: gJobs.boxUpdateCheckerJob.cronTime.zone // hack
+        timeZone: tz
     });
 }
 
-function appAutoupdatePatternChanged(pattern) {
+function appAutoupdatePatternChanged(pattern, tz) {
     assert.strictEqual(typeof pattern, 'string');
-    assert(gJobs.boxUpdateCheckerJob);
+    assert.strictEqual(typeof tz, 'string');
 
-    debug('Apps auto update pattern changed to %s', pattern);
+    debug(`appAutoupdatePatternChanged: pattern ${pattern} (${tz})`);
 
     if (gJobs.appAutoUpdater) gJobs.appAutoUpdater.stop();
 
@@ -246,13 +238,12 @@ function appAutoupdatePatternChanged(pattern) {
             }
         },
         start: true,
-        timeZone: gJobs.boxUpdateCheckerJob.cronTime.zone // hack
+        timeZone: tz
     });
 }
 
 function dynamicDnsChanged(enabled) {
     assert.strictEqual(typeof enabled, 'boolean');
-    assert(gJobs.boxUpdateCheckerJob);
 
     debug('Dynamic DNS setting changed to %s', enabled);
 
@@ -260,8 +251,7 @@ function dynamicDnsChanged(enabled) {
         gJobs.dynamicDns = new CronJob({
             cronTime: '5 * * * * *',    // we only update the records if the ip has changed.
             onTick: dyndns.sync.bind(null, auditSource.CRON, NOOP_CALLBACK),
-            start: true,
-            timeZone: gJobs.boxUpdateCheckerJob.cronTime.zone // hack
+            start: true
         });
     } else {
         if (gJobs.dynamicDns) gJobs.dynamicDns.stop();

src/custom.js

@@ -1,66 +0,0 @@
-'use strict';
-
-let debug = require('debug')('box:custom'),
-    lodash = require('lodash'),
-    paths = require('./paths.js'),
-    safe = require('safetydance'),
-    yaml = require('js-yaml');
-
-exports = module.exports = {
-    uiSpec: uiSpec,
-    spec: spec
-};
-
-const DEFAULT_SPEC = {
-    appstore: {
-        blacklist: [],
-        whitelist: null // null imples, not set. this is an object and not an array
-    },
-    backups: {
-        configurable: true
-    },
-    domains: {
-        dynamicDns: true,
-        changeDashboardDomain: true
-    },
-    subscription: {
-        configurable: true
-    },
-    support: {
-        email: 'support@cloudron.io',
-        remoteSupport: true,
-        ticketFormBody:
-            'Use this form to open support tickets. You can also write directly to [support@cloudron.io](mailto:support@cloudron.io).\n\n'
-            + '* [Knowledge Base & App Docs](https://cloudron.io/documentation/apps/?support_view)\n'
-            + '* [Custom App Packaging & API](https://cloudron.io/developer/packaging/?support_view)\n'
-            + '* [Forum](https://forum.cloudron.io/)\n\n',
-        submitTickets: true
-    },
-    alerts: {
-        email: '',
-        notifyCloudronAdmins: true
-    },
-    footer: {
-        body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
-    }
-};
-
-const gSpec = (function () {
-    try {
-        if (!safe.fs.existsSync(paths.CUSTOM_FILE)) return DEFAULT_SPEC;
-        const c = yaml.safeLoad(safe.fs.readFileSync(paths.CUSTOM_FILE, 'utf8'));
-        return lodash.merge({}, DEFAULT_SPEC, c);
-    } catch (e) {
-        debug(`Error loading features file from ${paths.CUSTOM_FILE} : ${e.message}`);
-        return DEFAULT_SPEC;
-    }
-})();
-
-// flags sent to the UI. this is separate because we have values that are secret to the backend
-function uiSpec() {
-    return gSpec;
-}
-
-function spec() {
-    return gSpec;
-}

src/database.js

@@ -104,10 +104,7 @@ function clear(callback) {
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name,
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name);
 
-    async.series([
-        child_process.exec.bind(null, cmd),
-        require('./clients.js').addDefaultClients.bind(null, 'https://admin-localhost')
-    ], callback);
+    child_process.exec(cmd, callback);
 }
 
 function beginTransaction(callback) {
@@ -204,7 +201,11 @@ function exportToFile(file, callback) {
     assert.strictEqual(typeof file, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
+    // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
+    // this option must not be set in production cloudrons which still use the old mysqldump
+    const disableColStats = (constants.TEST && process.env.DESKTOP_SESSION !== 'ubuntu') ? '--column-statistics=0' : '';
+
+    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${disableColStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
 
     child_process.exec(cmd, callback);
 }

src/dns/cloudflare.js

@@ -48,15 +48,29 @@ function translateRequestError(result, callback) {
     callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
 }
 
+function createRequest(method, url, dnsConfig) {
+    assert.strictEqual(typeof method, 'string');
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof dnsConfig, 'object');
+
+    let request = superagent(method, url)
+        .timeout(30 * 1000);
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        request.set('X-Auth-Key', dnsConfig.token).set('X-Auth-Email', dnsConfig.email);
+    } else {
+        request.set('Authorization', 'Bearer ' + dnsConfig.token);
+    }
+
+    return request;
+}
+
 function getZoneByName(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active')
-        .set('X-Auth-Key', dnsConfig.token)
-        .set('X-Auth-Email', dnsConfig.email)
-        .timeout(30 * 1000)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active', dnsConfig)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -74,11 +88,8 @@ function getDnsRecords(dnsConfig, zoneId, fqdn, type, callback) {
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-        .set('X-Auth-Key',dnsConfig.token)
-        .set('X-Auth-Email',dnsConfig.email)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
         .query({ type: type, name: fqdn })
-        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -132,11 +143,8 @@ function upsert(domainObject, location, type, values, callback) {
                 if (i >= dnsRecords.length) { // create a new record
                     debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: false`);
 
-                    superagent.post(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('POST', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             if (error && !error.response) return iteratorCallback(error);
                             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, iteratorCallback);
@@ -148,11 +156,8 @@ function upsert(domainObject, location, type, values, callback) {
 
                     debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
 
-                    superagent.put(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id)
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('PUT', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id, dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             ++i; // increment, as we have consumed the record
 
@@ -217,10 +222,7 @@ function del(domainObject, location, type, values, callback) {
             if (tmp.length === 0) return callback(null);
 
             async.eachSeries(tmp, function (record, callback) {
-                superagent.del(CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id)
-                    .set('X-Auth-Key', dnsConfig.token)
-                    .set('X-Auth-Email', dnsConfig.email)
-                    .timeout(30 * 1000)
+                createRequest('DELETE', CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id, dnsConfig)
                     .end(function (error, result) {
                         if (error && !error.response) return callback(error);
                         if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -277,14 +279,20 @@ function verifyDnsConfig(domainObject, callback) {
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
+    // token can be api token or global api key
     if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
-    if (!dnsConfig.email || typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    if (dnsConfig.tokenType !== 'GlobalApiKey' && dnsConfig.tokenType !== 'ApiToken')  return callback(new BoxError(BoxError.BAD_FIELD, 'tokenType is required', { field: 'tokenType' }));
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        if ('email' in dnsConfig && typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    }
 
     const ip = '127.0.0.1';
 
     var credentials = {
         token: dnsConfig.token,
-        email: dnsConfig.email
+        tokenType: dnsConfig.tokenType,
+        email: dnsConfig.email || null
     };
 
     if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here

src/docker.js

@@ -14,6 +14,7 @@ exports = module.exports = {
     downloadImage: downloadImage,
     createContainer: createContainer,
     startContainer: startContainer,
+    restartContainer: restartContainer,
     stopContainer: stopContainer,
     stopContainerByName: stopContainer,
     stopContainers: stopContainers,
@@ -107,9 +108,10 @@ function ping(callback) {
 
     connection.ping(function (error, result) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
-        if (result !== 'OK') return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon'));
+        if (Buffer.isBuffer(result) && result.toString('utf8') === 'OK') return callback(null); // sometimes it returns buffer
+        if (result === 'OK') return callback(null);
 
-        callback(null);
+        callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to ping the docker daemon'));
     });
 }
 
@@ -141,7 +143,8 @@ function pullImage(manifest, callback) {
         debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${authConfig ? 'yes' : 'no'}`);
 
         gConnection.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
-            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to pull image. Please check the network or if the image needs authentication. statusCode: ' + error.statusCode));
+            if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${manifest.dockerImage}. message: ${error.message} statusCode: ${error.statusCode}`));
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`));
 
             // https://github.com/dotcloud/docker/issues/1074 says each status message
             // is emitted as a chunk
@@ -178,14 +181,10 @@ function downloadImage(manifest, callback) {
 
     var attempt = 1;
 
-    async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
+    async.retry({ times: 10, interval: 5000, errorFilter: e => e.reason !== BoxError.NOT_FOUND }, function (retryCallback) {
         debug('Downloading image %s. attempt: %s', manifest.dockerImage, attempt++);
 
-        pullImage(manifest, function (error) {
-            if (error) console.error(error);
-
-            retryCallback(error);
-        });
+        pullImage(manifest, retryCallback);
     }, callback);
 }
 
@@ -295,7 +294,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                     'Name': isAppContainer ? 'unless-stopped' : 'no',
                     'MaximumRetryCount': 0
                 },
-                CpuShares: 512, // relative to 1024 for system processes
+                CpuShares: app.cpuShares,
                 VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ],
                 NetworkMode: 'cloudron', // user defined bridge network
                 Dns: ['172.18.0.1'], // use internal dns
@@ -344,7 +343,23 @@ function startContainer(containerId, callback) {
     container.start(function (error) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
         if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
-        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); // 304 means already started
+
+        return callback(null);
+    });
+}
+
+function restartContainer(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = gConnection.getContainer(containerId);
+    debug('Restarting container %s', containerId);
+
+    container.restart(function (error) {
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+        if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
+        if (error && error.statusCode !== 204) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         return callback(null);
     });
@@ -431,7 +446,7 @@ function stopContainers(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug('stopping containers of %s', appId);
+    debug('Stopping containers of %s', appId);
 
     gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

src/domaindb.js

@@ -16,7 +16,7 @@ var assert = require('assert'),
     database = require('./database.js'),
     safe = require('safetydance');
 
-var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'locked' ].join(',');
+var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson' ].join(',');
 
 function postProcess(data) {
     data.config = safe.JSON.parse(data.configJson);
@@ -24,8 +24,6 @@ function postProcess(data) {
     delete data.configJson;
     delete data.tlsConfigJson;
 
-    data.locked = !!data.locked; // make it bool
-
     return data;
 }
 
@@ -103,7 +101,7 @@ function del(domain, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     database.query('DELETE FROM domains WHERE domain=?', [ domain ], function (error, result) {
-        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') return callback(new BoxError(BoxError.CONFLICT));
+        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') return callback(new BoxError(BoxError.CONFLICT, error.message));
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
         if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
 

src/domains.js

@@ -86,9 +86,9 @@ function verifyDnsConfig(dnsConfig, domain, zoneName, provider, callback) {
 
     const domainObject = { config: dnsConfig, domain: domain, zoneName: zoneName };
     api(provider).verifyDnsConfig(domainObject, function (error, result) {
-        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, 'Incorrect configuration. Access denied'));
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, 'Zone not found'));
-        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, 'Configuration error: ' + error.message));
+        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`));
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`));
+        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`));
         if (error) return callback(error);
 
         result.hyphenatedSubdomains = !!dnsConfig.hyphenatedSubdomains;
@@ -253,6 +253,8 @@ function update(domain, data, auditSource, callback) {
 
     let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
 
+    if (settings.isDemo() && (domain === settings.adminDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+
     domaindb.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
@@ -307,6 +309,13 @@ function del(domain, auditSource, callback) {
     if (domain === settings.adminDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain'));
 
     domaindb.del(domain, function (error) {
+        if (error && error.reason === BoxError.CONFLICT) {
+            if (error.message.indexOf('apps_mailDomain_constraint') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use as the mailbox of an app. Check the Email section of each app.'));
+            if (error.message.indexOf('subdomains') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by an app. Move the app first to a different location.'));
+            if (error.message.indexOf('mail') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by a mailbox. Delete mailboxes first in the Email view.'));
+
+            // intentional fall through
+        }
         if (error) return callback(error);
 
         eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
@@ -440,13 +449,13 @@ function waitForDnsRecord(location, domain, type, value, options, callback) {
 
 // removes all fields that are strictly private and should never be returned by API calls
 function removePrivateFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate', 'locked');
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate');
     return api(result.provider).removePrivateFields(result);
 }
 
 // removes all fields that are not accessible by a normal user
 function removeRestrictedFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'locked');
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider');
 
     // always ensure config object
     result.config = { hyphenatedSubdomains: !!domain.config.hyphenatedSubdomains };

src/eventlog.js

@@ -40,8 +40,10 @@ exports = module.exports = {
     ACTION_MAIL_DISABLED: 'mail.disabled',
     ACTION_MAIL_MAILBOX_ADD: 'mail.box.add',
     ACTION_MAIL_MAILBOX_REMOVE: 'mail.box.remove',
+    ACTION_MAIL_MAILBOX_UPDATE: 'mail.box.update',
     ACTION_MAIL_LIST_ADD: 'mail.list.add',
     ACTION_MAIL_LIST_REMOVE: 'mail.list.remove',
+    ACTION_MAIL_LIST_UPDATE: 'mail.list.update',
 
     ACTION_PROVISION: 'cloudron.provision',
     ACTION_RESTORE: 'cloudron.restore', // unused
@@ -57,6 +59,9 @@ exports = module.exports = {
 
     ACTION_DYNDNS_UPDATE: 'dyndns.update',
 
+    ACTION_SUPPORT_TICKET: 'support.ticket',
+    ACTION_SUPPORT_SSH: 'support.ssh',
+
     ACTION_PROCESS_CRASH: 'system.crash'
 };
 
@@ -82,11 +87,9 @@ function add(action, source, data, callback) {
     api(uuid.v4(), action, source, data, function (error, id) {
         if (error) return callback(error);
 
-        notifications.onEvent(id, action, source, data, function (error) {
-            if (error) return callback(error);
+        callback(null, { id: id });
 
-            callback(null, { id: id });
-        });
+        notifications.onEvent(id, action, source, data, NOOP_CALLBACK);
     });
 }
 

src/externalldap.js

@@ -301,7 +301,7 @@ function sync(progressCallback, callback) {
                     } else if (result.email !== user.email || result.displayName !== user.displayName) {
                         debug(`[updating user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
 
-                        users.update(result.id, { email: user.email, fallbackEmail: user.email, displayName: user.displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
+                        users.update(result, { email: user.email, fallbackEmail: user.email, displayName: user.displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
                             if (error) debug('Failed to update user', user, error);
 
                             iteratorCallback();

src/infra_version.js

@@ -15,11 +15,11 @@ exports = module.exports = {
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.0.2@sha256:a28320f313785816be60e3f865e09065504170a3d20ed37de675c719b32b01eb' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.1.0@sha256:eee0dfd3829d563f2063084bc0d7c8802c4bdd6e233159c6226a17ff7a9a3503' },
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:2.0.2@sha256:6dcee0731dfb9b013ed94d56205eee219040ee806c7e251db3b3886eaa4947ff' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:2.1.0@sha256:6d1bf221cfe6124957e2c58b57c0a47214353496009296acb16adf56df1da9d5' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.1.0@sha256:f2cda21bd15c21bbf44432df412525369ef831a2d53860b5c5b1675e6f384de2' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.5.0@sha256:086ae1c9433d90a820326aa43914a2afe94ad707074ef2bc05a7ef4798e83655' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.6.0@sha256:c881d513ddbdea73f29d92d392a7435039314b13e5e3659e9e85f6b26476e365' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.2.0@sha256:fc9ca69d16e6ebdbd98ed53143d4a0d2212eef60cb638dc71219234e6f427a2c' },
         'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:0.1.0@sha256:e177c5bf5f38c84ce1dea35649c22a1b05f96eec67a54a812c5a35e585670f0f' }
     }

src/janitor.js

@@ -2,7 +2,6 @@
 
 var assert = require('assert'),
     async = require('async'),
-    authcodedb = require('./authcodedb.js'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:janitor'),
     Docker = require('dockerode'),
@@ -39,26 +38,13 @@ function cleanupExpiredTokens(callback) {
     });
 }
 
-function cleanupExpiredAuthCodes(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    authcodedb.delExpired(function (error, result) {
-        if (error) return callback(error);
-
-        debug('Cleaned up %s expired authcodes.', result);
-
-        callback(null);
-    });
-}
-
 function cleanupTokens(callback) {
     assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
 
     debug('Cleaning up expired tokens');
 
     async.series([
-        ignoreError(cleanupExpiredTokens),
-        ignoreError(cleanupExpiredAuthCodes)
+        ignoreError(cleanupExpiredTokens)
     ], callback);
 }
 

src/ldap.js

@@ -126,16 +126,16 @@ function userSearch(req, res, next) {
         var results = [];
 
         // send user objects
-        result.forEach(function (entry) {
+        result.forEach(function (user) {
             // skip entries with empty username. Some apps like owncloud can't deal with this
-            if (!entry.username) return;
+            if (!user.username) return;
 
-            var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+            var dn = ldap.parseDN('cn=' + user.id + ',ou=users,dc=cloudron');
 
             var groups = [ GROUP_USERS_DN ];
-            if (entry.admin) groups.push(GROUP_ADMINS_DN);
+            if (users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) groups.push(GROUP_ADMINS_DN);
 
-            var displayName = entry.displayName || entry.username || ''; // displayName can be empty and username can be null
+            var displayName = user.displayName || user.username || ''; // displayName can be empty and username can be null
             var nameParts = displayName.split(' ');
             var firstName = nameParts[0];
             var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
@@ -143,17 +143,18 @@ function userSearch(req, res, next) {
             var obj = {
                 dn: dn.toString(),
                 attributes: {
-                    objectclass: ['user'],
+                    objectclass: ['user', 'inetorgperson', 'person' ],
                     objectcategory: 'person',
-                    cn: entry.id,
-                    uid: entry.id,
-                    mail: entry.email,
-                    mailAlternateAddress: entry.fallbackEmail,
+                    cn: user.id,
+                    uid: user.id,
+                    entryuuid: user.id, // to support OpenLDAP clients
+                    mail: user.email,
+                    mailAlternateAddress: user.fallbackEmail,
                     displayname: displayName,
                     givenName: firstName,
-                    username: entry.username,
-                    samaccountname: entry.username,      // to support ActiveDirectory clients
-                    isadmin: entry.admin,
+                    username: user.username,
+                    samaccountname: user.username,      // to support ActiveDirectory clients
+                    isadmin: users.compareRoles(user.role, users.ROLE_ADMIN) >= 0,
                     memberof: groups
                 }
             };
@@ -193,7 +194,7 @@ function groupSearch(req, res, next) {
 
         groups.forEach(function (group) {
             var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-            var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+            var members = group.admin ? result.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : result;
 
             var obj = {
                 dn: dn.toString(),
@@ -241,8 +242,8 @@ function groupAdminsCompare(req, res, next) {
 
         // we only support memberuid here, if we add new group attributes later add them here
         if (req.attribute === 'memberuid') {
-            var found = result.find(function (u) { return u.id === req.value; });
-            if (found && found.admin) return res.end(true);
+            var user = result.find(function (u) { return u.id === req.value; });
+            if (user && users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) return res.end(true);
         }
 
         res.end(false);
@@ -283,10 +284,11 @@ function mailboxSearch(req, res, next) {
                 res.end();
             }
         });
-    } else if (req.dn.rdns[0].attrs.domain) {
+    } else if (req.dn.rdns[0].attrs.domain) { // legacy ldap mailbox search for old sogo
         var domain = req.dn.rdns[0].attrs.domain.value.toLowerCase();
 
         mailboxdb.listMailboxes(domain, 1, 1000, function (error, result) {
+
             if (error) return next(new ldap.OperationsError(error.toString()));
 
             var results = [];
@@ -309,7 +311,6 @@ function mailboxSearch(req, res, next) {
                 // ensure all filter values are also lowercase
                 var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
                 if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
                 if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
                     results.push(obj);
                 }
@@ -317,8 +318,55 @@ function mailboxSearch(req, res, next) {
 
             finalSend(results, req, res, next);
         });
-    } else {
-        return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    } else { // new sogo
+        mailboxdb.listAllMailboxes(1, 1000, function (error, mailboxes) {
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            var results = [];
+
+            // send mailbox objects
+            async.eachSeries(mailboxes, function (mailbox, callback) {
+                var dn = ldap.parseDN(`cn=${mailbox.name}@${mailbox.domain},ou=mailboxes,dc=cloudron`);
+
+                users.get(mailbox.ownerId, function (error, userObject) {
+                    if (error) return callback(); // skip mailboxes with unknown owner
+
+                    var obj = {
+                        dn: dn.toString(),
+                        attributes: {
+                            objectclass: ['mailbox'],
+                            objectcategory: 'mailbox',
+                            displayname: userObject.displayName,
+                            cn: `${mailbox.name}@${mailbox.domain}`,
+                            uid: `${mailbox.name}@${mailbox.domain}`,
+                            mail: `${mailbox.name}@${mailbox.domain}`
+                        }
+                    };
+
+                    mailboxdb.getAliasesForName(mailbox.name, mailbox.domain, function (error, aliases) {
+                        if (error) return callback(error);
+
+                        aliases.forEach(function (a, idx) {
+                            obj.attributes['mail' + idx] = `${a}@${mailbox.domain}`;
+                        });
+
+                        // ensure all filter values are also lowercase
+                        var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+                        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+                        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                            results.push(obj);
+                        }
+
+                        callback();
+                    });
+                });
+            }, function (error) {
+                if (error) return next(new ldap.OperationsError(error.toString()));
+
+                finalSend(results, req, res, next);
+            });
+        });
     }
 }
 
@@ -419,7 +467,7 @@ function authenticateUser(req, res, next) {
         api = users.verifyWithUsername;
     }
 
-    api(commonName, req.credentials || '', function (error, user) {
+    api(commonName, req.credentials || '', req.app.id, function (error, user) {
         if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.message));
@@ -465,7 +513,7 @@ function authenticateUserMailbox(req, res, next) {
             if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
             if (error) return next(new ldap.OperationsError(error.message));
 
-            users.verify(mailbox.ownerId, req.credentials || '', function (error, result) {
+            users.verify(mailbox.ownerId, req.credentials || '', users.AP_MAIL, function (error, result) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
@@ -487,7 +535,7 @@ function authenticateSftp(req, res, next) {
     if (parts.length !== 2) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
     // actual user bind
-    users.verifyWithUsername(parts[0], req.credentials, function (error) {
+    users.verifyWithUsername(parts[0], req.credentials, users.AP_SFTP, function (error) {
         if (error) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
 
         debug('sftp auth: success');
@@ -576,7 +624,7 @@ function authenticateMailAddon(req, res, next) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
 
-                users.verify(mailbox.ownerId, req.credentials || '', function (error, result) {
+                users.verify(mailbox.ownerId, req.credentials || '', users.AP_MAIL, function (error, result) {
                     if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                     if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                     if (error) return next(new ldap.OperationsError(error.message));

src/mail.js

@@ -199,7 +199,7 @@ function checkDkim(mailDomain, callback) {
     };
 
     var dkimKey = readDkimPublicKeySync(domain);
-    if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, `Failed to read dkim public key of ${domain}`));
+    if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, `Failed to read dkim public key of ${domain}`), dkim);
 
     dkim.expected = 'v=DKIM1; t=s; p=' + dkimKey;
 
@@ -456,7 +456,7 @@ function getStatus(domain, callback) {
 
     // ensure we always have a valid toplevel properties for the api
     var results = {
-        dns: {}, // { mx/dmar/dkim/spf/ptr: { expected, value, name, domain, type } }
+        dns: {}, // { mx/dmarc/dkim/spf/ptr: { expected, value, name, domain, type, status } }
         rbl: {}, // { status, ip, servers: [{name,site,dns}]} optional. only for cloudron-smtp
         relay: {} // { status, value } always checked
     };
@@ -466,7 +466,7 @@ function getStatus(domain, callback) {
             func(function (error, result) {
                 if (error) debug('Ignored error - ' + what + ':', error);
 
-                safe.set(results, what, result);
+                safe.set(results, what, result || {});
 
                 callback();
             });
@@ -654,7 +654,6 @@ function configureMail(mailFqdn, mailDomain, callback) {
                             -v "${paths.MAIL_DATA_DIR}:/app/data" \
                             -v "${paths.PLATFORM_DATA_DIR}/addons/mail:/etc/mail" \
                             ${ports} \
-                            -p 127.0.0.1:2020:2020 \
                             --label isCloudronManaged=true \
                             --read-only -v /run -v /tmp ${tag}`;
 
@@ -1106,18 +1105,25 @@ function addMailbox(name, domain, userId, auditSource, callback) {
     });
 }
 
-function updateMailboxOwner(name, domain, userId, callback) {
+function updateMailboxOwner(name, domain, userId, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     name = name.toLowerCase();
 
-    mailboxdb.updateMailboxOwner(name, domain, userId, function (error) {
+    getMailbox(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        callback(null);
+        mailboxdb.updateMailboxOwner(name, domain, userId, function (error) {
+            if (error) return callback(error);
+
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: result.userId, userId });
+
+            callback(null);
+        });
     });
 }
 
@@ -1196,12 +1202,12 @@ function getLists(domain, callback) {
     });
 }
 
-function getList(domain, listName, callback) {
+function getList(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof listName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    mailboxdb.getList(listName, domain, function (error, result) {
+    mailboxdb.getList(name, domain, function (error, result) {
         if (error) return callback(error);
 
         callback(null, result);
@@ -1227,16 +1233,17 @@ function addList(name, domain, members, auditSource, callback) {
     mailboxdb.addList(name, domain, members, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members });
 
         callback();
     });
 }
 
-function updateList(name, domain, members, callback) {
+function updateList(name, domain, members, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert(Array.isArray(members));
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     name = name.toLowerCase();
@@ -1248,10 +1255,16 @@ function updateList(name, domain, members, callback) {
         if (!validator.isEmail(members[i])) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid email: ' + members[i]));
     }
 
-    mailboxdb.updateList(name, domain, members, function (error) {
+    getList(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        callback(null);
+        mailboxdb.updateList(name, domain, members, function (error) {
+            if (error) return callback(error);
+
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members });
+
+            callback(null);
+        });
     });
 }
 
@@ -1264,7 +1277,7 @@ function removeList(name, domain, auditSource, callback) {
     mailboxdb.del(name, domain, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
 
         callback();
     });

src/mail_templates/welcome_user.ejs

@@ -5,7 +5,7 @@ Dear <%= user.displayName || user.username || user.email %>,
 Welcome to <%= cloudronName %>!
 
 Follow the link to get started.
-<%- setupLink %>
+<%- inviteLink %>
 
 <% if (invitor && invitor.email) { %>
 You are receiving this email because you were invited by <%= invitor.email %>.
@@ -25,7 +25,7 @@ Powered by https://cloudron.io
 <h2>Welcome to <%= cloudronName %>!</h2>
 
 <p>
-    <a href="<%= setupLink %>">Get started</a>.
+    <a href="<%= inviteLink %>">Get started</a>.
 </p>
 
 <br/>

src/mailboxdb.js

@@ -12,6 +12,8 @@ exports = module.exports = {
     listMailboxes: listMailboxes,
     getLists: getLists,
 
+    listAllMailboxes: listAllMailboxes,
+
     get: get,
     getMailbox: getMailbox,
     getList: getList,
@@ -214,6 +216,21 @@ function listMailboxes(domain, page, perPage, callback) {
         });
 }
 
+function listAllMailboxes(page, perPage, callback) {
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? ORDER BY name LIMIT ${(page-1)*perPage},${perPage}`,
+        [ exports.TYPE_MAILBOX ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
 function getLists(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');

src/mailer.js

@@ -3,7 +3,7 @@
 exports = module.exports = {
     userAdded: userAdded,
     userRemoved: userRemoved,
-    adminChanged: adminChanged,
+    roleChanged: roleChanged,
     passwordReset: passwordReset,
     appUpdatesAvailable: appUpdatesAvailable,
 
@@ -26,7 +26,6 @@ exports = module.exports = {
 
 var assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    custom = require('./custom.js'),
     debug = require('debug')('box:mailer'),
     ejs = require('ejs'),
     mail = require('./mail.js'),
@@ -47,15 +46,16 @@ function getMailConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     settings.getCloudronName(function (error, cloudronName) {
-        // this is not fatal
-        if (error) {
-            debug(error);
-            cloudronName = 'Cloudron';
-        }
+        if (error) debug('Error getting cloudron name: ', error);
 
-        callback(null, {
-            cloudronName: cloudronName,
-            notificationFrom: `"${cloudronName}" <no-reply@${settings.adminDomain()}>`
+        settings.getSupportConfig(function (error, supportConfig) {
+            if (error) debug('Error getting support config: ', error);
+
+            callback(null, {
+                cloudronName: cloudronName || '',
+                notificationFrom: `"${cloudronName}" <no-reply@${settings.adminDomain()}>`,
+                supportEmail: supportConfig.email
+            });
         });
     });
 }
@@ -125,9 +125,10 @@ function mailUserEvent(mailTo, user, event) {
     });
 }
 
-function sendInvite(user, invitor) {
+function sendInvite(user, invitor, inviteLink) {
     assert.strictEqual(typeof user, 'object');
-    assert(typeof invitor === 'object');
+    assert.strictEqual(typeof invitor, 'object');
+    assert.strictEqual(typeof inviteLink, 'string');
 
     debug('Sending invite mail');
 
@@ -137,7 +138,7 @@ function sendInvite(user, invitor) {
         var templateData = {
             user: user,
             webadminUrl: settings.adminOrigin(),
-            setupLink: `${settings.adminOrigin()}/api/v1/session/account/setup.html?reset_token=${user.resetToken}&email=${encodeURIComponent(user.email)}`,
+            inviteLink: inviteLink,
             invitor: invitor,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
@@ -203,14 +204,13 @@ function userRemoved(mailTo, user) {
     mailUserEvent(mailTo, user, 'was removed');
 }
 
-function adminChanged(mailTo, user, isAdmin) {
+function roleChanged(mailTo, user) {
     assert.strictEqual(typeof mailTo, 'string');
     assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof isAdmin, 'boolean');
 
-    debug('Sending mail for adminChanged');
+    debug('Sending mail for roleChanged');
 
-    mailUserEvent(mailTo, user, isAdmin ? 'is now an admin' : 'is no more an admin');
+    mailUserEvent(mailTo, user, `now has the role '${user.role}`);
 }
 
 function passwordReset(user) {
@@ -223,7 +223,7 @@ function passwordReset(user) {
 
         var templateData = {
             user: user,
-            resetLink: `${settings.adminOrigin()}/api/v1/session/password/reset.html?reset_token=${user.resetToken}&email=${encodeURIComponent(user.email)}`,
+            resetLink: `${settings.adminOrigin()}/login.html?resetToken=${user.resetToken}`,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
         };
@@ -279,7 +279,7 @@ function appDied(mailTo, app) {
             from: mailConfig.notificationFrom,
             to: mailTo,
             subject: util.format('[%s] App %s is down', mailConfig.cloudronName, app.fqdn),
-            text: render('app_down.ejs', { title: app.manifest.title, appFqdn: app.fqdn, supportEmail: custom.spec().support.email, format: 'text' })
+            text: render('app_down.ejs', { title: app.manifest.title, appFqdn: app.fqdn, supportEmail: mailConfig.supportEmail, format: 'text' })
         };
 
         sendMail(mailOptions);

src/middleware/index.js

@@ -1,15 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    cookieParser: require('cookie-parser'),
     cors: require('./cors'),
-    csrf: require('csurf'),
     json: require('body-parser').json,
     morgan: require('morgan'),
     proxy: require('proxy-middleware'),
     lastMile: require('connect-lastmile'),
     multipart: require('./multipart.js'),
-    session: require('express-session'),
     timeout: require('connect-timeout'),
     urlencoded: require('body-parser').urlencoded
 };

src/nginxconfig.ejs

@@ -108,7 +108,9 @@ server {
 <% } -%>
 
     proxy_http_version 1.1;
+    # intercept errors (>= 400) and use the error_page handler
     proxy_intercept_errors on;
+    # nginx will return 504 on connect/timeout errors
     proxy_read_timeout       3500;
     proxy_connect_timeout    3250;
 
@@ -125,7 +127,15 @@ server {
 
     # only serve up the status page if we get proxy gateway errors
     root <%= sourceDir %>/dashboard/dist;
-    error_page 502 503 504 /appstatus.html;
+    # some apps use 503 to indicate updating or maintenance
+    error_page 502 504 /app_error_page;
+    location /app_error_page {
+        root /home/yellowtent/boxdata;
+        # the first argument looks for file under the root
+        try_files /custom_pages/$request_uri /custom_pages/app_not_responding.html /appstatus.html;
+        # internal means this is for internal routing and cannot be accessed as URL from browser
+        internal;
+    }
     location /appstatus.html {
         internal;
     }

src/notifications.js

@@ -25,7 +25,6 @@ let assert = require('assert'),
     auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     changelog = require('./changelog.js'),
-    custom = require('./custom.js'),
     debug = require('debug')('box:notifications'),
     eventlog = require('./eventlog.js'),
     mailer = require('./mailer.js'),
@@ -100,7 +99,8 @@ function actionForAllAdmins(skippingUserIds, iterator, callback) {
     assert.strictEqual(typeof iterator, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    users.getAllAdmins(function (error, result) {
+    users.getAdmins(function (error, result) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback();
         if (error) return callback(error);
 
         // filter out users we want to skip (like the user who did the action or the user the action was performed on)
@@ -134,14 +134,14 @@ function userRemoved(performedBy, eventId, user, callback) {
     }, callback);
 }
 
-function adminChanged(performedBy, eventId, user, callback) {
+function roleChanged(performedBy, eventId, user, callback) {
     assert.strictEqual(typeof performedBy, 'string');
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     actionForAllAdmins([ performedBy, user.id ], function (admin, done) {
-        mailer.adminChanged(admin.email, user, user.admin);
-        add(admin.id, eventId, `User '${user.displayName} ' ${user.admin ? 'is now an admin' : 'is no more an admin'}`, `User '${user.username || user.email || user.fallbackEmail}' ${user.admin ? 'is now an admin' : 'is no more an admin'}.`, done);
+        mailer.roleChanged(admin.email, user);
+        add(admin.id, eventId, `User '${user.displayName}'s role changed`, `User '${user.username || user.email || user.fallbackEmail}' now has the role '${user.role}'.`, done);
     }, callback);
 }
 
@@ -167,9 +167,6 @@ function oomEvent(eventId, app, addon, containerId, event, callback) {
         message = 'The container has been restarted automatically. Consider increasing the [memory limit](https://docs.docker.com/v17.09/edge/engine/reference/commandline/update/#update-a-containers-kernel-memory-constraints)';
     }
 
-    if (custom.spec().alerts.email) mailer.oomEvent(custom.spec().alerts.email, program, event);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, done) {
         mailer.oomEvent(admin.email, program, event);
 
@@ -182,9 +179,6 @@ function appUp(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.appUp(custom.spec().alerts.email, app);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, done) {
         mailer.appUp(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is back online`, `The application ${app.manifest.title} installed at ${app.fqdn} is back online.`, done);
@@ -196,9 +190,6 @@ function appDied(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.appDied(custom.spec().alerts.email, app);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, callback) {
         mailer.appDied(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is down`, `The application ${app.manifest.title} installed at ${app.fqdn} is not responding.`, callback);
@@ -246,9 +237,6 @@ function boxUpdateError(eventId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.boxUpdateError(custom.spec().alerts.email, errorMessage);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, done) {
         mailer.boxUpdateError(admin.email, errorMessage);
         add(admin.id, eventId, 'Cloudron update failed', `Failed to update Cloudron: ${errorMessage}. Update will be retried in 4 hours`, done);
@@ -261,9 +249,6 @@ function certificateRenewalError(eventId, vhost, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.certificateRenewalError(custom.spec().alerts.email, vhost, errorMessage);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, callback) {
         mailer.certificateRenewalError(admin.email, vhost, errorMessage);
         add(admin.id, eventId, `Certificate renewal of ${vhost} failed`, `Failed to new certs of ${vhost}: ${errorMessage}. Renewal will be retried in 12 hours`, callback);
@@ -276,9 +261,6 @@ function backupFailed(eventId, taskId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (custom.spec().alerts.email) mailer.backupFailed(custom.spec().alerts.email, errorMessage, `${settings.adminOrigin()}/logs.html?taskId=${taskId}`);
-    if (!custom.spec().alerts.notifyCloudronAdmins) return callback();
-
     actionForAllAdmins([], function (admin, callback) {
         mailer.backupFailed(admin.email, errorMessage, `${settings.adminOrigin()}/logs.html?taskId=${taskId}`);
         add(admin.id, eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}). Will be retried in 4 hours`, callback);
@@ -343,8 +325,8 @@ function onEvent(id, action, source, data, callback) {
         return userRemoved(source.userId, id, data.user, callback);
 
     case eventlog.ACTION_USER_UPDATE:
-        if (!data.adminStatusChanged) return callback();
-        return adminChanged(source.userId, id, data.user, callback);
+        if (!data.roleChanged) return callback();
+        return roleChanged(source.userId, id, data.user, callback);
 
     case eventlog.ACTION_APP_OOM:
         return oomEvent(id, data.app, data.addon, data.containerId, data.event, callback);

src/oauth2views/account_setup.ejs

@@ -1,82 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<script>
-
-'use strict';
-
-// very basic angular app
-var app = angular.module('Application', []);
-app.controller('Controller', ['$scope', function ($scope) {
-    $scope.username = '<%= (user && user.username) ? user.username : '' %>';
-    $scope.displayName = '<%= (user && user.displayName) ? user.displayName : '' %>';
-}]);
-
-</script>
-
-<div class="layout-content">
-
-  <center>
-    <br/>
-    <h4>Hello <%= (user && user.email) ? user.email : '' %>, welcome to <%= cloudronName %>!</h4>
-    <h2>Setup your account and password</h2>
-  </center>
-
-  <div class="container" ng-app="Application" ng-controller="Controller">
-    <div class="row">
-      <div class="col-md-6 col-md-offset-3">
-        <form action="/api/v1/session/account/setup" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
-          <input type="password" style="display: none;">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <input type="hidden" name="email" value="<%= email %>"/>
-          <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
-
-          <center><p class="has-error"><%= error %></p></center>
-
-<% if (user && user.username) { %>
-          <div class="form-group"">
-            <label class="control-label">Username</label>
-            <input type="text" class="form-control" ng-model="username" name="username" readonly required>
-          </div>
-<% } else { %>
-          <div class="form-group" ng-class="{ 'has-error': (setupForm.username.$dirty && setupForm.username.$invalid) }">
-            <label class="control-label">Username</label>
-            <div class="control-label" ng-show="setupForm.username.$dirty && setupForm.username.$invalid">
-              <small ng-show="setupForm.username.$error.minlength">The username is too short</small>
-              <small ng-show="setupForm.username.$error.maxlength">The username is too long</small>
-              <small ng-show="setupForm.username.$dirty && setupForm.username.$invalid">Not a valid username</small>
-            </div>
-            <input type="text" class="form-control" ng-model="username" name="username" required autofocus>
-          </div>
-<% } %>
-
-          <div class="form-group">
-            <label class="control-label">Full Name</label>
-            <input type="displayName" class="form-control" ng-model="displayName" name="displayName" required>
-          </div>
-
-          <div class="form-group" ng-class="{ 'has-error': (setupForm.password.$dirty && setupForm.password.$invalid) }">
-            <label class="control-label">New Password</label>
-            <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
-              <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be atleast 8 characters</small>
-            </div>
-            <input type="password" class="form-control" ng-model="password" name="password" ng-pattern="/^.{8,}$/" required>
-          </div>
-
-          <div class="form-group" ng-class="{ 'has-error': (setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)) }">
-            <label class="control-label">Repeat Password</label>
-            <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
-              <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
-            </div>
-            <input type="password" class="form-control" ng-model="passwordRepeat" name="passwordRepeat" required>
-          </div>
-
-          <center><input class="btn btn-primary btn-outline" type="submit" value="Setup" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/></center>
-        </form>
-      </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/callback.ejs

@@ -1,45 +0,0 @@
-<!-- callback tester -->
-
-<script>
-
-'use strict';
-
-var code = null;
-var redirectURI = null;
-var accessToken = null;
-var tokenType = null;
-var state = null;
-
-var args = window.location.search.slice(1).split('&');
-args.forEach(function (arg) {
-    var tmp = arg.split('=');
-    if (tmp[0] === 'redirectURI') {
-        redirectURI = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'code') {
-        code = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'state') {
-        state = window.decodeURIComponent(tmp[1]);
-    }
-});
-
-args = window.location.hash.slice(1).split('&');
-args.forEach(function (arg) {
-    var tmp = arg.split('=');
-    if (tmp[0] === 'access_token') {
-        accessToken = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'token_type') {
-        tokenType = window.decodeURIComponent(tmp[1]);
-    } else if (tmp[0] === 'state') {
-        state = window.decodeURIComponent(tmp[1]);
-    }
-});
-
-if (code && redirectURI) {
-    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'code=' + code + (state ? '&state=' + state : '');
-} else if (redirectURI && accessToken) {
-    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'token=' + accessToken + (state ? '&state=' + state : '');
-} else {
-    window.location.href = '/api/v1/session/login';
-}
-
-</script>

src/oauth2views/error.ejs

@@ -1,27 +0,0 @@
-<% include header %>
-
-<!-- error tester -->
-
-<div class="layout-content">
-
-  <div class="container" style="margin-top: 50px;">
-    <div class="row">
-      <div class="col-md-2"></div>
-      <div class="col-md-8">
-        <div class="alert alert-danger">
-          <%- message %>
-        </div>
-      </div>
-      <div class="col-md-2"></div>
-    </div>
-    <div class="row">
-      <div class="col-md-2"></div>
-      <div class="col-md-8 text-center">
-        <a href="<%- adminOrigin %>">Back</a>
-      </div>
-      <div class="col-md-2"></div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/footer.ejs

@@ -1,9 +0,0 @@
-
-    <footer class="text-center">
-      <span class="text-muted">&copy; 2016-19 <a href="https://cloudron.io" target="_blank">Cloudron</a></span>
-    </footer>
-
-	</div>
-
-</body>
-</html>

src/oauth2views/header.ejs

@@ -1,41 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, height=device-height" />
-    <meta http-equiv="Content-Security-Policy" content="default-src 'unsafe-inline' 'unsafe-eval' 'self'; img-src 'self'" />
-
-    <title> <%= title %> </title>
-
-    <link href="/api/v1/cloudron/avatar?<%= Math.random() %>" rel="icon" type="image/png">
-
-    <!-- Theme CSS -->
-    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
-
-    <!-- Custom Fonts -->
-    <link href="<%= adminOrigin %>/3rdparty/fontawesome/css/all.min.css" rel="stylesheet" rel="stylesheet" type="text/css">
-
-    <!-- jQuery-->
-    <script src="<%= adminOrigin %>/3rdparty/js/jquery.min.js"></script>
-
-    <!-- Bootstrap Core JavaScript -->
-    <script src="<%= adminOrigin %>/3rdparty/js/bootstrap.min.js"></script>
-
-    <!-- Angularjs scripts -->
-    <script src="<%= adminOrigin %>/3rdparty/js/angular.min.js"></script>
-    <script src="<%= adminOrigin %>/3rdparty/js/angular-loader.min.js"></script>
-
-</head>
-
-<body>
-
-  <div class="layout-root">
-
-    <nav class="navbar navbar-default navbar-static-top shadow" role="navigation" style="margin-bottom: 0">
-      <div class="container-fluid">
-        <div class="navbar-header">
-          <a href="/" class="navbar-brand navbar-brand-icon"><img src="/api/v1/cloudron/avatar?<%= Math.random() %>" width="40" height="40"/></a>
-          <a href="/" class="navbar-brand"><%= cloudronName %></a>
-        </div>
-      </div>
-    </nav>

src/oauth2views/login.ejs

@@ -1,58 +0,0 @@
-<% include header %>
-
-<!-- login tester -->
-
-<div class="layout-content">
-  <div class="card" style="padding: 20px; margin-top: 50px; max-width: 620px;">
-    <div class="row">
-      <div class="col-md-12" style="text-align: center;">
-        <img width="128" height="128" src="<%= applicationLogo %>?<%= Math.random() %>"/>
-        <h1><small>Login to</small> <%= applicationName %></h1>
-        <br/>
-      </div>
-    </div>
-    <br/>
-<% if (error) { -%>
-    <div class="row">
-      <div class="col-md-12">
-        <h4 class="has-error"><%= error %></h4>
-      </div>
-    </div>
-<% } -%>
-    <div class="row">
-      <div class="col-md-12">
-        <form id="loginForm" action="" method="post">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <div class="form-group">
-            <label class="control-label" for="inputUsername">Username</label>
-            <input type="text" class="form-control" id="inputUsername" name="username" value="<%= username %>" autofocus required>
-          </div>
-          <div class="form-group">
-            <label class="control-label" for="inputPassword">Password</label>
-            <input type="password" class="form-control" name="password" id="inputPassword" value="<%= password %>" required>
-          </div>
-          <div class="form-group">
-            <label class="control-label" for="inputPassword">2FA Token (if enabled)</label>
-            <input type="text" class="form-control" name="totpToken" id="inputTotpToken" value="">
-          </div>
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Sign in"/>
-        </form>
-        <a href="/api/v1/session/password/resetRequest.html">Reset password</a>
-      </div>
-    </div>
-  </div>
-</div>
-
-<script>
-
-(function () {
-    'use strict';
-
-    var search = window.location.search.slice(1).split('&').map(function (item) { return item.split('='); }).reduce(function (o, k) { o[k[0]] = k[1]; return o; }, {});
-
-    document.getElementById('loginForm').action = '/api/v1/session/login?returnTo=' + search.returnTo;
-})();
-
-</script>
-
-<% include footer %>

src/oauth2views/password_reset.ejs

@@ -1,53 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<script>
-
-'use strict';
-
-// very basic angular app
-var app = angular.module('Application', []);
-app.controller('Controller', [function () {}]);
-
-</script>
-
-<div class="layout-content">
-
-  <center>
-    <h2>Hello <%= user.username %>, set a new password</h2>
-  </center>
-
-  <br/>
-
-  <div class="container" ng-app="Application" ng-controller="Controller">
-    <div class="row">
-      <div class="col-md-6 col-md-offset-3">
-        <form action="/api/v1/session/password/reset" method="post" name="resetForm" autocomplete="off" role="form" novalidate>
-          <input type="password" style="display: none;">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <input type="hidden" name="email" value="<%= email %>"/>
-          <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
-
-          <div class="form-group" ng-class="{ 'has-error': resetForm.password.$dirty && resetForm.password.$invalid }">
-            <label class="control-label" for="inputPassword">New Password</label>
-            <div class="control-label" ng-show="resetForm.password.$dirty && resetForm.password.$invalid">
-              <small ng-show="resetForm.password.$dirty && resetForm.password.$invalid">Password must be atleast 8 characters</small>
-            </div>
-            <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^.{8,30}$/" autofocus required>
-          </div>
-          <div class="form-group" ng-class="{ 'has-error': resetForm.passwordRepeat.$dirty && (password !== passwordRepeat) }">
-            <label class="control-label" for="inputPasswordRepeat">Repeat Password</label>
-            <div class="control-label" ng-show="resetForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
-              <small ng-show="resetForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
-            </div>
-            <input type="password" class="form-control" id="inputPasswordRepeat" ng-model="passwordRepeat" name="passwordRepeat" required>
-          </div>
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Set New Password" ng-disabled="resetForm.$invalid || password !== passwordRepeat"/>
-        </form>
-      </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/password_reset_request.ejs

@@ -1,30 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<div class="layout-content">
-
-  <center>
-    <h2>Reset password</h2>
-  </center>
-
-  <br/>
-
-  <div class="container">
-    <div class="row">
-      <div class="col-md-6 col-md-offset-3">
-        <form action="/api/v1/session/password/resetRequest" method="post" autocomplete="off">
-          <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-          <div class="form-group">
-            <label class="control-label" for="inputIdentifier">Username</label>
-            <input type="text" class="form-control" id="inputIdentifier" name="identifier" autofocus required>
-          </div>
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Reset"/>
-        </form>
-        <a href="/api/v1/session/login">Login</a>
-      </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/oauth2views/password_reset_sent.ejs

@@ -1,25 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<div class="layout-content">
-
-  <center>
-    <h2>Password reset successful</h2>
-  </center>
-
-  <br/>
-
-  <div class="container">
-    <div class="row">
-        <div class="col-md-6 col-md-offset-3 text-center">
-          <p>An email was sent to you with a link to set a new password.</p>
-          <br/>
-          <br/>
-          If you have not received any email, simply <a href="/api/v1/session/password/resetRequest.html">try again</a>.
-        </div>
-    </div>
-  </div>
-</div>
-
-<% include footer %>

src/paths.js

@@ -24,8 +24,6 @@ exports = module.exports = {
     APPS_DATA_DIR: path.join(baseDir(), 'appsdata'),
     BOX_DATA_DIR: path.join(baseDir(), 'boxdata'),
 
-    CUSTOM_FILE: path.join(baseDir(), 'boxdata/custom.yml'),
-
     ACME_CHALLENGES_DIR: path.join(baseDir(), 'platformdata/acme'),
     ADDON_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons'),
     COLLECTD_APPCONFIG_DIR: path.join(baseDir(), 'platformdata/collectd/collectd.conf.d'),
@@ -37,11 +35,9 @@ exports = module.exports = {
     UPDATE_DIR: path.join(baseDir(), 'platformdata/update'),
     SNAPSHOT_INFO_FILE: path.join(baseDir(), 'platformdata/backup/snapshot-info.json'),
     DYNDNS_INFO_FILE: path.join(baseDir(), 'platformdata/dyndns-info.json'),
+    FEATURES_INFO_FILE: path.join(baseDir(), 'platformdata/features-info.json'),
     VERSION_FILE: path.join(baseDir(), 'platformdata/VERSION'),
 
-    SESSION_SECRET_FILE: path.join(baseDir(), 'boxdata/session.secret'),
-    SESSION_DIR: path.join(baseDir(), 'platformdata/sessions'),
-
     // this is not part of appdata because an icon may be set before install
     APP_ICONS_DIR: path.join(baseDir(), 'boxdata/appicons'),
     PROFILE_ICONS_DIR: path.join(baseDir(), 'boxdata/profileicons'),

src/provision.js

@@ -15,7 +15,6 @@ var appstore = require('./appstore.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    clients = require('./clients.js'),
     cloudron = require('./cloudron.js'),
     debug = require('debug')('box:provision'),
     domains = require('./domains.js'),
@@ -27,9 +26,9 @@ var appstore = require('./appstore.js'),
     semver = require('semver'),
     settings = require('./settings.js'),
     sysinfo = require('./sysinfo.js'),
-    superagent = require('superagent'),
     users = require('./users.js'),
     tld = require('tldjs'),
+    tokens = require('./tokens.js'),
     _ = require('underscore');
 
 const NOOP_CALLBACK = function (error) { if (error) debug(error); };
@@ -151,31 +150,6 @@ function setup(dnsConfig, sysinfoConfig, auditSource, callback) {
     });
 }
 
-function setTimeZone(ip, callback) {
-    assert.strictEqual(typeof ip, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('setTimeZone ip:%s', ip);
-
-    superagent.get('https://geolocation.cloudron.io/json').query({ ip: ip }).timeout(10 * 1000).end(function (error, result) {
-        if ((error && !error.response) || result.statusCode !== 200) {
-            debug('Failed to get geo location: %s', error.message);
-            return callback(null);
-        }
-
-        var timezone = safe.query(result.body, 'location.time_zone');
-
-        if (!timezone || typeof timezone !== 'string') {
-            debug('No timezone in geoip response : %j', result.body);
-            return callback(null);
-        }
-
-        debug('Setting timezone to ', timezone);
-
-        settings.setTimeZone(timezone, callback);
-    });
-}
-
 function activate(username, password, email, displayName, ip, auditSource, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
@@ -187,13 +161,11 @@ function activate(username, password, email, displayName, ip, auditSource, callb
 
     debug('activating user:%s email:%s', username, email);
 
-    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
-
     users.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
         if (error && error.reason === BoxError.ALREADY_EXISTS) return callback(new BoxError(BoxError.CONFLICT, 'Already activated'));
         if (error) return callback(error);
 
-        clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+        tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
             if (error) return callback(error);
 
             eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
@@ -249,7 +221,7 @@ function restore(backupConfig, backupId, version, sysinfoConfig, auditSource, ca
                     backups.restore.bind(null, backupConfig, backupId, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)),
                     settings.setSysinfoConfig.bind(null, sysinfoConfig),
                     cloudron.setupDashboard.bind(null, auditSource, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)),
-                    settings.setBackupConfig.bind(null, backupConfig), // update with the latest backupConfig
+                    settings.setBackupCredentials.bind(null, backupConfig), // update just the credentials and not the policy and flags
                     eventlog.add.bind(null, eventlog.ACTION_RESTORE, auditSource, { backupId }),
                 ], function (error) {
                     gProvisionStatus.restore.active = false;
@@ -268,14 +240,16 @@ function getStatus(callback) {
     users.isActivated(function (error, activated) {
         if (error) return callback(error);
 
-        settings.getCloudronName(function (error, cloudronName) {
+        settings.getAll(function (error, allSettings) {
             if (error) return callback(error);
 
             callback(null, _.extend({
                 version: constants.VERSION,
                 apiServerOrigin: settings.apiServerOrigin(), // used by CaaS tool
+                webServerOrigin: settings.webServerOrigin(), // used by CaaS tool
                 provider: settings.provider(),
-                cloudronName: cloudronName,
+                cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
+                footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
                 adminFqdn: settings.adminDomain() ? settings.adminFqdn() : null,
                 activated: activated,
             }, gProvisionStatus));

src/reverseproxy.js

@@ -582,6 +582,8 @@ function renewCerts(options, auditSource, progressCallback, callback) {
 
         // add app main
         allApps.forEach(function (app) {
+            if (app.runState === apps.RSTATE_STOPPED) return; // do not renew certs of stopped apps
+
             appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'main', app: app, nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf') });
 
             app.alternateDomains.forEach(function (alternateDomain) {

src/routes/accesscontrol.js

@@ -1,159 +1,134 @@
 'use strict';
 
 exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize,
+    passwordAuth: passwordAuth,
+    tokenAuth: tokenAuth,
 
-    scope: scope,
+    authorize: authorize,
     websocketAuth: websocketAuth
 };
 
 var accesscontrol = require('../accesscontrol.js'),
     assert = require('assert'),
-    BasicStrategy = require('passport-http').BasicStrategy,
-    BearerStrategy = require('passport-http-bearer').Strategy,
     BoxError = require('../boxerror.js'),
-    clients = require('../clients.js'),
-    ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
     externalLdap = require('../externalldap.js'),
     HttpError = require('connect-lastmile').HttpError,
-    LocalStrategy = require('passport-local').Strategy,
-    passport = require('passport'),
-    users = require('../users.js');
+    users = require('../users.js'),
+    speakeasy = require('speakeasy');
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function passwordAuth(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
 
-    // serialize user into session
-    passport.serializeUser(function (user, callback) {
-        callback(null, user.id);
-    });
+    if (!req.body.username || typeof req.body.username !== 'string') return next(new HttpError(400, 'A username must be non-empty string'));
+    if (!req.body.password || typeof req.body.password !== 'string') return next(new HttpError(400, 'A password must be non-empty string'));
 
-    // deserialize user from session
-    passport.deserializeUser(function(userId, callback) {
-        users.get(userId, function (error, result) {
-            if (error) return callback(null, null /* user */, error.message); // will end up as a 401. can happen if user with active session got deleted
+    const username = req.body.username;
+    const password = req.body.password;
 
-            callback(null, result);
-        });
-    });
+    function check2FA(user) {
+        assert.strictEqual(typeof user, 'object');
 
+        if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
+            if (!req.body.totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
 
-    // used when username/password is sent in request body. used in CLI login & oauth2 login route
-    passport.use(new LocalStrategy(function (username, password, callback) {
-
-        // TODO we should only do this for dashboard logins
-        function createAndVerifyUserIfNotExist(identifier, password, callback) {
-            assert.strictEqual(typeof identifier, 'string');
-            assert.strictEqual(typeof password, 'string');
-            assert.strictEqual(typeof callback, 'function');
-
-            externalLdap.createAndVerifyUserIfNotExist(identifier.toLowerCase(), password, function (error, result) {
-                if (error && error.reason === BoxError.BAD_STATE) return callback(null, false);
-                if (error && error.reason === BoxError.BAD_FIELD) return callback(null, false);
-                if (error && error.reason === BoxError.CONFLICT) return callback(null, false);
-                if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === BoxError.INVALID_CREDENTIALS) return callback(null, false);
-                if (error) return callback(error);
-
-                callback(null, result);
-            });
+            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
+            if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
         }
 
-        if (username.indexOf('@') === -1) {
-            users.verifyWithUsername(username, password, function (error, result) {
-                if (error && error.reason === BoxError.NOT_FOUND) return createAndVerifyUserIfNotExist(username, password, callback);
-                if (error && error.reason === BoxError.INVALID_CREDENTIALS) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, result);
-            });
-        } else {
-            users.verifyWithEmail(username, password, function (error, result) {
-                if (error && error.reason === BoxError.NOT_FOUND) return createAndVerifyUserIfNotExist(username, password, callback);
-                if (error && error.reason === BoxError.INVALID_CREDENTIALS) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, result);
-            });
-        }
-    }));
-
-    // Used to authenticate a OAuth2 client which uses clientId and clientSecret in the Authorization header
-    passport.use(new BasicStrategy(function (clientId, clientSecret, callback) {
-        clients.get(clientId, function (error, client) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-            if (error) return callback(error);
-            if (client.clientSecret !== clientSecret) return callback(null, false);
-            callback(null, client);
-        });
-    }));
-
-    // Used to authenticate a OAuth2 client which uses clientId and clientSecret in the request body (client_id, client_secret)
-    passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clients.get(clientId, function(error, client) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-            if (error) { return callback(error); }
-            if (client.clientSecret !== clientSecret) { return callback(null, false); }
-            callback(null, client);
-        });
-    }));
-
-    // used for "Authorization: Bearer token" or access_token query param authentication
-    passport.use(new BearerStrategy(function (token, callback) {
-        accesscontrol.validateToken(token, callback);
-    }));
-
-    callback(null);
-}
-
-function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    callback(null);
-}
-
-//  The scope middleware provides an auth middleware for routes.
-//
-//  It is used for API routes, which are authenticated using accesstokens.
-//  Those accesstokens carry OAuth scopes and the middleware takes the required
-//  scope as an argument and will verify the accesstoken against it.
-//
-//  See server.js:
-//    var profileScope = routes.oauth2.scope('profile');
-//
-function scope(requiredScope) {
-    assert.strictEqual(typeof requiredScope, 'string');
-
-    var requiredScopes = requiredScope.split(',');
-
-    return [
-        passport.authenticate(['bearer'], { session: false }),
-
-        function (req, res, next) {
-            assert(req.authInfo && typeof req.authInfo === 'object');
-
-            var error = accesscontrol.hasScopes(req.authInfo.authorizedScopes, requiredScopes);
-            if (error) return next(new HttpError(403, error.message));
-
-            next();
-        }
-    ];
-}
-
-function websocketAuth(requiredScopes, req, res, next) {
-    assert(Array.isArray(requiredScopes));
-
-    if (typeof req.query.access_token !== 'string') return next(new HttpError(401, 'Unauthorized'));
-
-    accesscontrol.validateToken(req.query.access_token, function (error, user, info) {
-        if (error) return next(new HttpError(500, error.message));
-        if (!user) return next(new HttpError(401, 'Unauthorized'));
-
         req.user = user;
 
-        var e = accesscontrol.hasScopes(info.authorizedScopes, requiredScopes);
-        if (e) return next(new HttpError(403, e.message));
+        next();
+    }
+
+    function createAndVerifyUserIfNotExist(identifier, password) {
+        assert.strictEqual(typeof identifier, 'string');
+        assert.strictEqual(typeof password, 'string');
+
+        externalLdap.createAndVerifyUserIfNotExist(identifier.toLowerCase(), password, function (error, result) {
+            if (error && error.reason === BoxError.BAD_STATE) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.CONFLICT) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.NOT_FOUND) return next(new HttpError(401, 'Unauthorized'));
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+            if (error) return next(new HttpError(500, error));
+
+            check2FA(result);
+        });
+    }
+
+    if (username.indexOf('@') === -1) {
+        users.verifyWithUsername(username, password, users.AP_WEBADMIN, function (error, result) {
+            if (error && error.reason === BoxError.NOT_FOUND) return createAndVerifyUserIfNotExist(username, password);
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+            if (error) return next(new HttpError(500, error));
+            if (!result) return next(new HttpError(401, 'Unauthorized'));
+
+            check2FA(result);
+        });
+    } else {
+        users.verifyWithEmail(username, password, users.AP_WEBADMIN, function (error, result) {
+            if (error && error.reason === BoxError.NOT_FOUND) return createAndVerifyUserIfNotExist(username, password);
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+            if (error) return next(new HttpError(500, error));
+            if (!result) return next(new HttpError(401, 'Unauthorized'));
+
+            check2FA(result);
+        });
+    }
+}
+
+function tokenAuth(req, res, next) {
+    var token;
+
+    // this determines the priority
+    if (req.body && req.body.access_token) token = req.body.access_token;
+    if (req.query && req.query.access_token) token = req.query.access_token;
+    if (req.headers && req.headers.authorization) {
+        var parts = req.headers.authorization.split(' ');
+        if (parts.length == 2) {
+            var scheme = parts[0];
+            var credentials = parts[1];
+
+            if (/^Bearer$/i.test(scheme)) token = credentials;
+        }
+    }
+
+    if (!token) return next(new HttpError(401, 'Unauthorized'));
+
+    accesscontrol.verifyToken(token, function (error, user) {
+        if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+        if (error) return next(new HttpError(500, error.message));
+
+        req.user = user;
+
+        next();
+    });
+}
+
+function authorize(requiredRole) {
+    assert.strictEqual(typeof requiredRole, 'string');
+
+    return function (req, res, next) {
+        assert.strictEqual(typeof req.user, 'object');
+
+        if (users.compareRoles(req.user.role, requiredRole) < 0) return next(new HttpError(403, `role '${requiredRole}' is required but user has only '${req.user.role}'`));
+
+        next();
+    };
+}
+
+function websocketAuth(requiredRole, req, res, next) {
+    assert.strictEqual(typeof requiredRole, 'string');
+
+    if (typeof req.query.access_token !== 'string') return next(new HttpError(401, 'Unauthorized'));
+
+    accesscontrol.verifyToken(req.query.access_token, function (error, user) {
+        if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
+        if (error) return next(new HttpError(500, error.message));
+
+        req.user = user;
+
+        if (users.compareRoles(req.user.role, requiredRole) < 0) return next(new HttpError(403, `role '${requiredRole}' is required but user has only '${user.role}'`));
 
         next();
     });

src/routes/apppasswords.js

@@ -0,0 +1,61 @@
+'use strict';
+
+exports = module.exports = {
+    list: list,
+    get: get,
+    del: del,
+    add: add
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    users = require('../users.js');
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    users.getAppPassword(req.params.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function add(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+    if (typeof req.body.identifier !== 'string') return next(new HttpError(400, 'identifier must be string'));
+
+    users.addAppPassword(req.user.id, req.body.identifier, req.body.name, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(201, result));
+    });
+}
+
+function list(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    users.getAppPasswords(req.user.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { appPasswords: result }));
+    });
+}
+
+function del(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    // TODO: verify userId owns the id ?
+    users.delAppPassword(req.params.id, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(204, {}));
+    });
+}

src/routes/apps.js

@@ -20,6 +20,7 @@ exports = module.exports = {
     setTags: setTags,
     setIcon: setIcon,
     setMemoryLimit: setMemoryLimit,
+    setCpuShares: setCpuShares,
     setAutomaticBackup: setAutomaticBackup,
     setAutomaticUpdate: setAutomaticUpdate,
     setReverseProxyConfig: setReverseProxyConfig,
@@ -32,6 +33,7 @@ exports = module.exports = {
 
     stopApp: stopApp,
     startApp: startApp,
+    restartApp: restartApp,
     exec: exec,
     execWebSocket: execWebSocket,
 
@@ -206,6 +208,19 @@ function setMemoryLimit(req, res, next) {
     });
 }
 
+function setCpuShares(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    if (typeof req.body.cpuShares !== 'number') return next(new HttpError(400, 'cpuShares is not a number'));
+
+    apps.setCpuShares(req.params.id, req.body.cpuShares, auditSource.fromRequest(req), function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, { taskId: result.taskId }));
+    });
+}
+
 function setAutomaticBackup(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
@@ -350,7 +365,11 @@ function repairApp(req, res, next) {
     const data = req.body;
 
     if ('manifest' in data) {
-        if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'backupId must be an object'));
+        if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    }
+
+    if ('dockerImage' in data) {
+        if (!data.dockerImage || typeof data.dockerImage !== 'string') return next(new HttpError(400, 'dockerImage must be a string'));
     }
 
     apps.repair(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
@@ -480,6 +499,18 @@ function stopApp(req, res, next) {
     });
 }
 
+function restartApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    debug('Restart app id:%s', req.params.id);
+
+    apps.restart(req.params.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, { taskId: result.taskId }));
+    });
+}
+
 function updateApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
     assert.strictEqual(typeof req.body, 'object');

src/routes/appstore.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     getApp: getApp,
     getAppVersion: getAppVersion,
 
+    createUserToken: createUserToken,
     registerCloudron: registerCloudron,
     getSubscription: getSubscription
 };
@@ -12,35 +13,20 @@ exports = module.exports = {
 var appstore = require('../appstore.js'),
     assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    custom = require('../custom.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
-function isAppAllowed(appstoreId) {
-    if (custom.spec().appstore.blacklist.includes(appstoreId)) return false;
-
-    if (!custom.spec().appstore.whitelist) return true;
-    if (!custom.spec().appstore.whitelist[appstoreId]) return false;
-
-    return true;
-}
-
 function getApps(req, res, next) {
     appstore.getApps(function (error, apps) {
         if (error) return next(BoxError.toHttpError(error));
 
-        let filteredApps = apps.filter((app) => !custom.spec().appstore.blacklist.includes(app.id));
-        if (custom.spec().appstore.whitelist) filteredApps = filteredApps.filter((app) => app.id in custom.spec().appstore.whitelist);
-
-        next(new HttpSuccess(200, { apps: filteredApps }));
+        next(new HttpSuccess(200, { apps }));
     });
 }
 
 function getApp(req, res, next) {
     assert.strictEqual(typeof req.params.appstoreId, 'string');
 
-    if (!isAppAllowed(req.params.appstoreId)) return next(new HttpError(405, 'feature disabled by admin'));
-
     appstore.getApp(req.params.appstoreId, function (error, app) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -52,8 +38,6 @@ function getAppVersion(req, res, next) {
     assert.strictEqual(typeof req.params.appstoreId, 'string');
     assert.strictEqual(typeof req.params.versionId, 'string');
 
-    if (!isAppAllowed(req.params.appstoreId)) return next(new HttpError(405, 'feature disabled by admin'));
-
     appstore.getAppVersion(req.params.appstoreId, req.params.versionId, function (error, manifest) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -61,6 +45,14 @@ function getAppVersion(req, res, next) {
     });
 }
 
+function createUserToken(req, res, next) {
+    appstore.getUserToken(function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(201, { accessToken: result }));
+    });
+}
+
 function registerCloudron(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
@@ -68,6 +60,7 @@ function registerCloudron(req, res, next) {
     if (typeof req.body.password !== 'string' || !req.body.password) return next(new HttpError(400, 'password must be string'));
     if ('totpToken' in req.body && typeof req.body.totpToken !== 'string') return next(new HttpError(400, 'totpToken must be string'));
     if (typeof req.body.signup !== 'boolean') return next(new HttpError(400, 'signup must be a boolean'));
+    if ('purpose' in req.body && typeof req.body.purpose !== 'string') return next(new HttpError(400, 'purpose must be string'));
 
     appstore.registerWithLoginCredentials(req.body, function (error) {
         if (error) return next(BoxError.toHttpError(error));
@@ -82,6 +75,6 @@ function getSubscription(req, res, next) {
     appstore.getSubscription(function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
-        next(new HttpSuccess(200, result)); // { email, cloudronId, plan, cancel_at, status }
+        next(new HttpSuccess(200, result)); // { email, cloudronId, cloudronCreatedAt, plan, current_period_end, canceled_at, cancel_at, status, features }
     });
 }

src/routes/clients.js

@@ -1,124 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    add: add,
-    get: get,
-    del: del,
-    getAll: getAll,
-    addToken: addToken,
-    getTokens: getTokens,
-    delTokens: delTokens,
-    delToken: delToken
-};
-
-var assert = require('assert'),
-    BoxError = require('../boxerror.js'),
-    clients = require('../clients.js'),
-    constants = require('../constants.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    validUrl = require('valid-url');
-
-function add(req, res, next) {
-    var data = req.body;
-
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (typeof data.appId !== 'string' || !data.appId) return next(new HttpError(400, 'appId is required'));
-    if (typeof data.redirectURI !== 'string' || !data.redirectURI) return next(new HttpError(400, 'redirectURI is required'));
-    if (typeof data.scope !== 'string' || !data.scope) return next(new HttpError(400, 'scope is required'));
-    if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));
-
-    clients.add(data.appId, clients.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(201, result));
-    });
-}
-
-function get(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-
-    clients.get(req.params.clientId, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, result));
-    });
-}
-
-function del(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-
-    clients.get(req.params.clientId, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        // we do not allow to use the REST API to delete addon clients
-        if (result.type !== clients.TYPE_EXTERNAL) return next(new HttpError(405, 'Deleting app addon clients is not allowed.'));
-
-        clients.del(req.params.clientId, function (error, result) {
-            if (error) return next(BoxError.toHttpError(error));
-
-            next(new HttpSuccess(204, result));
-        });
-    });
-}
-
-function getAll(req, res, next) {
-    clients.getAll(function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, { clients: result }));
-    });
-}
-
-function addToken(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-    assert.strictEqual(typeof req.body, 'object');
-
-    var data = req.body;
-    var expiresAt = data.expiresAt ? parseInt(data.expiresAt, 10) : Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
-    if (isNaN(expiresAt) || expiresAt <= Date.now()) return next(new HttpError(400, 'expiresAt must be a timestamp in the future'));
-    if ('name' in req.body && typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be a string'));
-
-    clients.addTokenByUserId(req.params.clientId, req.user.id, expiresAt, { name: req.body.name || '' }, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(201, { token: result }));
-    });
-}
-
-function getTokens(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-
-    clients.getTokensByUserId(req.params.clientId, req.user.id, function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        result = result.map(clients.removeTokenPrivateFields);
-
-        next(new HttpSuccess(200, { tokens: result }));
-    });
-}
-
-function delTokens(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-
-    clients.delTokensByUserId(req.params.clientId, req.user.id, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(204));
-    });
-}
-
-function delToken(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-    assert.strictEqual(typeof req.params.tokenId, 'string');
-    assert.strictEqual(typeof req.user, 'object');
-
-    clients.delToken(req.params.clientId, req.params.tokenId, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(204));
-    });
-}

src/routes/cloudron.js

@@ -1,6 +1,11 @@
 'use strict';
 
 exports = module.exports = {
+    login: login,
+    logout: logout,
+    passwordResetRequest: passwordResetRequest,
+    passwordReset: passwordReset,
+    setupAccount: setupAccount,
     reboot: reboot,
     isRebootRequired: isRebootRequired,
     getConfig: getConfig,
@@ -23,15 +28,135 @@ let assert = require('assert'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
     cloudron = require('../cloudron.js'),
-    custom = require('../custom.js'),
+    constants = require('../constants.js'),
+    debug = require('debug')('box:routes/cloudron'),
+    eventlog = require('../eventlog.js'),
     externalLdap = require('../externalldap.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     sysinfo = require('../sysinfo.js'),
     system = require('../system.js'),
+    tokendb = require('../tokendb.js'),
+    tokens = require('../tokens.js'),
     updater = require('../updater.js'),
+    users = require('../users.js'),
     updateChecker = require('../updatechecker.js');
 
+function login(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    if ('type' in req.body && typeof req.body.type !== 'string') return next(new HttpError(400, 'type must be a string'));
+
+    const type = req.body.type || tokens.ID_WEBADMIN;
+    const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    const auditSource = { authType: 'basic', ip: ip };
+
+    const error = tokens.validateTokenType(type);
+    if (error) return next(new HttpError(400, error.message));
+
+    tokens.add(type, req.user.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function logout(req, res) {
+    var token;
+
+    // this determines the priority
+    if (req.body && req.body.access_token) token = req.body.access_token;
+    if (req.query && req.query.access_token) token = req.query.access_token;
+    if (req.headers && req.headers.authorization) {
+        var parts = req.headers.authorization.split(' ');
+        if (parts.length == 2) {
+            var scheme = parts[0];
+            var credentials = parts[1];
+
+            if (/^Bearer$/i.test(scheme)) token = credentials;
+        }
+    }
+
+    if (!token) return res.redirect('/login.html');
+
+    tokendb.delByAccessToken(token, function () { res.redirect('/login.html'); });
+}
+
+function passwordResetRequest(req, res, next) {
+    if (!req.body.identifier || typeof req.body.identifier !== 'string') return next(new HttpError(401, 'A identifier must be non-empty string'));
+
+    users.resetPasswordByIdentifier(req.body.identifier, function (error) {
+        if (error && error.reason !== BoxError.NOT_FOUND) console.error(error);
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
+function passwordReset(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
+    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
+
+    users.getByResetToken(req.body.resetToken, function (error, userObject) {
+        if (error) return next(new HttpError(401, 'Invalid resetToken'));
+
+        if (!userObject.username) return next(new HttpError(409, 'No username set'));
+
+        // setPassword clears the resetToken
+        users.setPassword(userObject, req.body.password, function (error) {
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+            if (error) return next(new HttpError(500, error));
+
+            tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+                if (error) return next(new HttpError(500, error));
+
+                next(new HttpSuccess(202, { accessToken: result.accessToken }));
+            });
+        });
+    });
+}
+
+function setupAccount(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.email || typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be a non-empty string'));
+    if (!req.body.resetToken || typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'resetToken must be a non-empty string'));
+    if (!req.body.password || typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be a non-empty string'));
+    if (!req.body.username || typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be a non-empty string'));
+    if (!req.body.displayName || typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be a non-empty string'));
+
+    debug(`setupAccount: for email ${req.body.email} and username ${req.body.username} with token ${req.body.resetToken}`);
+
+    users.getByResetToken(req.body.resetToken, function (error, userObject) {
+        if (error) return next(new HttpError(401, 'Invalid Reset Token'));
+
+        users.update(userObject, { username: req.body.username, displayName: req.body.displayName }, auditSource.fromRequest(req), function (error) {
+            if (error && error.reason === BoxError.ALREADY_EXISTS) return next(new HttpError(409, 'Username already used'));
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+            if (error && error.reason === BoxError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
+            if (error) return next(new HttpError(500, error));
+
+            userObject.username = req.body.username;
+            userObject.displayName = req.body.displayName;
+
+            // setPassword clears the resetToken
+            users.setPassword(userObject, req.body.password, function (error) {
+                if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+                if (error) return next(new HttpError(500, error));
+
+                tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+                    if (error) return next(new HttpError(500, error));
+
+                    next(new HttpSuccess(201, { accessToken: result.accessToken }));
+                });
+            });
+        });
+    });
+}
+
 function reboot(req, res, next) {
     // Finish the request, to let the appstore know we triggered the reboot
     next(new HttpSuccess(202, {}));
@@ -165,8 +290,6 @@ function getLogStream(req, res, next) {
 function setDashboardAndMailDomain(req, res, next) {
     if (!req.body.domain || typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
 
-    if (!custom.spec().domains.changeDashboardDomain) return next(new HttpError(405, 'feature disabled by admin'));
-
     cloudron.setDashboardAndMailDomain(req.body.domain, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -177,8 +300,6 @@ function setDashboardAndMailDomain(req, res, next) {
 function prepareDashboardDomain(req, res, next) {
     if (!req.body.domain || typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
 
-    if (!custom.spec().domains.changeDashboardDomain) return next(new HttpError(405, 'feature disabled by admin'));
-
     cloudron.prepareDashboardDomain(req.body.domain, auditSource.fromRequest(req), function (error, taskId) {
         if (error) return next(BoxError.toHttpError(error));
 

src/routes/developer.js

@@ -1,35 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    login: login
-};
-
-var clients = require('../clients.js'),
-    passport = require('passport'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    speakeasy = require('speakeasy');
-
-function login(req, res, next) {
-    passport.authenticate('local', function (error, user) {
-        if (error) return next(new HttpError(500, error));
-        if (!user) return next(new HttpError(401, 'Invalid credentials'));
-
-        var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-
-        if (!user.ghost && user.twoFactorAuthenticationEnabled) {
-            if (!req.body.totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
-
-            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
-            if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
-        }
-
-        const auditSource = { authType: 'cli', ip: ip };
-        clients.issueDeveloperToken(user, auditSource, function (error, result) {
-            if (error) return next(new HttpError(500, error));
-
-            next(new HttpSuccess(200, result));
-        });
-    })(req, res, next);
-}
-

src/routes/domains.js

@@ -8,8 +8,6 @@ exports = module.exports = {
     del: del,
 
     checkDnsRecords: checkDnsRecords,
-
-    verifyDomainLock: verifyDomainLock
 };
 
 var assert = require('assert'),
@@ -19,18 +17,6 @@ var assert = require('assert'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
-function verifyDomainLock(req, res, next) {
-    assert.strictEqual(typeof req.params.domain, 'string');
-
-    domains.get(req.params.domain, function (error, domain) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        if (domain.locked) return next(new HttpError(423, 'This domain is locked'));
-
-        next();
-    });
-}
-
 function add(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
@@ -159,4 +145,4 @@ function checkDnsRecords(req, res, next) {
 
         next(new HttpSuccess(200, { needsOverwrite: result.needsOverwrite }));
     });
-}
+}

src/routes/index.js

@@ -2,18 +2,17 @@
 
 exports = module.exports = {
     accesscontrol: require('./accesscontrol.js'),
+    appPasswords: require('./apppasswords.js'),
     apps: require('./apps.js'),
     appstore: require('./appstore.js'),
     backups: require('./backups.js'),
-    clients: require('./clients.js'),
     cloudron: require('./cloudron.js'),
-    developer: require('./developer.js'),
     domains: require('./domains.js'),
     eventlog: require('./eventlog.js'),
     graphs: require('./graphs.js'),
     groups: require('./groups.js'),
-    oauth2: require('./oauth2.js'),
     mail: require('./mail.js'),
+    mailserver: require('./mailserver.js'),
     notifications: require('./notifications.js'),
     profile: require('./profile.js'),
     provision: require('./provision.js'),
@@ -21,5 +20,6 @@ exports = module.exports = {
     settings: require('./settings.js'),
     support: require('./support.js'),
     tasks: require('./tasks.js'),
+    tokens: require('./tokens.js'),
     users: require('./users.js')
 };

src/routes/mail.js

@@ -3,7 +3,6 @@
 exports = module.exports = {
     getDomain: getDomain,
     addDomain: addDomain,
-    getDomainStats: getDomainStats,
     removeDomain: removeDomain,
 
     setDnsRecords: setDnsRecords,
@@ -31,7 +30,7 @@ exports = module.exports = {
     getList: getList,
     addList: addList,
     updateList: updateList,
-    removeList: removeList
+    removeList: removeList,
 };
 
 var assert = require('assert'),
@@ -39,11 +38,7 @@ var assert = require('assert'),
     BoxError = require('../boxerror.js'),
     mail = require('../mail.js'),
     HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    middleware = require('../middleware/index.js'),
-    url = require('url');
-
-var mailProxy = middleware.proxy(url.parse('http://127.0.0.1:2020'));
+    HttpSuccess = require('connect-lastmile').HttpSuccess;
 
 function getDomain(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
@@ -67,19 +62,6 @@ function addDomain(req, res, next) {
     });
 }
 
-function getDomainStats(req, res, next) {
-    assert.strictEqual(typeof req.params.domain, 'string');
-
-    var parsedUrl = url.parse(req.url, true /* parseQueryString */);
-    delete parsedUrl.query['access_token'];
-    delete req.headers['authorization'];
-    delete req.headers['cookies'];
-
-    req.url = url.format({ pathname: req.params.domain, query: parsedUrl.query });
-
-    mailProxy(req, res, next);
-}
-
 function setDnsRecords(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.domain, 'string');
@@ -197,10 +179,10 @@ function listMailboxes(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
-    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a positive number'));
 
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
-    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a positive number'));
 
     mail.listMailboxes(req.params.domain, page, perPage, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -239,7 +221,7 @@ function updateMailbox(req, res, next) {
 
     if (typeof req.body.userId !== 'string') return next(new HttpError(400, 'userId must be a string'));
 
-    mail.updateMailboxOwner(req.params.name, req.params.domain, req.body.userId, function (error) {
+    mail.updateMailboxOwner(req.params.name, req.params.domain, req.body.userId, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));
@@ -261,10 +243,10 @@ function listAliases(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
-    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a positive number'));
 
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
-    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a positive number'));
 
     mail.listAliases(req.params.domain, page, perPage, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -316,7 +298,7 @@ function getList(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.params.name, 'string');
 
-    mail.getList(req.params.domain, req.params.name, function (error, result) {
+    mail.getList(req.params.name, req.params.domain, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { list: result }));
@@ -353,7 +335,7 @@ function updateList(req, res, next) {
         if (typeof req.body.members[i] !== 'string') return next(new HttpError(400, 'member must be a string'));
     }
 
-    mail.updateList(req.params.name, req.params.domain, req.body.members, function (error) {
+    mail.updateList(req.params.name, req.params.domain, req.body.members, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));

src/routes/mailserver.js

@@ -0,0 +1,43 @@
+'use strict';
+
+exports = module.exports = {
+    proxy
+};
+
+var addons = require('../addons.js'),
+    assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    middleware = require('../middleware/index.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    url = require('url');
+
+function proxy(req, res, next) {
+    assert.strictEqual(typeof req.params.pathname, 'string');
+
+    let parsedUrl = url.parse(req.url, true /* parseQueryString */);
+
+    // do not proxy protected values
+    delete parsedUrl.query['access_token'];
+    delete req.headers['authorization'];
+    delete req.headers['cookies'];
+
+    addons.getServiceDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        parsedUrl.query['access_token'] = addonDetails.token;
+        req.url = url.format({ pathname: req.params.pathname, query: parsedUrl.query });
+
+        const proxyOptions = url.parse(`https://${addonDetails.ip}:3000`);
+        proxyOptions.rejectUnauthorized = false;
+        const mailserverProxy = middleware.proxy(proxyOptions);
+
+        mailserverProxy(req, res, function (error) {
+            if (!error) return next();
+
+            if (error.code === 'ECONNREFUSED') return next(new HttpError(424, 'Unable to connect to mail server'));
+            if (error.code === 'ECONNRESET') return next(new HttpError(424, 'Unable to query mail server'));
+
+            next(new HttpError(500, error));
+        });
+    });
+}

src/routes/oauth2.js

@@ -1,563 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize,
-    loginForm: loginForm,
-    login: login,
-    logout: logout,
-    sessionCallback: sessionCallback,
-    passwordResetRequestSite: passwordResetRequestSite,
-    passwordResetRequest: passwordResetRequest,
-    passwordSentSite: passwordSentSite,
-    passwordResetSite: passwordResetSite,
-    passwordReset: passwordReset,
-    accountSetupSite: accountSetupSite,
-    accountSetup: accountSetup,
-    authorization: authorization,
-    token: token,
-    csrf: csrf
-};
-
-var apps = require('../apps.js'),
-    assert = require('assert'),
-    async = require('async'),
-    authcodedb = require('../authcodedb.js'),
-    BoxError = require('../boxerror.js'),
-    clients = require('../clients'),
-    constants = require('../constants.js'),
-    debug = require('debug')('box:routes/oauth2'),
-    eventlog = require('../eventlog.js'),
-    hat = require('../hat.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    middleware = require('../middleware/index.js'),
-    oauth2orize = require('oauth2orize'),
-    passport = require('passport'),
-    querystring = require('querystring'),
-    session = require('connect-ensure-login'),
-    settings = require('../settings.js'),
-    speakeasy = require('speakeasy'),
-    url = require('url'),
-    users = require('../users.js'),
-    util = require('util'),
-    _ = require('underscore');
-
-// appId is optional here
-function auditSource(req, appId) {
-    var tmp = {
-        authType: 'oauth',
-        ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress || null
-    };
-
-    if (appId) tmp.appId = appId;
-
-    return tmp;
-}
-
-var gServer = null;
-
-function initialize() {
-    assert.strictEqual(gServer, null);
-
-    gServer = oauth2orize.createServer();
-
-    gServer.serializeClient(function (client, callback) {
-        return callback(null, client.id);
-    });
-
-    gServer.deserializeClient(function (id, callback) {
-        clients.get(id, callback);
-    });
-
-    // grant authorization code that can be exchanged for access tokens. this is used by external oauth clients
-    gServer.grant(oauth2orize.grant.code({ scopeSeparator: ',' }, function (client, redirectURI, user, ares, callback) {
-        debug('grant code:', client.id, redirectURI, user.id, ares);
-
-        var code = hat(256);
-        var expiresAt = Date.now() + 60 * 60000; // 1 hour
-
-        authcodedb.add(code, client.id, user.id, expiresAt, function (error) {
-            if (error) return callback(error);
-
-            debug('grant code: new auth code for client %s code %s', client.id, code);
-
-            callback(null, code);
-        });
-    }));
-
-    // exchange authorization codes for access tokens. this is used by external oauth clients
-    gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI, callback) {
-        authcodedb.get(code, function (error, authCode) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-            if (error) return callback(error);
-            if (client.id !== authCode.clientId) return callback(null, false);
-
-            authcodedb.del(code, function (error) {
-                if(error) return callback(error);
-
-                clients.addTokenByUserId(client.id, authCode.userId, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                    if (error) return callback(error);
-
-                    debug('exchange: new access token for client %s user %s token %s', client.id, authCode.userId, result.accessToken.slice(0, 6)); // partial token for security
-
-                    callback(null, result.accessToken);
-                });
-            });
-        });
-    }));
-
-    // implicit token grant that skips issuing auth codes. this is used by our webadmin
-    gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client, user, ares, callback) {
-        clients.addTokenByUserId(client.id, user.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-            if (error) return callback(error);
-
-            debug('grant token: new access token for client %s user %s token %s', client.id, user.id, result.accessToken.slice(0, 6)); // partial token for security
-
-            callback(null, result.accessToken);
-        });
-    }));
-
-    // overwrite the session.ensureLoggedIn to not use res.redirect() due to a chrome bug not sending cookies on redirects
-    session.ensureLoggedIn = function (redirectTo) {
-        assert.strictEqual(typeof redirectTo, 'string');
-
-        return function (req, res, next) {
-            if (!req.isAuthenticated || !req.isAuthenticated()) {
-                if (req.session) {
-                    req.session.returnTo = req.originalUrl || req.url;
-                }
-
-                res.status(200).send(util.format('<script>window.location.href = "%s";</script>', redirectTo));
-            } else {
-                next();
-            }
-        };
-    };
-}
-
-function uninitialize() {
-    gServer = null;
-}
-
-function renderTemplate(res, template, data) {
-    assert.strictEqual(typeof res, 'object');
-    assert.strictEqual(typeof template, 'string');
-    assert.strictEqual(typeof data, 'object');
-
-    settings.getCloudronName(function (error, cloudronName) {
-        if (error) {
-            console.error(error);
-            cloudronName = 'Cloudron';
-        }
-
-        // amend template properties, for example used in the header
-        data.title = data.title || 'Cloudron';
-        data.adminOrigin = settings.adminOrigin();
-        data.cloudronName = cloudronName;
-
-        res.render(template, data);
-    });
-}
-
-function sendErrorPageOrRedirect(req, res, message) {
-    assert.strictEqual(typeof req, 'object');
-    assert.strictEqual(typeof res, 'object');
-    assert.strictEqual(typeof message, 'string');
-
-    debug('sendErrorPageOrRedirect: returnTo %s.', req.query.returnTo, message);
-
-    if (typeof req.query.returnTo !== 'string') {
-        renderTemplate(res, 'error', {
-            message: message,
-            title: 'Cloudron Error'
-        });
-    } else {
-        var u = url.parse(req.query.returnTo);
-        if (!u.protocol || !u.host) {
-            return renderTemplate(res, 'error', {
-                message: 'Invalid request. returnTo query is not a valid URI. ' + message,
-                title: 'Cloudron Error'
-            });
-        }
-
-        res.redirect(util.format('%s//%s', u.protocol, u.host));
-    }
-}
-
-// use this instead of sendErrorPageOrRedirect(), in case we have a returnTo provided in the query, to avoid login loops
-// This usually happens when the OAuth client ID is wrong
-function sendError(req, res, message) {
-    assert.strictEqual(typeof req, 'object');
-    assert.strictEqual(typeof res, 'object');
-    assert.strictEqual(typeof message, 'string');
-
-    renderTemplate(res, 'error', {
-        message: message,
-        title: 'Cloudron Error'
-    });
-}
-
-// -> GET /api/v1/session/login
-function loginForm(req, res) {
-    if (typeof req.session.returnTo !== 'string') return sendErrorPageOrRedirect(req, res, 'Invalid login request. No returnTo provided.');
-
-    var u = url.parse(req.session.returnTo, true);
-    if (!u.query.client_id) return sendErrorPageOrRedirect(req, res, 'Invalid login request. No client_id provided.');
-
-    function render(applicationName, applicationLogo) {
-        var error = req.query.error || null;
-
-        renderTemplate(res, 'login', {
-            csrf: req.csrfToken(),
-            applicationName: applicationName,
-            applicationLogo: applicationLogo,
-            error: error,
-            username: settings.isDemo() ? constants.DEMO_USERNAME : '',
-            password: settings.isDemo() ? 'cloudron' : '',
-            title: applicationName + ' Login'
-        });
-    }
-
-    function renderBuiltIn() {
-        settings.getCloudronName(function (error, cloudronName) {
-            if (error) {
-                console.error(error);
-                cloudronName = 'Cloudron';
-            }
-
-            render(cloudronName, '/api/v1/cloudron/avatar');
-        });
-    }
-
-    clients.get(u.query.client_id, function (error, result) {
-        if (error) return sendError(req, res, 'Unknown OAuth client');
-
-        switch (result.type) {
-        case clients.TYPE_BUILT_IN: return renderBuiltIn();
-        case clients.TYPE_EXTERNAL: return render(result.appId, '/api/v1/cloudron/avatar');
-        default: break;
-        }
-
-        apps.get(result.appId, function (error, result) {
-            if (error) return sendErrorPageOrRedirect(req, res, 'Unknown Application for those OAuth credentials');
-
-            var applicationName = result.fqdn;
-            render(applicationName, '/api/v1/apps/' + result.id + '/icon');
-        });
-    });
-}
-
-// -> POST /api/v1/session/login
-function login(req, res) {
-    var returnTo = req.session.returnTo || req.query.returnTo;
-
-    var failureQuery = querystring.stringify({ error: 'Invalid username or password', returnTo: returnTo });
-    passport.authenticate('local', {
-        failureRedirect: '/api/v1/session/login?' + failureQuery
-    })(req, res, function (error) {
-        if (error) return res.redirect('/api/v1/session/login?' + failureQuery); // on some exception in the handlers
-
-        if (!req.user.ghost && req.user.twoFactorAuthenticationEnabled) {
-            if (!req.body.totpToken) {
-                let failureQuery = querystring.stringify({ error: 'A 2FA token is required', returnTo: returnTo });
-                return res.redirect('/api/v1/session/login?' + failureQuery);
-            }
-
-            let verified = speakeasy.totp.verify({ secret: req.user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
-            if (!verified) {
-                let failureQuery = querystring.stringify({ error: 'The 2FA token is invalid', returnTo: returnTo });
-                return res.redirect('/api/v1/session/login?' + failureQuery);
-            }
-        }
-
-        res.redirect(returnTo);
-    });
-}
-
-// -> GET /api/v1/session/logout
-function logout(req, res) {
-    function done() {
-        req.logout();
-
-        if (req.query && req.query.redirect) res.redirect(req.query.redirect);
-        else res.redirect('/');
-    }
-
-    if (!req.query.all) return done();
-
-    // find and destroy all login sessions by this user - this got rather complex quickly
-    req.sessionStore.list(function (error, result) {
-        if (error) {
-            console.error('Error listing sessions', error);
-            return done();
-        }
-
-        // WARNING fix this if we change the storage backend - Great stuff!
-        var sessionIds = result.map(function(s) { return s.replace('.json', ''); });
-
-        async.each(sessionIds, function (id, callback) {
-            req.sessionStore.get(id, function (error, result) {
-                if (error) {
-                    console.error(`Error getting session ${id}`, error);
-                    return callback();
-                }
-
-                // ignore empty or non passport sessions
-                if (!result || !result.passport || !result.passport.user) return callback();
-
-                // not this user
-                if (result.passport.user !== req.user.id) return callback();
-
-                req.sessionStore.destroy(id, function (error) {
-                    if (error) console.error(`Unable to destroy session ${id}`, error);
-                    callback();
-                });
-            });
-        }, done);
-    });
-}
-
-// Form to enter email address to send a password reset request mail
-// -> GET /api/v1/session/password/resetRequest.html
-function passwordResetRequestSite(req, res) {
-    var data = {
-        csrf: req.csrfToken(),
-        title: 'Password Reset'
-    };
-
-    renderTemplate(res, 'password_reset_request', data);
-}
-
-// This route is used for above form submission
-// -> POST /api/v1/session/password/resetRequest
-function passwordResetRequest(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.identifier !== 'string') return next(new HttpError(400, 'Missing identifier')); // email or username
-
-    debug('passwordResetRequest: email or username %s.', req.body.identifier);
-
-    users.resetPasswordByIdentifier(req.body.identifier, function (error) {
-        if (error && error.reason !== BoxError.NOT_FOUND) {
-            console.error(error);
-            return sendErrorPageOrRedirect(req, res, 'User not found');
-        }
-
-        res.redirect('/api/v1/session/password/sent.html');
-    });
-}
-
-// -> GET /api/v1/session/password/sent.html
-function passwordSentSite(req, res) {
-    renderTemplate(res, 'password_reset_sent', { title: 'Cloudron Password Reset' });
-}
-
-function renderAccountSetupSite(res, req, userObject, error) {
-    renderTemplate(res, 'account_setup', {
-        user: userObject,
-        error: error,
-        csrf: req.csrfToken(),
-        resetToken: req.query.reset_token || req.body.resetToken,
-        email: req.query.email || req.body.email,
-        title: 'Account Setup'
-    });
-}
-
-// -> GET /api/v1/session/account/setup.html
-function accountSetupSite(req, res) {
-    if (!req.query.reset_token) return sendError(req, res, 'Missing Reset Token');
-    if (!req.query.email) return sendError(req, res, 'Missing Email');
-
-    users.getByResetToken(req.query.email, req.query.reset_token, function (error, userObject) {
-        if (error) return sendError(req, res, 'Invalid Email or Reset Token');
-
-        renderAccountSetupSite(res, req, userObject, '');
-    });
-}
-
-// -> POST /api/v1/session/account/setup
-function accountSetup(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.email !== 'string') return next(new HttpError(400, 'Missing email'));
-    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
-    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
-    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'Missing username'));
-    if (typeof req.body.displayName !== 'string') return next(new HttpError(400, 'Missing displayName'));
-
-    debug(`acountSetup: for email ${req.body.email} with token ${req.body.resetToken}`);
-
-    users.getByResetToken(req.body.email, req.body.resetToken, function (error, userObject) {
-        if (error) return sendError(req, res, 'Invalid Reset Token');
-
-        var data = _.pick(req.body, 'username', 'displayName');
-        users.update(userObject.id, data, auditSource(req), function (error) {
-            if (error && error.reason === BoxError.ALREADY_EXISTS) return renderAccountSetupSite(res, req, userObject, 'Username already exists');
-            if (error && error.reason === BoxError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
-            if (error && error.reason === BoxError.NOT_FOUND) return renderAccountSetupSite(res, req, userObject, 'No such user');
-            if (error) return next(new HttpError(500, error));
-
-            userObject.username = req.body.username;
-            userObject.displayName = req.body.displayName;
-
-            // setPassword clears the resetToken
-            users.setPassword(userObject.id, req.body.password, function (error) {
-                if (error && error.reason === BoxError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
-                if (error) return next(new HttpError(500, error));
-
-                clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                    if (error) return next(new HttpError(500, error));
-
-                    res.redirect(`${settings.adminOrigin()}?accessToken=${result.accessToken}&expiresAt=${result.expires}`);
-                });
-            });
-        });
-    });
-}
-
-// -> GET /api/v1/session/password/reset.html
-function passwordResetSite(req, res, next) {
-    if (!req.query.email) return next(new HttpError(400, 'Missing email'));
-    if (!req.query.reset_token) return next(new HttpError(400, 'Missing reset_token'));
-
-    users.getByResetToken(req.query.email, req.query.reset_token, function (error, user) {
-        if (error) return next(new HttpError(401, 'Invalid email or reset token'));
-
-        renderTemplate(res, 'password_reset', {
-            user: user,
-            csrf: req.csrfToken(),
-            resetToken: req.query.reset_token,
-            email: req.query.email,
-            title: 'Password Reset'
-        });
-    });
-}
-
-// -> POST /api/v1/session/password/reset
-function passwordReset(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.email !== 'string') return next(new HttpError(400, 'Missing email'));
-    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
-    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
-
-    debug(`passwordReset: for ${req.body.email} with token ${req.body.resetToken}`);
-
-    users.getByResetToken(req.body.email, req.body.resetToken, function (error, userObject) {
-        if (error) return next(new HttpError(401, 'Invalid email or resetToken'));
-
-        if (!userObject.username) return next(new HttpError(401, 'No username set'));
-
-        // setPassword clears the resetToken
-        users.setPassword(userObject.id, req.body.password, function (error) {
-            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(406, error.message));
-            if (error) return next(new HttpError(500, error));
-
-            clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                if (error) return next(new HttpError(500, error));
-
-                res.redirect(`${settings.adminOrigin()}?accessToken=${result.accessToken}&expiresAt=${result.expires}`);
-            });
-        });
-    });
-}
-
-
-// The callback page takes the redirectURI and the authCode and redirects the browser accordingly
-//
-// -> GET /api/v1/session/callback
-function sessionCallback() {
-    return [
-        session.ensureLoggedIn('/api/v1/session/login'),
-        function (req, res) {
-            renderTemplate(res, 'callback', { callbackServer: req.query.redirectURI });
-        }
-    ];
-}
-
-// The authorization endpoint is the entry point for an OAuth login.
-//
-// Each app would start OAuth by redirecting the user to:
-//
-//    /api/v1/oauth/dialog/authorize?response_type=code&client_id=<clientId>&redirect_uri=<callbackURL>&scope=<ignored>
-//
-//  - First, this will ensure the user is logged in.
-//  - Then it will redirect the browser to the given <callbackURL> containing the authcode in the query
-//
-// -> GET /api/v1/oauth/dialog/authorize
-function authorization() {
-    return [
-        function (req, res, next) {
-            if (!req.query.redirect_uri) return sendErrorPageOrRedirect(req, res, 'Invalid request. redirect_uri query param is not set.');
-            if (!req.query.client_id) return sendErrorPageOrRedirect(req, res, 'Invalid request. client_id query param is not set.');
-            if (!req.query.response_type) return sendErrorPageOrRedirect(req, res, 'Invalid request. response_type query param is not set.');
-            if (req.query.response_type !== 'code' && req.query.response_type !== 'token') return sendErrorPageOrRedirect(req, res, 'Invalid request. Only token and code response types are supported.');
-
-            session.ensureLoggedIn('/api/v1/session/login?returnTo=' + req.query.redirect_uri)(req, res, next);
-        },
-        gServer.authorization({}, function (clientId, redirectURI, callback) {
-            debug('authorization: client %s with callback to %s.', clientId, redirectURI);
-
-            clients.get(clientId, function (error, client) {
-                if (error && error.reason === BoxError.NOT_FOUND) return callback(null, false);
-                if (error) return callback(error);
-
-                // ignore the origin passed into form the client, but use the one from the clientdb
-                var redirectPath = url.parse(redirectURI).path;
-                var redirectOrigin = client.redirectURI;
-
-                callback(null, client, '/api/v1/session/callback?redirectURI=' + encodeURIComponent(url.resolve(redirectOrigin, redirectPath)));
-            });
-        }),
-        function (req, res, next) {
-            // Handle our different types of oauth clients
-            var type = req.oauth2.client.type;
-
-            if (type === clients.TYPE_EXTERNAL || type === clients.TYPE_BUILT_IN) {
-                eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, req.oauth2.client.appId), { userId: req.oauth2.user.id, user: users.removePrivateFields(req.oauth2.user) });
-                return next();
-            }
-
-            apps.get(req.oauth2.client.appId, function (error, appObject) {
-                if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
-
-                apps.hasAccessTo(appObject, req.oauth2.user, function (error, access) {
-                    if (error) return sendError(req, res, 'Internal error');
-                    if (!access) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
-
-                    eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, appObject.id), { userId: req.oauth2.user.id, user: users.removePrivateFields(req.oauth2.user) });
-
-                    next();
-                });
-            });
-        },
-        gServer.decision({ loadTransaction: false })
-    ];
-}
-
-//  The token endpoint allows an OAuth client to exchange an authcode with an accesstoken.
-//
-//  Authcodes are obtained using the authorization endpoint. The route is authenticated by
-//  providing a Basic auth with clientID as username and clientSecret as password.
-//  An authcode is only good for one such exchange to an accesstoken.
-//
-// -> POST /api/v1/oauth/token
-function token() {
-    return [
-        passport.authenticate(['basic', 'oauth2-client-password'], { session: false }),
-        gServer.token(), // will call the token grant callback registered in initialize()
-        gServer.errorHandler()
-    ];
-}
-
-// Cross-site request forgery protection middleware for login form
-function csrf() {
-    return [
-        middleware.csrf(),
-        function (err, req, res, next) {
-            if (err.code !== 'EBADCSRFTOKEN') return next(err);
-
-            sendErrorPageOrRedirect(req, res, 'Form expired');
-        }
-    ];
-}

src/routes/profile.js

@@ -37,9 +37,9 @@ function get(req, res, next) {
         fallbackEmail: req.user.fallbackEmail,
         displayName: req.user.displayName,
         twoFactorAuthenticationEnabled: req.user.twoFactorAuthenticationEnabled,
-        admin: req.user.admin,
+        role: req.user.role,
         source: req.user.source,
-        avatarUrl: fs.existsSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id)) ? `${settings.adminOrigin()}/api/v1/profile/avatar/${req.user.id}` : `https://www.gravatar.com/avatar/${emailHash}.jpg?s=128&d=mm`
+        avatarUrl: fs.existsSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id)) ? `${settings.adminOrigin()}/api/v1/profile/avatar/${req.user.id}` : `https://www.gravatar.com/avatar/${emailHash}.jpg`
     }));
 }
 
@@ -53,7 +53,7 @@ function update(req, res, next) {
 
     var data = _.pick(req.body, 'email', 'fallbackEmail', 'displayName');
 
-    users.update(req.user.id, data, auditSource.fromRequest(req), function (error) {
+    users.update(req.user, data, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));
@@ -88,7 +88,7 @@ function changePassword(req, res, next) {
 
     if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'newPassword must be a string'));
 
-    users.setPassword(req.user.id, req.body.newPassword, function (error) {
+    users.setPassword(req.user, req.body.newPassword, function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));

src/routes/provision.js

@@ -9,14 +9,15 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    appstore = require('../appstore.js'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:routes/setup'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     provision = require('../provision.js'),
-    settings = require('../settings.js'),
-    superagent = require('superagent');
+    request = require('request'),
+    settings = require('../settings.js');
 
 function providerTokenAuth(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
@@ -24,11 +25,11 @@ function providerTokenAuth(req, res, next) {
     if (settings.provider() === 'ami') {
         if (typeof req.body.providerToken !== 'string' || !req.body.providerToken) return next(new HttpError(400, 'providerToken must be a non empty string'));
 
-        superagent.get('http://169.254.169.254/latest/meta-data/instance-id').timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return next(new HttpError(500, error));
-            if (result.statusCode !== 200) return next(new HttpError(500, 'Unable to get meta data'));
+        request.get('http://169.254.169.254/latest/meta-data/instance-id', { timeout: 30 * 1000 }, function (error, result) {
+            if (error) return next(new HttpError(422, `Network error getting EC2 metadata: ${error.message}`));
+            if (result.statusCode !== 200) return next(new HttpError(422, `Unable to get EC2 meta data. statusCode: ${result.statusCode}`));
 
-            if (result.text !== req.body.providerToken) return next(new HttpError(401, 'Invalid providerToken'));
+            if (result.body !== req.body.providerToken) return next(new HttpError(422, 'Instance ID does not match'));
 
             next();
         });
@@ -62,6 +63,8 @@ function setup(req, res, next) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
+
+        appstore.trackFinishedSetup(dnsConfig.domain);
     });
 }
 
@@ -116,5 +119,10 @@ function getStatus(req, res, next) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, status));
+
+        // check if Cloudron is not in setup state nor activated and let appstore know of the attempt
+        if (!status.activated && !status.setup.active && !status.restore.active) {
+            appstore.trackBeginSetup(status.provider);
+        }
     });
 }

src/routes/services.js

@@ -40,10 +40,11 @@ function configure(req, res, next) {
     assert.strictEqual(typeof req.params.service, 'string');
 
     if (typeof req.body.memory !== 'number') return next(new HttpError(400, 'memory must be a number'));
+    if (typeof req.body.memorySwap !== 'number') return next(new HttpError(400, 'memorySwap must be a number'));
 
     const data = {
         memory: req.body.memory,
-        memorySwap: req.body.memory * 2
+        memorySwap: req.body.memorySwap
     };
 
     addons.configureService(req.params.service, data, function (error) {

src/routes/settings.js

@@ -10,7 +10,6 @@ exports = module.exports = {
 var assert = require('assert'),
     backups = require('../backups.js'),
     BoxError = require('../boxerror.js'),
-    custom = require('../custom.js'),
     docker = require('../docker.js'),
     externalLdap = require('../externalldap.js'),
     HttpError = require('connect-lastmile').HttpError,
@@ -98,6 +97,34 @@ function setTimeZone(req, res, next) {
     });
 }
 
+function getFooter(req, res, next) {
+    settings.getFooter(function (error, footer) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { footer }));
+    });
+}
+
+function setFooter(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.footer !== 'string') return next(new HttpError(400, 'footer is required'));
+
+    settings.setFooter(req.body.footer, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, {}));
+    });
+}
+
+function getSupportConfig(req, res, next) {
+    settings.getSupportConfig(function (error, supportConfig) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, supportConfig));
+    });
+}
+
 function setCloudronAvatar(req, res, next) {
     assert.strictEqual(typeof req.files, 'object');
 
@@ -127,11 +154,6 @@ function getBackupConfig(req, res, next) {
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return next(BoxError.toHttpError(error));
 
-        // always send provider as it is used by the UI to figure if backups are disabled ('noop' backend)
-        if (!custom.spec().backups.configurable) {
-            return next(new HttpSuccess(200, { provider: backupConfig.provider }));
-        }
-
         next(new HttpSuccess(200, backups.removePrivateFields(backupConfig)));
     });
 }
@@ -139,8 +161,6 @@ function getBackupConfig(req, res, next) {
 function setBackupConfig(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (!custom.spec().backups.configurable) return next(new HttpError(405, 'feature disabled by admin'));
-
     if (typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider is required'));
     if (typeof req.body.retentionSecs !== 'number') return next(new HttpError(400, 'retentionSecs is required'));
     if (typeof req.body.intervalSecs !== 'number') return next(new HttpError(400, 'intervalSecs is required'));
@@ -225,8 +245,6 @@ function getDynamicDnsConfig(req, res, next) {
 function setDynamicDnsConfig(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (!custom.spec().domains.dynamicDns) return next(new HttpError(405, 'feature disabled by admin'));
-
     if (typeof req.body.enabled !== 'boolean') return next(new HttpError(400, 'enabled boolean is required'));
 
     settings.setDynamicDnsConfig(req.body.enabled, function (error) {
@@ -316,8 +334,12 @@ function get(req, res, next) {
     case settings.TIME_ZONE_KEY: return getTimeZone(req, res, next);
     case settings.CLOUDRON_NAME_KEY: return getCloudronName(req, res, next);
 
+    case settings.FOOTER_KEY: return getFooter(req, res, next);
+
     case settings.CLOUDRON_AVATAR_KEY: return getCloudronAvatar(req, res, next);
 
+    case settings.SUPPORT_CONFIG_KEY: return getSupportConfig(req, res, next);
+
     default: return next(new HttpError(404, 'No such setting'));
     }
 }
@@ -339,6 +361,8 @@ function set(req, res, next) {
     case settings.TIME_ZONE_KEY: return setTimeZone(req, res, next);
     case settings.CLOUDRON_NAME_KEY: return setCloudronName(req, res, next);
 
+    case settings.FOOTER_KEY: return setFooter(req, res, next);
+
     case settings.CLOUDRON_AVATAR_KEY: return setCloudronAvatar(req, res, next);
 
     default: return next(new HttpError(404, 'No such setting'));

src/routes/support.js

@@ -4,22 +4,35 @@ exports = module.exports = {
     createTicket: createTicket,
 
     getRemoteSupport: getRemoteSupport,
-    enableRemoteSupport: enableRemoteSupport
+    enableRemoteSupport: enableRemoteSupport,
+
+    canCreateTicket: canCreateTicket,
+    canEnableRemoteSupport: canEnableRemoteSupport
 };
 
 var appstore = require('../appstore.js'),
     assert = require('assert'),
-    custom = require('../custom.js'),
+    auditSource = require('../auditsource.js'),
+    constants = require('../constants.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    settings = require('../settings.js'),
     support = require('../support.js'),
     _ = require('underscore');
 
+function canCreateTicket(req, res, next) {
+    settings.getSupportConfig(function (error, supportConfig) {
+        if (error) return next(new HttpError(503, error.message));
+
+        if (!supportConfig.submitTickets) return next(new HttpError(405, 'feature disabled by admin'));
+
+        next();
+    });
+}
+
 function createTicket(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    if (!custom.spec().support.submitTickets) return next(new HttpError(405, 'feature disabled by admin'));
-
     const VALID_TYPES = [ 'feedback', 'ticket', 'app_missing', 'app_error', 'upgrade_request', 'email_error' ];
 
     if (typeof req.body.type !== 'string' || !req.body.type) return next(new HttpError(400, 'type must be string'));
@@ -29,21 +42,34 @@ function createTicket(req, res, next) {
     if (req.body.appId && typeof req.body.appId !== 'string') return next(new HttpError(400, 'appId must be string'));
     if (req.body.altEmail && typeof req.body.altEmail !== 'string') return next(new HttpError(400, 'altEmail must be string'));
 
-    appstore.createTicket(_.extend({ }, req.body, { email: req.user.email, displayName: req.user.displayName }), function (error) {
-        if (error) return next(new HttpError(503, `Error contacting cloudron.io: ${error.message}. Please email ${custom.spec().support.email}`));
+    settings.getSupportConfig(function (error, supportConfig) {
+        if (error) return next(new HttpError(503, `Error getting support config: ${error.message}`));
+        if (supportConfig.email !== constants.SUPPORT_EMAIL) return next(new HttpError(503, 'Sending to non-cloudron email not implemented yet'));
 
-        next(new HttpSuccess(201, { message: `An email for sent to ${custom.spec().support.email}. We will get back shortly!` }));
+        appstore.createTicket(_.extend({ }, req.body, { email: req.user.email, displayName: req.user.displayName }), auditSource.fromRequest(req), function (error, result) {
+            if (error) return next(new HttpError(503, `Error contacting cloudron.io: ${error.message}. Please email ${constants.SUPPORT_EMAIL}`));
+
+            next(new HttpSuccess(201, result));
+        });
+    });
+}
+
+function canEnableRemoteSupport(req, res, next) {
+    settings.getSupportConfig(function (error, supportConfig) {
+        if (error) return next(new HttpError(503, error.message));
+
+        if (!supportConfig.remoteSupport) return next(new HttpError(405, 'feature disabled by admin'));
+
+        next();
     });
 }
 
 function enableRemoteSupport(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (!custom.spec().support.remoteSupport) return next(new HttpError(405, 'feature disabled by admin'));
-
     if (typeof req.body.enable !== 'boolean') return next(new HttpError(400, 'enabled is required'));
 
-    support.enableRemoteSupport(req.body.enable, function (error) {
+    support.enableRemoteSupport(req.body.enable, auditSource.fromRequest(req), function (error) {
         if (error) return next(new HttpError(503, 'Error enabling remote support. Try running "cloudron-support --enable-ssh" on the server'));
 
         next(new HttpSuccess(202, {}));

src/routes/test/accesscontrol-test.js

@@ -1,97 +1,27 @@
 /* jslint node:true */
 /* global it:false */
 /* global describe:false */
-/* global before:false */
-/* global after:false */
 
 'use strict';
 
 var accesscontrol = require('../accesscontrol.js'),
     expect = require('expect.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    passport = require('passport');
+    HttpError = require('connect-lastmile').HttpError;
 
-describe('scopes middleware', function () {
-    var passportAuthenticateSave = null;
-
-    before(function () {
-        passportAuthenticateSave = passport.authenticate;
-        passport.authenticate = function () {
-            return function (req, res, next) { next(); };
-        };
+describe('access control middleware', function () {
+    describe('passwordAuth', function () {
+        // TBD
     });
 
-    after(function () {
-        passport.authenticate = passportAuthenticateSave;
+    describe('tokenAuth', function () {
+        // TBD
     });
 
-    it('fails due to empty scope in request', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { authorizedScopes: [ ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.be.a(HttpError);
-            done();
-        });
+    describe('authorize', function () {
+        // TBD
     });
 
-    it('fails due to wrong scope in request', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { authorizedScopes: [ 'foobar', 'something' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.be.a(HttpError);
-            done();
-        });
-    });
-
-    it('fails due to wrong scope in request', function (done) {
-        var mw = accesscontrol.scope('admin,users')[1];
-        var req = { authInfo: { authorizedScopes: [ 'foobar', 'admin' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.be.a(HttpError);
-            done();
-        });
-    });
-
-    it('succeeds with one requested scope and one provided scope', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { authorizedScopes: [ 'admin' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.not.be.ok();
-            done();
-        });
-    });
-
-    it('succeeds with one requested scope and two provided scopes', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { authorizedScopes: [ 'foobar', 'admin' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.not.be.ok();
-            done();
-        });
-    });
-
-    it('succeeds with two requested scope and two provided scopes', function (done) {
-        var mw = accesscontrol.scope('admin,foobar')[1];
-        var req = { authInfo: { authorizedScopes: [ 'foobar', 'admin' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.not.be.ok();
-            done();
-        });
-    });
-
-    it('succeeds with two requested scope and provided wildcard scope', function (done) {
-        var mw = accesscontrol.scope('admin,foobar')[1];
-        var req = { authInfo: { authorizedScopes: [ '*' ] } };
-
-        mw(req, null, function (error) {
-            expect(error).to.not.be.ok();
-            done();
-        });
+    describe('websocketAuth', function () {
+        // TBD
     });
 });

src/routes/test/apps-test.js

@@ -7,7 +7,6 @@
 let apps = require('../../apps.js'),
     async = require('async'),
     child_process = require('child_process'),
-    clients = require('../../clients.js'),
     constants = require('../../constants.js'),
     crypto = require('crypto'),
     database = require('../../database.js'),
@@ -29,6 +28,7 @@ let apps = require('../../apps.js'),
     settingsdb = require('../../settingsdb.js'),
     superagent = require('superagent'),
     tokendb = require('../../tokendb.js'),
+    tokens = require('../../tokens.js'),
     url = require('url');
 
 var SERVER_URL = 'http://localhost:' + constants.PORT;
@@ -36,8 +36,8 @@ var SERVER_URL = 'http://localhost:' + constants.PORT;
 const docker = new Docker({ socketPath: '/var/run/docker.sock' });
 
 // Test image information
-var TEST_IMAGE_REPO = 'cloudron/test';
-var TEST_IMAGE_TAG = '25.19.0';
+var TEST_IMAGE_REPO = 'docker.io/cloudron/io.cloudron.testapp';
+var TEST_IMAGE_TAG = '20200207-233155-725d9e015';
 var TEST_IMAGE = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
 
 const DOMAIN_0 = {
@@ -213,7 +213,7 @@ function startBox(done) {
                     token_1 = hat(8 * 32);
 
                     // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                    tokendb.add({ id: 'tid-1', accessToken: token_1, identifier: user_1_id, clientId: clients.ID_SDK, expires: Date.now() + 1000000, scope: 'apps', name: '' }, callback);
+                    tokendb.add({ id: 'tid-1', accessToken: token_1, identifier: user_1_id, clientId: tokens.ID_SDK, expires: Date.now() + 1000000, scope: 'apps', name: '' }, callback);
                 });
         },
 
@@ -633,22 +633,6 @@ describe('App API', function () {
             });
         });
 
-        it('oauth addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                expect(error).to.not.be.ok();
-
-                clients.getByAppIdAndType(APP_ID, clients.TYPE_OAUTH, function (error, client) {
-                    expect(error).to.not.be.ok();
-                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
-                    expect(client.clientSecret.length).to.be(256); // 32 hex chars (8 * 256 bits)
-                    expect(data.Config.Env).to.contain('CLOUDRON_OAUTH_CLIENT_ID=' + client.id);
-                    expect(data.Config.Env).to.contain('CLOUDRON_OAUTH_CLIENT_SECRET=' + client.clientSecret);
-                    done();
-                });
-            });
-        });
-
         it('installation - app can populate addons', function (done) {
             superagent.get(`http://localhost:${appEntry.httpPort}/populate_addons`).end(function (error, res) {
                 expect(!error).to.be.ok();
@@ -1570,6 +1554,34 @@ describe('App API', function () {
                     });
             });
         });
+
+        it('can restart app', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/restart')
+                .query({ access_token: token })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(202);
+                    taskId = res.body.taskId;
+                    done();
+                });
+        });
+
+        it('wait for app to restart', function (done) {
+            waitForTask(taskId, function () { setTimeout(done, 12000); }); // give app 12 seconds (to die and start)
+        });
+
+        it('did restart the app', function (done) {
+            apps.get(APP_ID, function (error, app) {
+                if (error) return done(error);
+
+                superagent.get('http://localhost:' + app.httpPort + APP_MANIFEST.healthCheckPath)
+                    .end(function (err, res) {
+                        if (res && res.statusCode === 200) return done();
+                        done(new Error('app is not running'));
+                    });
+            });
+        });
+
+
     });
 
     describe('uninstall', function () {

src/routes/test/clients-test.js

@@ -1,546 +0,0 @@
-'use strict';
-
-/* global it:false */
-/* global describe:false */
-/* global before:false */
-/* global after:false */
-
-var accesscontrol = require('../../accesscontrol.js'),
-    async = require('async'),
-    constants = require('../../constants.js'),
-    clients = require('../../clients.js'),
-    database = require('../../database.js'),
-    oauth2 = require('../oauth2.js'),
-    expect = require('expect.js'),
-    uuid = require('uuid'),
-    hat = require('../../hat.js'),
-    superagent = require('superagent'),
-    server = require('../../server.js');
-
-var SERVER_URL = 'http://localhost:' + constants.PORT;
-
-var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
-var token = null;
-
-function setup(done) {
-    async.series([
-        server.start,
-        database._clear,
-
-        function (callback) {
-            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
-                .query({ setupToken: 'somesetuptoken' })
-                .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
-                .end(function (error, result) {
-                    expect(result).to.be.ok();
-                    expect(result.statusCode).to.equal(201);
-
-                    // stash token for further use
-                    token = result.body.token;
-
-                    callback();
-                });
-        }
-    ], done);
-}
-
-function cleanup(done) {
-    database._clear(function (error) {
-        expect(error).to.not.be.ok();
-
-        server.stop(done);
-    });
-}
-
-describe('OAuth Clients API', function () {
-    describe('add', function () {
-        before(setup),
-        after(cleanup);
-
-        it('fails without token', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails without appId', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with empty appId', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: '', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails without scope', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: 'http://foobar.com' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with empty scope', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: '' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails without redirectURI', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with empty redirectURI', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: '', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with malformed redirectURI', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: 'foobar', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('fails with invalid name', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: '$"$%^45asdfasdfadf.adf.', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-        });
-
-        it('succeeds with dash', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'fo-1234-bar', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(201);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: accesscontrol.SCOPE_PROFILE })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(201);
-                    expect(result.body.id).to.be.a('string');
-                    expect(result.body.appId).to.be.a('string');
-                    expect(result.body.redirectURI).to.be.a('string');
-                    expect(result.body.clientSecret).to.be.a('string');
-                    expect(result.body.scope).to.be.a('string');
-                    expect(result.body.type).to.equal(clients.TYPE_EXTERNAL);
-
-                    done();
-                });
-        });
-    });
-
-    describe('get', function () {
-        var CLIENT_0 = {
-            id: '',
-            appId: 'someAppId-0',
-            redirectURI: 'http://some.callback0',
-            scope: accesscontrol.SCOPE_PROFILE
-        };
-
-        before(function (done) {
-            async.series([
-                setup,
-
-                function (callback) {
-                    superagent.post(SERVER_URL + '/api/v1/clients')
-                        .query({ access_token: token })
-                        .send({ appId: CLIENT_0.appId, redirectURI: CLIENT_0.redirectURI, scope: CLIENT_0.scope })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(201);
-
-                            CLIENT_0 = result.body;
-
-                            callback();
-                        });
-                }
-            ], done);
-        });
-
-        after(cleanup);
-
-        it('fails without token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-
-        it('fails with unknown id', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id.toUpperCase())
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-                    expect(result.body).to.eql(CLIENT_0);
-                    done();
-                });
-        });
-    });
-
-    describe('del', function () {
-        var CLIENT_0 = {
-            id: '',
-            appId: 'someAppId-0',
-            redirectURI: 'http://some.callback0',
-            scope: accesscontrol.SCOPE_PROFILE
-        };
-
-        var CLIENT_1 = {
-            id: '',
-            appId: 'someAppId-1',
-            redirectURI: 'http://some.callback1',
-            scope: accesscontrol.SCOPE_PROFILE,
-            type: clients.TYPE_OAUTH
-        };
-
-        before(function (done) {
-            async.series([
-                setup,
-
-                function (callback) {
-                    superagent.post(SERVER_URL + '/api/v1/clients')
-                        .query({ access_token: token })
-                        .send({ appId: CLIENT_0.appId, redirectURI: CLIENT_0.redirectURI, scope: CLIENT_0.scope })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(201);
-
-                            CLIENT_0 = result.body;
-
-                            callback();
-                        });
-                }
-            ], done);
-        });
-
-        after(cleanup);
-
-        it('fails without token', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-
-        it('fails with unknown id', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id.toUpperCase())
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(204);
-
-                    superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_0.id)
-                        .query({ access_token: token })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(404);
-
-                            done();
-                        });
-                });
-        });
-
-        it('fails for cid-webadmin', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(405);
-
-                    superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin')
-                        .query({ access_token: token })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(200);
-
-                            done();
-                        });
-                });
-        });
-
-        it('fails for addon auth client', function (done) {
-            clients.add(CLIENT_1.appId, CLIENT_1.type, CLIENT_1.redirectURI, CLIENT_1.scope, function (error, result) {
-                expect(error).to.equal(null);
-
-                CLIENT_1.id = result.id;
-
-                superagent.del(SERVER_URL + '/api/v1/clients/' + CLIENT_1.id)
-                    .query({ access_token: token })
-                    .end(function (error, result) {
-                        expect(result.statusCode).to.equal(405);
-
-                        superagent.get(SERVER_URL + '/api/v1/clients/' + CLIENT_1.id)
-                            .query({ access_token: token })
-                            .end(function (error, result) {
-                                expect(result.statusCode).to.equal(200);
-
-                                done();
-                            });
-                    });
-            });
-        });
-    });
-});
-
-describe('Clients', function () {
-    var USER_0 = {
-        userId: uuid.v4(),
-        username: 'someusername',
-        password: 'Strong#$%2345',
-        email: 'some@email.com',
-        admin: true,
-        salt: 'somesalt',
-        createdAt: (new Date()).toISOString(),
-        modifiedAt: (new Date()).toISOString(),
-        resetToken: hat(256)
-    };
-
-    // make csrf always succeed for testing
-    oauth2.csrf = function () {
-        return function (req, res, next) {
-            req.csrfToken = function () { return hat(256); };
-            next();
-        };
-    };
-
-    function setup2(done) {
-        async.series([
-            setup,
-
-            function (callback) {
-                superagent.get(SERVER_URL + '/api/v1/profile')
-                    .query({ access_token: token })
-                    .end(function (error, result) {
-                        expect(result).to.be.ok();
-                        expect(result.statusCode).to.eql(200);
-
-                        USER_0.id = result.body.id;
-
-                        callback();
-                    });
-            }
-        ], done);
-    }
-
-    describe('get', function () {
-        before(setup2);
-        after(cleanup);
-
-        it('fails due to missing token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to empty token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: '' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to wrong token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token.toUpperCase() })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-
-                    expect(result.body.clients.length).to.eql(3);
-
-                    done();
-                });
-        });
-    });
-
-    describe('get tokens by client', function () {
-        before(setup2);
-        after(cleanup);
-
-        it('fails due to missing token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to empty token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: '' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to wrong token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: token.toUpperCase() })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to unkown client', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/CID-WEBADMIN/tokens')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-
-                    expect(result.body.tokens.length).to.eql(1);
-                    expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
-
-                    done();
-                });
-        });
-    });
-
-    describe('delete tokens by client', function () {
-        before(setup2);
-        after(cleanup);
-
-        it('fails due to missing token', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to empty token', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: '' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to wrong token', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: token.toUpperCase() })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails due to unkown client', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/clients/CID-WEBADMIN/tokens')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-
-                    expect(result.body.tokens.length).to.eql(1);
-                    expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
-
-                    superagent.del(SERVER_URL + '/api/v1/clients/cid-webadmin/tokens')
-                        .query({ access_token: token })
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(204);
-
-                            // further calls with this token should not work
-                            superagent.get(SERVER_URL + '/api/v1/profile')
-                                .query({ access_token: token })
-                                .end(function (error, result) {
-                                    expect(result.statusCode).to.equal(401);
-                                    done();
-                                });
-                        });
-                });
-        });
-    });
-});

src/routes/test/cloudron-test.js

@@ -13,8 +13,9 @@ let async = require('async'),
     http = require('http'),
     nock = require('nock'),
     os = require('os'),
-    superagent = require('superagent'),
     server = require('../../server.js'),
+    speakeasy = require('speakeasy'),
+    superagent = require('superagent'),
     settings = require('../../settings.js'),
     tokendb = require('../../tokendb.js');
 
@@ -186,23 +187,239 @@ describe('Cloudron', function () {
                     expect(result.body.webServerOrigin).to.eql('https://cloudron.io');
                     expect(result.body.adminFqdn).to.eql(settings.adminFqdn());
                     expect(result.body.version).to.eql(constants.VERSION);
-                    expect(result.body.memory).to.eql(os.totalmem());
                     expect(result.body.cloudronName).to.be.a('string');
 
                     done();
                 });
         });
 
-        it('fails (non-admin)', function (done) {
+        it('succeeds (non-admin)', function (done) {
             superagent.get(SERVER_URL + '/api/v1/config')
                 .query({ access_token: token_1 })
                 .end(function (error, result) {
-                    expect(result.statusCode).to.equal(403);
+                    expect(result.statusCode).to.equal(200);
+                    expect(result.body.apiServerOrigin).to.eql('http://localhost:6060');
+                    expect(result.body.webServerOrigin).to.eql('https://cloudron.io');
+                    expect(result.body.adminFqdn).to.eql(settings.adminFqdn());
+                    expect(result.body.version).to.eql(constants.VERSION);
+                    expect(result.body.cloudronName).to.be.a('string');
                     done();
                 });
         });
     });
 
+    describe('login', function () {
+        before(function (done) {
+            async.series([
+                setup,
+                function (callback) {
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                        .query({ setupToken: 'somesetuptoken' })
+                        .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                        .end(function (error, result) {
+                            expect(result).to.be.ok();
+
+                            callback();
+                        });
+                },
+            ], done);
+        });
+
+        after(cleanup);
+
+        it('fails without body', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails without username', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails without password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails with empty username', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: '', password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails with empty password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME, password: '' })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('fails with unknown username', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME + USERNAME, password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(401);
+                    done();
+                });
+        });
+
+        it('fails with unknown email', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME + EMAIL, password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(401);
+                    done();
+                });
+        });
+
+        it('fails with wrong password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME, password: PASSWORD.toUpperCase() })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(401);
+                    done();
+                });
+        });
+
+        it('with username succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME, password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
+                    expect(result.body.accessToken).to.be.a('string');
+                    done();
+                });
+        });
+
+        it('with uppercase username succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
+                    expect(result.body.accessToken).to.be.a('string');
+                    done();
+                });
+        });
+
+        it('with email succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: EMAIL, password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
+                    expect(result.body.accessToken).to.be.a('string');
+                    done();
+                });
+        });
+
+        it('with uppercase email succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/login')
+                .send({ username: EMAIL.toUpperCase(), password: PASSWORD })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
+                    expect(result.body.accessToken).to.be.a('string');
+                    done();
+                });
+        });
+    });
+
+    describe('2fa login', function () {
+        var secret, accessToken;
+
+        before(function (done) {
+            async.series([
+                setup,
+                function (callback) {
+                    superagent.post(`${SERVER_URL}/api/v1/cloudron/activate`).query({ setupToken: 'somesetuptoken' }).send({ username: USERNAME, password: PASSWORD, email: EMAIL }).end(function (error) {
+                        callback(error);
+                    });
+                },
+                function (callback) {
+                    superagent.post(`${SERVER_URL}/api/v1/cloudron/login`).send({ username: USERNAME, password: PASSWORD }).end(function (error, result) {
+                        accessToken = result.body.accessToken;
+                        callback(error);
+                    });
+                },
+                function (callback) {
+                    superagent.post(`${SERVER_URL}/api/v1/profile/twofactorauthentication`).query({ access_token: accessToken }).end(function (error, result) {
+                        secret = result.body.secret;
+                        callback(error);
+                    });
+                },
+                function (callback) {
+                    var totpToken = speakeasy.totp({
+                        secret: secret,
+                        encoding: 'base32'
+                    });
+
+                    superagent.post(`${SERVER_URL}/api/v1/profile/twofactorauthentication/enable`).query({ access_token: accessToken }).send({ totpToken: totpToken }).end(function (error) {
+                        callback(error);
+                    });
+                }
+            ], done);
+        });
+
+        after(function (done) {
+            async.series([
+                function (callback) {
+                    superagent.post(`${SERVER_URL}/api/v1/profile/twofactorauthentication/disable`).query({ access_token: accessToken }).send({ password: PASSWORD }).end(function (error) {
+                        callback(error);
+                    });
+                },
+                cleanup
+            ], done);
+        });
+
+        it('fails due to missing token', function (done) {
+            superagent.post(`${SERVER_URL}/api/v1/cloudron/login`).send({ username: USERNAME, password: PASSWORD }).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails due to wrong token', function (done) {
+            superagent.post(`${SERVER_URL}/api/v1/cloudron/login`).send({ username: USERNAME, password: PASSWORD }).send({ totpToken: 'wrongtoken' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            var totpToken = speakeasy.totp({
+                secret: secret,
+                encoding: 'base32'
+            });
+
+            superagent.post(`${SERVER_URL}/api/v1/cloudron/login`).send({ username: USERNAME, password: PASSWORD }).send({ totpToken: totpToken }).end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(200);
+                expect(result.body).to.be.an(Object);
+                expect(result.body.accessToken).to.be.a('string');
+                done();
+            });
+        });
+    });
+
     describe('logs', function () {
         before(function (done) {
             async.series([
@@ -271,4 +488,75 @@ describe('Cloudron', function () {
             req.on('error', done);
         });
     });
+
+    describe('misc routes', function () {
+        before(function (done) {
+            async.series([
+                setup,
+
+                function (callback) {
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                        .query({ setupToken: 'somesetuptoken' })
+                        .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                        .end(function (error, result) {
+                            expect(result).to.be.ok();
+
+                            // stash token for further use
+                            token = result.body.token;
+
+                            callback();
+                        });
+                },
+
+                function (callback) {
+                    superagent.post(SERVER_URL + '/api/v1/users')
+                        .query({ access_token: token })
+                        .send({ username: USERNAME_1, email: EMAIL_1, invite: false })
+                        .end(function (error, result) {
+                            expect(result).to.be.ok();
+                            expect(result.statusCode).to.eql(201);
+
+                            token_1 = hat(8 * 32);
+                            userId_1 = result.body.id;
+
+                            // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+                            tokendb.add({ id: 'tid-1', accessToken: token_1, identifier: userId_1, clientId: 'test-client-id', expires: Date.now() + 100000, scope: 'cloudron', name: '' }, callback);
+                        });
+                }
+            ], done);
+        });
+
+        after(cleanup);
+
+        describe('memory', function () {
+            it('cannot get without token', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/memory')
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(401);
+                        done();
+                    });
+            });
+
+            it('succeeds (admin)', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/memory')
+                    .query({ access_token: token })
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(200);
+                        expect(result.body.memory).to.eql(os.totalmem());
+                        expect(result.body.swap).to.be.a('number');
+
+                        done();
+                    });
+            });
+
+            it('fails (non-admin)', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/memory')
+                    .query({ access_token: token_1 })
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(403);
+                        done();
+                    });
+            });
+        });
+    });
 });

src/routes/test/developer-test.js

@@ -1,248 +0,0 @@
-'use strict';
-
-/* jslint node:true */
-/* global it:false */
-/* global describe:false */
-/* global before:false */
-/* global after:false */
-
-var async = require('async'),
-    constants = require('../../constants.js'),
-    database = require('../../database.js'),
-    expect = require('expect.js'),
-    speakeasy = require('speakeasy'),
-    superagent = require('superagent'),
-    server = require('../../server.js');
-
-var SERVER_URL = 'http://localhost:' + constants.PORT;
-
-var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
-
-function setup(done) {
-    async.series([
-        server.start.bind(server),
-        database._clear
-    ], done);
-}
-
-function cleanup(done) {
-    database._clear(function (error) {
-        expect(error).to.not.be.ok();
-
-        server.stop(done);
-    });
-}
-
-describe('Developer API', function () {
-    describe('login', function () {
-        before(function (done) {
-            async.series([
-                setup,
-                function (callback) {
-                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
-                        .query({ setupToken: 'somesetuptoken' })
-                        .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
-                        .end(function (error, result) {
-                            expect(result).to.be.ok();
-
-                            callback();
-                        });
-                },
-            ], done);
-        });
-
-        after(cleanup);
-
-        it('fails without body', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails without username', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ password: PASSWORD })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails without password', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: USERNAME })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails with empty username', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: '', password: PASSWORD })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails with empty password', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: USERNAME, password: '' })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails with unknown username', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: USERNAME + USERNAME, password: PASSWORD })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails with unknown email', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: USERNAME + EMAIL, password: PASSWORD })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('fails with wrong password', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: USERNAME, password: PASSWORD.toUpperCase() })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('with username succeeds', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: USERNAME, password: PASSWORD })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
-                    expect(result.body.accessToken).to.be.a('string');
-                    done();
-                });
-        });
-
-        it('with uppercase username succeeds', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
-                    expect(result.body.accessToken).to.be.a('string');
-                    done();
-                });
-        });
-
-        it('with email succeeds', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: EMAIL, password: PASSWORD })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
-                    expect(result.body.accessToken).to.be.a('string');
-                    done();
-                });
-        });
-
-        it('with uppercase email succeeds', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: EMAIL.toUpperCase(), password: PASSWORD })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
-                    expect(result.body.accessToken).to.be.a('string');
-                    done();
-                });
-        });
-    });
-
-    describe('2fa login', function () {
-        var secret, accessToken;
-
-        before(function (done) {
-            async.series([
-                setup,
-                function (callback) {
-                    superagent.post(`${SERVER_URL}/api/v1/cloudron/activate`).query({ setupToken: 'somesetuptoken' }).send({ username: USERNAME, password: PASSWORD, email: EMAIL }).end(function (error) {
-                        callback(error);
-                    });
-                },
-                function (callback) {
-                    superagent.post(`${SERVER_URL}/api/v1/developer/login`).send({ username: USERNAME, password: PASSWORD }).end(function (error, result) {
-                        accessToken = result.body.accessToken;
-                        callback(error);
-                    });
-                },
-                function (callback) {
-                    superagent.post(`${SERVER_URL}/api/v1/profile/twofactorauthentication`).query({ access_token: accessToken }).end(function (error, result) {
-                        secret = result.body.secret;
-                        callback(error);
-                    });
-                },
-                function (callback) {
-                    var totpToken = speakeasy.totp({
-                        secret: secret,
-                        encoding: 'base32'
-                    });
-
-                    superagent.post(`${SERVER_URL}/api/v1/profile/twofactorauthentication/enable`).query({ access_token: accessToken }).send({ totpToken: totpToken }).end(function (error) {
-                        callback(error);
-                    });
-                }
-            ], done);
-        });
-
-        after(function (done) {
-            async.series([
-                function (callback) {
-                    superagent.post(`${SERVER_URL}/api/v1/profile/twofactorauthentication/disable`).query({ access_token: accessToken }).send({ password: PASSWORD }).end(function (error) {
-                        callback(error);
-                    });
-                },
-                cleanup
-            ], done);
-        });
-
-        it('fails due to missing token', function (done) {
-            superagent.post(`${SERVER_URL}/api/v1/developer/login`).send({ username: USERNAME, password: PASSWORD }).end(function (error, result) {
-                expect(result.statusCode).to.equal(401);
-                done();
-            });
-        });
-
-        it('fails due to wrong token', function (done) {
-            superagent.post(`${SERVER_URL}/api/v1/developer/login`).send({ username: USERNAME, password: PASSWORD }).send({ totpToken: 'wrongtoken' }).end(function (error, result) {
-                expect(result.statusCode).to.equal(401);
-                done();
-            });
-        });
-
-        it('succeeds', function (done) {
-            var totpToken = speakeasy.totp({
-                secret: secret,
-                encoding: 'base32'
-            });
-
-            superagent.post(`${SERVER_URL}/api/v1/developer/login`).send({ username: USERNAME, password: PASSWORD }).send({ totpToken: totpToken }).end(function (error, result) {
-                expect(error).to.be(null);
-                expect(result.statusCode).to.equal(200);
-                expect(result.body).to.be.an(Object);
-                expect(result.body.accessToken).to.be.a('string');
-                done();
-            });
-        });
-    });
-});

src/routes/test/domains-test.js

@@ -213,49 +213,6 @@ describe('Domains API', function () {
         });
     });
 
-    describe('locked', function () {
-        before(function (done) {
-            domaindb.update(DOMAIN_0.domain, { locked: true }, done);
-        });
-
-        after(function (done) {
-            domaindb.update(DOMAIN_0.domain, { locked: false }, done);
-        });
-
-        it('can list the domains', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/domains')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-                    expect(result.body.domains).to.be.an(Array);
-                    expect(result.body.domains.length).to.equal(2);
-
-                    expect(result.body.domains[0].domain).to.equal(DOMAIN_0.domain);
-                    expect(result.body.domains[1].domain).to.equal(DOMAIN_1.domain);
-
-                    done();
-                });
-        });
-
-        it('cannot get locked domain', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/domains/' + DOMAIN_0.domain)
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(423);
-                    done();
-                });
-        });
-
-        it('cannot delete locked domain', function (done) {
-            superagent.delete(SERVER_URL + '/api/v1/domains/' + DOMAIN_0.domain)
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(423);
-                    done();
-                });
-        });
-    });
-
     describe('delete', function () {
         it('fails for non-existing domain', function (done) {
             superagent.delete(SERVER_URL + '/api/v1/domains/' + DOMAIN_0.domain + DOMAIN_0.domain)

src/routes/test/eventlog-test.js

@@ -6,8 +6,7 @@
 
 'use strict';
 
-var accesscontrol = require('../../accesscontrol.js'),
-    async = require('async'),
+var async = require('async'),
     constants = require('../../constants.js'),
     database = require('../../database.js'),
     eventlogdb = require('../../eventlogdb.js'),
@@ -59,7 +58,7 @@ function setup(done) {
         function (callback) {
             superagent.post(SERVER_URL + '/api/v1/users')
                 .query({ access_token: token })
-                .send({ username: 'nonadmin', email: 'notadmin@server.test', invite: false })
+                .send({ username: 'nonadmin', email: 'notadmin@server.test' })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(201);
 
@@ -73,7 +72,7 @@ function setup(done) {
             token_1 = hat(8 * 32);
 
             // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-            tokendb.add({ id: 'tid-0', accessToken: token_1, identifier: USER_1_ID, clientId: 'test-client-id', expires: Date.now() + 100000, scope: accesscontrol.SCOPE_PROFILE, name: '' }, callback);
+            tokendb.add({ id: 'tid-0', accessToken: token_1, identifier: USER_1_ID, clientId: 'test-client-id', expires: Date.now() + 100000, scope: 'unused', name: '' }, callback);
         },
 
         function (callback) {
@@ -200,7 +199,7 @@ describe('Eventlog API', function () {
                 .query({ access_token: token, page: 1, per_page: 10, search: EMAIL })
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(200);
-                    expect(result.body.eventlogs.length).to.equal(2);
+                    expect(result.body.eventlogs.length).to.equal(1);
 
                     done();
                 });

src/routes/test/groups-test.js

@@ -6,8 +6,7 @@
 
 'use strict';
 
-var accesscontrol = require('../../accesscontrol.js'),
-    async = require('async'),
+var async = require('async'),
     constants = require('../../constants.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
@@ -67,7 +66,7 @@ function setup(done) {
                     userId_1 = result.body.id;
 
                     // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                    tokendb.add({ id: 'tid-1', accessToken: token_1, identifier: userId_1, clientId: 'test-client-id', expires: Date.now() + 100000, scope: accesscontrol.SCOPE_PROFILE, name: '' }, callback);
+                    tokendb.add({ id: 'tid-1', accessToken: token_1, identifier: userId_1, clientId: 'test-client-id', expires: Date.now() + 100000, scope: 'unused', name: '' }, callback);
                 });
         }
     ], done);

src/routes/test/profile-test.js

@@ -6,8 +6,7 @@
 
 'use strict';
 
-var accesscontrol = require('../../accesscontrol.js'),
-    constants = require('../../constants.js'),
+var constants = require('../../constants.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
     hat = require('../../hat.js'),
@@ -110,7 +109,7 @@ describe('Profile API', function () {
             var token = hat(8 * 32);
             var expires = Date.now() - 2000; // 1 sec
 
-            tokendb.add({ id: 'tid-3', accessToken: token, identifier: user_0.id, clientId: null, expires: expires, scope: accesscontrol.SCOPE_PROFILE, name: 'fromtest' }, function (error) {
+            tokendb.add({ id: 'tid-3', accessToken: token, identifier: user_0.id, clientId: null, expires: expires, scope: 'unused', name: 'fromtest' }, function (error) {
                 expect(error).to.not.be.ok();
 
                 superagent.get(SERVER_URL + '/api/v1/profile').query({ access_token: token }).end(function (error, result) {

src/routes/test/server-test.js

@@ -148,7 +148,7 @@ describe('REST API', function () {
             });
     });
 
-    it('dns setup twice succeeds', function (done) {
+    xit('dns setup twice succeeds', function (done) {
         superagent.post(SERVER_URL + '/api/v1/cloudron/setup')
             .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, DOMAIN, config: {} }, tlsConfig: { provider: 'fallback' } })
             .end(function (error, result) {