skip verification for prereleases because we remove it from release.json

This allows one to easily add "dev.staging@domain.com" etc without having to create
yet another domain. This plays well with the concept that we have a
mail domain for every domain. So we get mails from @domain.com working for
these subdomain installations.

CHANGES

@@ -1294,3 +1294,42 @@
 * Allow setting app visibility for non-SSO apps
 * Add Clone UI
 
+[2.3.2]
+* Fix issue where multi-db apps were not provisioned correctly
+* Improve setup, restore views to have field labels
+
+[2.4.0]
+* Use custom logging backend to have more control over log rotation
+* Make user explicitly confirm that fs backup dir is on external storage
+* Update node to 8.11.2
+* Update docker to 18.03.1
+* Fix docker exec terminal resize issue
+* Make the mailbox name follow the apps new location, if the user did not set it explicitly
+* Add backups view
+
+[2.4.1]
+* Use custom logging backend to have more control over log rotation
+* Mail logs and box logs UI
+* Make user explicitly confirm that fs backup dir is on external storage
+* Update node to 8.11.2
+* Update docker to 18.03.1
+* Fix docker exec terminal resize issue
+* Make the mailbox name follow the apps new location, if the user did not set it explicitly
+* Add backups view
+
+[3.0.0]
+* Support alternate app domains with redirects
+* Allow hyphen in mailbox names
+* Fix issue where the UI timesout when relay server is not reachable
+* Add support for personal spaces
+* Add UI to edit users in the groups dialog
+* Add UI to set groups when creating a user
+* Open logs and terminal in a new tab instead of a window
+* Add button to view backup logs
+* Add Mailjet mail relay support
+* Encryption support for incremental backups
+* Display restore errors in the UI
+* Update Haraka to 2.8.19
+* GPG verify releases
+* Allow subdomains in location field
+

baseimage/initializeBaseUbuntuImage.sh

@@ -61,7 +61,7 @@ echo "==> Installing Docker"
 mkdir -p /etc/systemd/system/docker.service.d
 echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
 
-curl -sL https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_17.09.0~ce-0~ubuntu_amd64.deb -o /tmp/docker.deb
+curl -sL https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_18.03.1~ce-0~ubuntu_amd64.deb -o /tmp/docker.deb
 # apt install with install deps (as opposed to dpkg -i)
 apt install -y /tmp/docker.deb
 rm /tmp/docker.deb

migrations/20180511222705-apps-add-owner-id.js

@@ -0,0 +1,28 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN ownerId VARCHAR(128)'),
+        function (next) {
+            db.all('SELECT id FROM users ORDER BY createdAt LIMIT 1', [ ], function (error, results) {
+                if (error || results.length === 0) return next(error);
+
+                var ownerId = results[0].id;
+                db.runSql('UPDATE apps SET ownerId=?', [ ownerId ], next);
+            });
+        },
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY ownerId VARCHAR(128) NOT NULL'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD CONSTRAINT apps_owner_constraint FOREIGN KEY(ownerId) REFERENCES users(id)'),
+        db.runSql.bind(db, 'COMMIT'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN ownerId', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20180627010633-apps-add-ts.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN ts ', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20180628162142-subdomains-create-table.js

@@ -0,0 +1,25 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    var cmd = 'CREATE TABLE IF NOT EXISTS subdomains(' +
+                'appId VARCHAR(128) NOT NULL,' +
+                'domain VARCHAR(128) NOT NULL,' +
+                'subdomain VARCHAR(128) NOT NULL,' +
+                'type VARCHAR(128) NOT NULL,' +
+                'dnsRecordId VARCHAR(512),' +
+                'FOREIGN KEY(domain) REFERENCES domains(domain),' +
+                'FOREIGN KEY(appId) REFERENCES apps(id),' +
+                'UNIQUE (subdomain, domain)) CHARACTER SET utf8 COLLATE utf8_bin';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE subdomains', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20180628163616-migrate-app-subdomains.js

@@ -0,0 +1,28 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * from apps', [ ], function (error, results) {
+        if (error) return done(error);
+
+        var queries = [
+            db.runSql.bind(db, 'START TRANSACTION;')
+        ];
+
+        results.forEach(function (app) {
+            queries.push(db.runSql.bind(db, 'INSERT INTO subdomains (appId, domain, subdomain, type, dnsRecordId) VALUES (?, ?, ?, ?, ?)', [ app.id, app.domain, app.location, 'primary', app.dnsRecordId ]));
+        });
+
+        queries.push(db.runSql.bind(db, 'COMMIT'));
+
+        async.series(queries, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE FROM subdomains', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20180629090543-remove-app-location-and-domain.js

@@ -0,0 +1,41 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP INDEX location_domain_unique_index, DROP FOREIGN KEY apps_domain_constraint, DROP COLUMN domain, DROP COLUMN location, DROP COLUMN dnsRecordId', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.all('SELECT * from subdomains WHERE type = ?', [ 'primary' ], function (error, results) {
+        if (error) return callback(error);
+
+        var cmd = 'ALTER TABLE apps'
+        + ' ADD COLUMN location VARCHAR(128),'
+        + ' ADD COLUMN domain VARCHAR(128),'
+        + ' ADD COLUMN dnsRecordId VARCHAR(512)';
+
+        db.runSql(cmd, function (error) {
+            if (error) return callback(error);
+
+            var queries = [ db.runSql.bind(db, 'START TRANSACTION;') ];
+            results.forEach(function (d) {
+                queries.push(db.runSql.bind(db, 'UPDATE apps SET domain = ?, location = ?, dnsRecordId = ? WHERE id = ?', [ d.domain, d.subdomain, d.appId, d.dnsRecordId ]));
+            });
+            queries.push(db.runSql.bind(db, 'COMMIT'));
+
+            async.series(queries, function (error) {
+                if (error) return callback(error);
+
+                var cmd = 'ALTER TABLE apps'
+                + ' ADD CONSTRAINT apps_domain_constraint FOREIGN KEY(domain) REFERENCES domains(domain),'
+                + ' ADD UNIQUE location_domain_unique_index (location, domain)';
+
+                db.runSql(cmd, callback);
+            });
+        });
+    });
+};

migrations/20180629201251-remove-subdomains-dnsRecordId.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains DROP COLUMN dnsRecordId', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains ADD COLUMN dnsRecordId VARCHAR(512)', function (error) {
+        if (error) return callback(error);
+        callback();
+    });
+};

migrations/20180726213634-users-add-admin.js

@@ -0,0 +1,34 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN admin BOOLEAN DEFAULT 0', function (error) {
+        if (error) return callback(error);
+
+        db.all('SELECT userId FROM groupMembers WHERE groupId=?', [ 'admin' ], function (error, results) {
+            if (error) return callback(error);
+
+            if (results.length === 0) return callback();
+
+            async.eachSeries(results, function (result, iteratorDone) {
+                db.runSql('UPDATE users SET admin=1 WHERE id=?', [ result.userId ], iteratorDone);
+            }, function (error) {
+                if (error) return callback(error);
+
+                async.series([
+                    db.runSql.bind(db, 'DELETE FROM groupMembers WHERE groupId=?', [ 'admin' ]),
+                    db.runSql.bind(db, 'DELETE FROM groups WHERE id=?', [ 'admin' ])
+                ], callback);
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN admin', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/schema.sql

@@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS users(
     fallbackEmail VARCHAR(512) DEFAULT "",
     twoFactorAuthenticationSecret VARCHAR(128) DEFAULT "",
     twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
+    admin BOOLEAN DEFAULT false,
 
     PRIMARY KEY(id));
 
@@ -68,10 +69,10 @@ CREATE TABLE IF NOT EXISTS apps(
     httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
     location VARCHAR(128) NOT NULL,
     domain VARCHAR(128) NOT NULL,
-    dnsRecordId VARCHAR(512), // tracks any id that we got back to track dns updates
     accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
-    createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updatedAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    creationTime TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP, // when the app was installed
+    updateTime TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,   // when the last app update was done
+    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, // when this db record was updated (useful for UI caching)
     memoryLimit BIGINT DEFAULT 0,
     xFrameOptions VARCHAR(512),
     sso BOOLEAN DEFAULT 1, // whether user chose to enable SSO
@@ -84,7 +85,9 @@ CREATE TABLE IF NOT EXISTS apps(
     oldConfigJson TEXT, // used to pass old config to apptask (configure, restore)
     updateConfigJson TEXT, // used to pass new config to apptask (update)
 
-    FOREIGN KEY(domain) REFERENCES domains(domain),
+    ownerId VARCHAR(128),
+
+    FOREIGN KEY(ownerId) REFERENCES users(id),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
@@ -176,3 +179,14 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     FOREIGN KEY(domain) REFERENCES mail(domain),
     UNIQUE (name, domain));
 
+CREATE TABLE IF NOT EXISTS subdomains(
+    appId VARCHAR(128) NOT NULL,
+    domain VARCHAR(128) NOT NULL,
+    subdomain VARCHAR(128) NOT NULL,
+    type VARCHAR(128) NOT NULL,
+
+    FOREIGN KEY(domain) REFERENCES domains(domain),
+    FOREIGN KEY(appId) REFERENCES apps(id),
+    UNIQUE (subdomain, domain))
+
+    CHARACTER SET utf8 COLLATE utf8_bin;

package.json

@@ -14,12 +14,12 @@
     "node": ">=4.0.0 <=4.1.1"
   },
   "dependencies": {
-    "@google-cloud/dns": "^0.7.1",
-    "@google-cloud/storage": "^1.6.0",
+    "@google-cloud/dns": "^0.7.2",
+    "@google-cloud/storage": "^1.7.0",
     "@sindresorhus/df": "^2.1.0",
-    "async": "^2.6.0",
-    "aws-sdk": "^2.201.0",
-    "body-parser": "^1.18.2",
+    "async": "^2.6.1",
+    "aws-sdk": "^2.253.1",
+    "body-parser": "^1.18.3",
     "cloudron-manifestformat": "^2.11.0",
     "connect-ensure-login": "^0.1.1",
     "connect-lastmile": "^1.0.2",
@@ -28,24 +28,23 @@
     "cookie-session": "^1.3.2",
     "cron": "^1.3.0",
     "csurf": "^1.6.6",
-    "db-migrate": "^0.10.5",
+    "db-migrate": "^0.11.1",
     "db-migrate-mysql": "^1.1.10",
     "debug": "^3.1.0",
-    "dockerode": "^2.5.4",
-    "ejs": "^2.5.7",
-    "ejs-cli": "^2.0.0",
-    "express": "^4.16.2",
+    "dockerode": "^2.5.5",
+    "ejs": "^2.6.1",
+    "ejs-cli": "^2.0.1",
+    "express": "^4.16.3",
     "express-session": "^1.15.6",
-    "hat": "0.0.3",
     "json": "^9.0.3",
     "ldapjs": "^1.0.2",
     "lodash.chunk": "^4.2.0",
-    "mime": "^2.2.0",
-    "moment-timezone": "^0.5.14",
+    "mime": "^2.3.1",
+    "moment-timezone": "^0.5.17",
     "morgan": "^1.9.0",
-    "multiparty": "^4.1.2",
+    "multiparty": "^4.1.4",
     "mysql": "^2.15.0",
-    "nodemailer": "^4.6.0",
+    "nodemailer": "^4.6.5",
     "nodemailer-smtp-transport": "^2.7.4",
     "oauth2orize": "^1.11.0",
     "once": "^1.3.2",
@@ -55,40 +54,39 @@
     "passport-http-bearer": "^1.0.1",
     "passport-local": "^1.0.0",
     "passport-oauth2-client-password": "^0.1.2",
-    "password-generator": "^2.2.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
     "qrcode": "^1.2.0",
-    "recursive-readdir": "^2.2.1",
-    "request": "^2.83.0",
-    "s3-block-read-stream": "^0.2.0",
+    "readdirp": "^2.1.0",
+    "request": "^2.87.0",
+    "rimraf": "^2.6.2",
+    "s3-block-read-stream": "^0.5.0",
     "safetydance": "^0.7.1",
     "semver": "^5.5.0",
-    "showdown": "^1.8.2",
+    "showdown": "^1.8.6",
     "speakeasy": "^2.0.0",
     "split": "^1.0.0",
-    "superagent": "^3.8.1",
-    "supererror": "^0.7.1",
-    "tar-fs": "^1.16.0",
-    "tar-stream": "^1.5.5",
+    "superagent": "^3.8.3",
+    "supererror": "^0.7.2",
+    "tar-fs": "^1.16.2",
+    "tar-stream": "^1.6.1",
     "tldjs": "^2.3.1",
-    "underscore": "^1.7.0",
+    "underscore": "^1.9.1",
     "uuid": "^3.2.1",
     "valid-url": "^1.0.9",
-    "validator": "^9.4.1",
-    "ws": "^3.3.3"
+    "validator": "^10.3.0",
+    "ws": "^5.2.0"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.3.2",
     "istanbul": "*",
     "js2xmlparser": "^3.0.0",
-    "mocha": "^5.0.1",
+    "mocha": "^5.2.0",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
     "nock": "^9.0.14",
     "node-sass": "^4.6.1",
-    "readdirp": "https://registry.npmjs.org/readdirp/-/readdirp-2.1.0.tgz",
-    "rimraf": "^2.6.2"
+    "recursive-readdir": "^2.2.2"
   },
   "scripts": {
     "migrate_local": "DATABASE_URL=mysql://root:@localhost/box node_modules/.bin/db-migrate up",

scripts/cloudron-activate

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"
+
+function get_status() {
+    key="$1"
+    if status=$($curl -q -f "http://localhost:3000/api/v1/cloudron/status" 2>/dev/null); then
+        currentValue=$(echo "${status}" | python3 -c 'import sys, json; print(json.dumps(json.load(sys.stdin)[sys.argv[1]]))' "${key}")
+        echo "${currentValue}"
+        return 0
+    fi
+
+    return 1
+}
+
+function wait_for_status() {
+    key="$1"
+    expectedValue="$2"
+
+    echo "wait_for_status: $key to be $expectedValue"
+    while true; do
+        if currentValue=$(get_status "${key}"); then
+            echo "wait_for_status: $key is current: $currentValue expecting: $expectedValue"
+            if [[ "${currentValue}" == $expectedValue ]]; then
+                break
+            fi
+        fi
+        sleep 3
+    done
+}
+
+domain=""
+domainProvider=""
+domainConfigJson="{}"
+domainTlsProvider="letsencrypt-prod"
+adminUsername="superadmin"
+adminPassword="Secret123#"
+adminEmail="admin@server.local"
+appstoreUserId=""
+appstoreToken=""
+backupDir="/var/backups"
+
+args=$(getopt -o "" -l "domain:,domain-provider:,domain-tls-provider:,admin-username:,admin-password:,admin-email:,appstore-user:,appstore-token:,backup-dir:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --domain) domain="$2"; shift 2;;
+    --domain-provider) domainProvider="$2"; shift 2;;
+    --domain-tls-provider) domainTlsProvider="$2"; shift 2;;
+    --admin-username) adminUsername="$2"; shift 2;;
+    --admin-password) adminPassword="$2"; shift 2;;
+    --admin-email) adminEmail="$2"; shift 2;;
+    --appstore-user) appstoreUser="$2"; shift 2;;
+    --appstore-token) appstoreToken="$2"; shift 2;;
+    --backup-dir) backupDir="$2"; shift 2;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+echo "=> Waiting for cloudron to be ready"
+wait_for_status "version" '*'
+
+if [[ $(get_status "webadminStatus") != *'"tls": true'* ]]; then
+    echo "=> Domain setup"
+    dnsSetupData=$(printf '{ "domain": "%s", "adminFqdn": "%s", "provider": "%s", "config": %s, "tlsConfig": { "provider": "%s" } }' "${domain}" "my.${domain}" "${domainProvider}" "$domainConfigJson" "${domainTlsProvider}")
+
+    if ! $curl -X POST -H "Content-Type: application/json" -d "${dnsSetupData}" http://localhost:3000/api/v1/cloudron/dns_setup; then
+        echo "DNS Setup Failed"
+        exit 1
+    fi
+
+    wait_for_status "webadminStatus" '*"tls": true*'
+else
+    echo "=> Skipping Domain setup"
+fi
+
+activationData=$(printf '{"username": "%s", "password":"%s", "email": "%s" }' "${adminUsername}" "${adminPassword}" "${adminEmail}")
+if [[ $(get_status "activated") == "false" ]]; then
+    echo "=> Activating"
+
+    if ! activationResult=$($curl -X POST -H "Content-Type: application/json" -d "${activationData}" http://localhost:3000/api/v1/cloudron/activate); then
+        echo "Failed to activate with ${activationData}: ${activationResult}"
+        exit 1
+    fi
+
+    wait_for_status "activated" "true"
+else
+    echo "=> Skipping Activation"
+fi
+
+echo "=> Getting token"
+if ! activationResult=$($curl -X POST -H "Content-Type: application/json" -d "${activationData}" http://localhost:3000/api/v1/developer/login); then
+    echo "Failed to login with ${activationData}: ${activationResult}"
+    exit 1
+fi
+
+accessToken=$(echo "${activationResult}" | python3 -c 'import sys, json; print(json.load(sys.stdin)[sys.argv[1]])' "accessToken")
+
+echo "=> Setting up App Store account with accessToken ${accessToken}"
+appstoreData=$(printf '{"userId":"%s", "token":"%s" }' "${appstoreUser}" "${appstoreToken}")
+
+if ! appstoreResult=$($curl -X POST -H "Content-Type: application/json" -d "${appstoreData}" "http://localhost:3000/api/v1/settings/appstore_config?access_token=${accessToken}"); then
+    echo "Failed to setup Appstore account with ${appstoreData}: ${appstoreResult}"
+    exit 1
+fi
+
+echo "=> Setting up Backup Directory with accessToken ${accessToken}"
+backupData=$(printf '{"provider":"filesystem", "key":"", "backupFolder":"%s", "retentionSecs": 864000, "format": "tgz", "externalDisk": true}' "${backupDir}")
+
+chown -R yellowtent:yellowtent "${backupDir}"
+
+if ! backupResult=$($curl -X POST -H "Content-Type: application/json" -d "${backupData}" "http://localhost:3000/api/v1/settings/backup_config?access_token=${accessToken}"); then
+    echo "Failed to setup backup configuration with ${backupDir}: ${backupResult}"
+    exit 1
+fi
+
+echo "=> Done!"
+

scripts/cloudron-setup

@@ -36,6 +36,11 @@ if [[ "${disk_size_gb}" -lt "${MINIMUM_DISK_SIZE_GB}" ]]; then
     exit 1
 fi
 
+if systemctl -q is-active box; then
+    echo "Error: Cloudron is already installed. To reinstall, start afresh"
+    exit 1
+fi
+
 initBaseImage="true"
 # provisioning data
 provider=""

scripts/createReleaseTarball

@@ -34,8 +34,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v8.9.3" ]]; then
-    echo "This script requires node 8.9.3"
+if [[ "$(node --version)" != "v8.11.2" ]]; then
+    echo "This script requires node 8.11.2"
     exit 1
 fi
 

scripts/installer.sh

@@ -35,11 +35,11 @@ while true; do
 done
 
 echo "==> installer: updating docker"
-if [[ $(docker version --format {{.Client.Version}}) != "18.03.0-ce" ]]; then
-    $curl -sL https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_18.03.0~ce-0~ubuntu_amd64.deb -o /tmp/docker.deb
+if [[ $(docker version --format {{.Client.Version}}) != "18.03.1-ce" ]]; then
+    $curl -sL https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_18.03.1~ce-0~ubuntu_amd64.deb -o /tmp/docker.deb
 
     # https://download.docker.com/linux/ubuntu/dists/xenial/stable/binary-amd64/Packages
-    if [[ $(sha256sum /tmp/docker.deb | cut -d' ' -f1) != "1f7315b5723b849fe542fe973b0edb4164a0200e926d386ac14363a968f9e4fc" ]]; then
+    if [[ $(sha256sum /tmp/docker.deb | cut -d' ' -f1) != "54f4c9268492a4fd2ec2e6bcc95553855b025f35dcc8b9f60ac34e0aa307279b" ]]; then
         echo "==> installer: docker binary download is corrupt"
         exit 5
     fi
@@ -69,14 +69,15 @@ if [[ $(docker version --format {{.Client.Version}}) != "18.03.0-ce" ]]; then
 fi
 
 echo "==> installer: updating node"
-if [[ "$(node --version)" != "v8.9.3" ]]; then
-    mkdir -p /usr/local/node-8.9.3
-    $curl -sL https://nodejs.org/dist/v8.9.3/node-v8.9.3-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-8.9.3
-    ln -sf /usr/local/node-8.9.3/bin/node /usr/bin/node
-    ln -sf /usr/local/node-8.9.3/bin/npm /usr/bin/npm
+if [[ "$(node --version)" != "v8.11.2" ]]; then
+    mkdir -p /usr/local/node-8.11.2
+    $curl -sL https://nodejs.org/dist/v8.11.2/node-v8.11.2-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-8.11.2
+    ln -sf /usr/local/node-8.11.2/bin/node /usr/bin/node
+    ln -sf /usr/local/node-8.11.2/bin/npm /usr/bin/npm
     rm -rf /usr/local/node-6.11.5
 fi
 
+# this is here (and not in updater.js) because rebuild requires the above node
 for try in `seq 1 10`; do
     # for reasons unknown, the dtrace package will fail. but rebuilding second time will work
 
@@ -93,6 +94,18 @@ if [[ ${try} -eq 10 ]]; then
     exit 4
 fi
 
+echo "==> installer: update cloudron-syslog"
+CLOUDRON_SYSLOG_DIR=/usr/local/cloudron-syslog
+CLOUDRON_SYSLOG="${CLOUDRON_SYSLOG_DIR}/bin/cloudron-syslog"
+CLOUDRON_SYSLOG_VERSION="1.0.3"
+while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLOUDRON_SYSLOG_VERSION} ]]; do
+    rm -rf "${CLOUDRON_SYSLOG_DIR}"
+    mkdir -p "${CLOUDRON_SYSLOG_DIR}"
+    if npm install --unsafe-perm -g --prefix "${CLOUDRON_SYSLOG_DIR}" cloudron-syslog@${CLOUDRON_SYSLOG_VERSION}; then break; fi
+    echo "===> installer: Failed to install cloudron-syslog, trying again"
+    sleep 5
+done
+
 if ! id "${USER}" 2>/dev/null; then
     useradd "${USER}" -m
 fi

setup/start.sh

@@ -76,6 +76,8 @@ mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
 mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
 mkdir -p "${PLATFORM_DATA_DIR}/acme"
 mkdir -p "${PLATFORM_DATA_DIR}/backup"
+mkdir -p "${PLATFORM_DATA_DIR}/logs/backup"
+mkdir -p "${PLATFORM_DATA_DIR}/update"
 
 mkdir -p "${BOX_DATA_DIR}/appicons"
 mkdir -p "${BOX_DATA_DIR}/certs"
@@ -120,6 +122,7 @@ echo "==> Adding systemd services"
 cp -r "${script_dir}/start/systemd/." /etc/systemd/system/
 systemctl daemon-reload
 systemctl enable unbound
+systemctl enable cloudron-syslog
 systemctl enable cloudron.target
 systemctl enable cloudron-firewall
 
@@ -132,6 +135,9 @@ systemctl enable --now cron
 # ensure unbound runs
 systemctl restart unbound
 
+# ensure cloudron-syslog runs
+systemctl restart cloudron-syslog
+
 echo "==> Configuring sudoers"
 rm -f /etc/sudoers.d/${USER}
 cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER}
@@ -146,6 +152,8 @@ echo "==> Configuring logrotate"
 if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then
     echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf
 fi
+cp "${script_dir}/start/app-logrotate" "${PLATFORM_DATA_DIR}/logrotate.d/app-logrotate"
+chown root:root "${PLATFORM_DATA_DIR}/logrotate.d/app-logrotate"
 
 echo "==> Adding motd message for admins"
 cp "${script_dir}/start/cloudron-motd" /etc/update-motd.d/92-cloudron
@@ -231,10 +239,13 @@ fi
 
 echo "==> Changing ownership"
 chown "${USER}:${USER}" -R "${CONFIG_DIR}"
-chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/logrotate.d" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup"
+chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update"
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 
+# logrotate files have to be owned by root, this is here to fixup existing installations where we were resetting the owner to yellowtent
+chown root:root -R "${PLATFORM_DATA_DIR}/logrotate.d"
+
 # do not chown the boxdata/mail directory; dovecot gets upset
 chown "${USER}:${USER}" "${BOX_DATA_DIR}"
 find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${BOX_DATA_DIR}/mail" -exec chown -R "${USER}:${USER}" {} \;

setup/start/app-logrotate

@@ -0,0 +1,10 @@
+# logrotate config for app logs
+
+/home/yellowtent/platformdata/logs/*/*.log {
+    # only keep one rotated file, we currently do not send that over the api
+    rotate 1
+    size 10M
+    # we never compress so we can simply tail the files
+    nocompress
+    copytruncate
+}

setup/start/nginx/appconfig.ejs

@@ -160,6 +160,7 @@ server {
         }
 
         # graphite paths (uncomment block below and visit /graphite/index.html)
+        # remember to comment out the CSP policy as well to access the graphite dashboard
         # location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
         #     proxy_pass   http://127.0.0.1:8000;
         #     client_max_body_size 1m;
@@ -171,6 +172,10 @@ server {
         }
 <% } else if ( endpoint === 'app' ) { %>
         proxy_pass http://127.0.0.1:<%= port %>;
+<% } else if ( endpoint === 'redirect' ) { %>
+        # redirect everything to the app. this is temporary because there is no way
+        # to clear a permanent redirect on the browser
+        return 302 https://<%= redirectTo %>$request_uri;
 <% } %>
     }
 }

setup/start/systemd/cloudron-syslog.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Cloudron Syslog
+After=network.target
+
+[Service]
+ExecStart=/usr/local/cloudron-syslog/bin/cloudron-syslog --port 2514 --logdir /home/yellowtent/platformdata/logs
+WorkingDirectory=/usr/local/cloudron-syslog
+Environment="NODE_ENV=production"
+Restart=always
+User=yellowtent
+Group=yellowtent
+
+[Install]
+WantedBy=multi-user.target

src/accesscontrol.js

@@ -1,190 +1,147 @@
 'use strict';
 
 exports = module.exports = {
-    SCOPE_APPS: 'apps',
+    SCOPE_APPS_READ: 'apps:read',
+    SCOPE_APPS_MANAGE: 'apps:manage',
     SCOPE_CLIENTS: 'clients',
     SCOPE_CLOUDRON: 'cloudron',
-    SCOPE_DOMAINS: 'domains',
+    SCOPE_DOMAINS_READ: 'domains:read',
+    SCOPE_DOMAINS_MANAGE: 'domains:manage',
     SCOPE_MAIL: 'mail',
     SCOPE_PROFILE: 'profile',
     SCOPE_SETTINGS: 'settings',
-    SCOPE_USERS: 'users',
-    VALID_SCOPES: [ 'apps', 'clients', 'cloudron', 'domains', 'mail', 'profile', 'settings', 'users' ],
+    SCOPE_USERS_READ: 'users:read',
+    SCOPE_USERS_MANAGE: 'users:manage',
+    SCOPE_APPSTORE: 'appstore',
+    VALID_SCOPES: [ 'apps', 'appstore', 'clients', 'cloudron', 'domains', 'mail', 'profile', 'settings', 'users' ], // keep this sorted
 
     SCOPE_ANY: '*',
 
-    initialize: initialize,
-    uninitialize: uninitialize,
-
-    accessTokenAuth: accessTokenAuth,
-
-    validateScope: validateScope,
-    validateRequestedScopes: validateRequestedScopes,
-    normalizeScope: normalizeScope,
-    canonicalScope: canonicalScope
+    validateScopeString: validateScopeString,
+    hasScopes: hasScopes,
+    canonicalScopeString: canonicalScopeString,
+    intersectScopes: intersectScopes,
+    validateToken: validateToken,
+    scopesForUser: scopesForUser
 };
 
 var assert = require('assert'),
-    BasicStrategy = require('passport-http').BasicStrategy,
-    BearerStrategy = require('passport-http-bearer').Strategy,
-    clients = require('./clients'),
-    ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
-    ClientsError = clients.ClientsError,
-    DatabaseError = require('./databaseerror'),
+    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:accesscontrol'),
-    LocalStrategy = require('passport-local').Strategy,
-    passport = require('passport'),
-    tokendb = require('./tokendb'),
+    settings = require('./settings.js'),
+    tokendb = require('./tokendb.js'),
     users = require('./users.js'),
     UsersError = users.UsersError,
     _ = require('underscore');
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
+// returns scopes that does not have wildcards and is sorted
+function canonicalScopeString(scope) {
+    if (scope === exports.SCOPE_ANY) return exports.VALID_SCOPES.join(',');
 
-    passport.serializeUser(function (user, callback) {
-        callback(null, user.id);
-    });
+    return scope.split(',').sort().join(',');
+}
 
-    passport.deserializeUser(function(userId, callback) {
-        users.get(userId, function (error, result) {
-            if (error) return callback(error);
+function intersectScopes(allowedScopes, wantedScopes) {
+    assert(Array.isArray(allowedScopes), 'Expecting sorted array');
+    assert(Array.isArray(wantedScopes), 'Expecting sorted array');
 
-            callback(null, result);
-        });
-    });
+    if (_.isEqual(allowedScopes, wantedScopes)) return allowedScopes; // quick path
 
-    passport.use(new LocalStrategy(function (username, password, callback) {
-        if (username.indexOf('@') === -1) {
-            users.verifyWithUsername(username, password, function (error, result) {
-                if (error && error.reason === UsersError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === UsersError.WRONG_PASSWORD) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, result);
-            });
-        } else {
-            users.verifyWithEmail(username, password, function (error, result) {
-                if (error && error.reason === UsersError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === UsersError.WRONG_PASSWORD) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, result);
-            });
+    let wantedScopesMap = new Map();
+    let results = [];
+
+    // make a map of scope -> [ subscopes ]
+    for (let w of wantedScopes) {
+        let parts = w.split(':');
+        let subscopes = wantedScopesMap.get(parts[0]) || new Set();
+        subscopes.add(parts[1] || '*');
+        wantedScopesMap.set(parts[0], subscopes);
+    }
+
+    for (let a of allowedScopes) {
+        let parts = a.split(':');
+        let as = parts[1] || '*';
+
+        let subscopes = wantedScopesMap.get(parts[0]);
+        if (!subscopes) continue;
+
+        if (subscopes.has('*') || subscopes.has(as)) {
+            results.push(a);
+        } else if (as === '*') {
+            results = results.concat(Array.from(subscopes).map(function (ss) { return `${a}:${ss}`; }));
         }
-    }));
+    }
 
-    passport.use(new BasicStrategy(function (username, password, callback) {
-        if (username.indexOf('cid-') === 0) {
-            debug('BasicStrategy: detected client id %s instead of username:password', username);
-            // username is actually client id here
-            // password is client secret
-            clients.get(username, function (error, client) {
-                if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
-                if (error) return callback(error);
-                if (client.clientSecret != password) return callback(null, false);
-                return callback(null, client);
-            });
-        } else {
-            users.verifyWithUsername(username, password, function (error, result) {
-                if (error && error.reason === UsersError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === UsersError.WRONG_PASSWORD) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, result);
-            });
-        }
-    }));
-
-    passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clients.get(clientId, function(error, client) {
-            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
-            if (error) { return callback(error); }
-            if (client.clientSecret != clientSecret) { return callback(null, false); }
-            return callback(null, client);
-        });
-    }));
-
-    passport.use(new BearerStrategy(accessTokenAuth));
-
-    callback(null);
+    return results;
 }
 
-function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    callback(null);
-}
-
-function canonicalScope(scope) {
-    return scope.replace(exports.SCOPE_ANY, exports.VALID_SCOPES.join(','));
-}
-
-function normalizeScope(allowedScope, wantedScope) {
-    assert.strictEqual(typeof allowedScope, 'string');
-    assert.strictEqual(typeof wantedScope, 'string');
-
-    const allowedScopes = allowedScope.split(',');
-    const wantedScopes = wantedScope.split(',');
-
-    if (allowedScopes.indexOf(exports.SCOPE_ANY) !== -1) return canonicalScope(wantedScope);
-    if (wantedScopes.indexOf(exports.SCOPE_ANY) !== -1) return canonicalScope(allowedScope);
-
-    return _.intersection(allowedScopes, wantedScopes).join(',');
-}
-
-function accessTokenAuth(accessToken, callback) {
-    assert.strictEqual(typeof accessToken, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.get(accessToken, function (error, token) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
-        if (error) return callback(error);
-
-        users.get(token.identifier, function (error, user) {
-            if (error && error.reason === UsersError.NOT_FOUND) return callback(null, false);
-            if (error) return callback(error);
-
-            // scopes here can define what capabilities that token carries
-            // passport put the 'info' object into req.authInfo, where we can further validate the scopes
-            var scope = normalizeScope(user.scope, token.scope);
-            var info = { scope: scope, clientId: token.clientId };
-
-            callback(null, user, info);
-        });
-    });
-}
-
-function validateScope(scope) {
+function validateScopeString(scope) {
     assert.strictEqual(typeof scope, 'string');
 
     if (scope === '') return new Error('Empty scope not allowed');
 
     // NOTE: this function intentionally does not allow '*'. This is only allowed in the db to allow
     // us not write a migration script every time we add a new scope
-    var allValid = scope.split(',').every(function (s) { return exports.VALID_SCOPES.indexOf(s) !== -1; });
+    var allValid = scope.split(',').every(function (s) { return exports.VALID_SCOPES.indexOf(s.split(':')[0]) !== -1; });
     if (!allValid) return new Error('Invalid scope. Available scopes are ' + exports.VALID_SCOPES.join(', '));
 
     return null;
 }
 
-// tests if all requestedScopes are attached to the request
-function validateRequestedScopes(authInfo, requestedScopes) {
-    assert.strictEqual(typeof authInfo, 'object');
-    assert(Array.isArray(requestedScopes));
+// tests if all requiredScopes are attached to the request
+function hasScopes(authorizedScopes, requiredScopes) {
+    assert(Array.isArray(authorizedScopes), 'Expecting array');
+    assert(Array.isArray(requiredScopes), 'Expecting array');
 
-    if (!authInfo || !authInfo.scope) return new Error('No scope found');
+    if (authorizedScopes.indexOf(exports.SCOPE_ANY) !== -1) return null;
 
-    var scopes = authInfo.scope.split(',');
+    for (var i = 0; i < requiredScopes.length; ++i) {
+        const scopeParts = requiredScopes[i].split(':');
 
-    if (scopes.indexOf(exports.SCOPE_ANY) !== -1) return null;
-
-    for (var i = 0; i < requestedScopes.length; ++i) {
-        if (scopes.indexOf(requestedScopes[i]) === -1) {
-            debug('scope: missing scope "%s".', requestedScopes[i]);
-            return new Error('Missing required scope "' + requestedScopes[i] + '"');
+        // this allows apps:write if the token has a higher apps scope
+        if (authorizedScopes.indexOf(requiredScopes[i]) === -1 && authorizedScopes.indexOf(scopeParts[0]) === -1) {
+            debug('scope: missing scope "%s".', requiredScopes[i]);
+            return new Error('Missing required scope "' + requiredScopes[i] + '"');
         }
     }
 
     return null;
 }
+
+function scopesForUser(user, callback) {
+    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (user.admin) return callback(null, exports.VALID_SCOPES);
+
+    settings.getSpacesConfig(function (error, spaces) {
+        if (error) return callback(error);
+
+        callback(null, spaces.enabled ? [ 'profile', 'apps', 'domains:read', 'users:read' ] : [ 'profile', 'apps:read' ]);
+    });
+}
+
+function validateToken(accessToken, callback) {
+    assert.strictEqual(typeof accessToken, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.get(accessToken, function (error, token) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
+        if (error) return callback(error); // this triggers 'internal error' in passport
+
+        users.get(token.identifier, function (error, user) {
+            if (error && error.reason === UsersError.NOT_FOUND) return callback(null, null /* user */, 'Invalid Token'); // will end up as a 401
+            if (error) return callback(error);
+
+            scopesForUser(user, function (error, userScopes) {
+                if (error) return callback(error);
+
+                var authorizedScopes = intersectScopes(userScopes, token.scope.split(','));
+                const skipPasswordVerification = token.clientId === 'cid-sdk' || token.clientId === 'cid-cli'; // these clients do not require password checks unlike UI
+                var info = { authorizedScopes: authorizedScopes, skipPasswordVerification: skipPasswordVerification }; // ends up in req.authInfo
+
+                callback(null, user, info);
+            });
+        });
+    });
+}

src/addons.js

@@ -28,7 +28,7 @@ var accesscontrol = require('./accesscontrol.js'),
     docker = require('./docker.js'),
     dockerConnection = docker.connection,
     fs = require('fs'),
-    hat = require('hat'),
+    hat = require('./hat.js'),
     infra = require('./infra_version.js'),
     mail = require('./mail.js'),
     mailboxdb = require('./mailboxdb.js'),
@@ -207,13 +207,21 @@ function getBindsSync(app, addons) {
     assert.strictEqual(typeof app, 'object');
     assert(!addons || typeof addons === 'object');
 
-    var binds = [ ];
+    let binds = [ ];
 
     if (!addons) return binds;
 
-    for (var addon in addons) {
+    for (let addon in addons) {
         switch (addon) {
-        case 'localstorage': binds.push(path.join(paths.APPS_DATA_DIR, app.id, 'data') + ':/app/data:rw'); break;
+        case 'localstorage':
+            binds.push(path.join(paths.APPS_DATA_DIR, app.id, 'data') + ':/app/data:rw');
+            if (!Array.isArray(addons[addon].bindMounts)) break;
+
+            for (let mount of addons[addon].bindMounts) {
+                let [ host, container ] = mount.split(':');
+                binds.push(path.join(paths.APPS_DATA_DIR, app.id, 'data', path.normalize(host)) + ':' + container);
+            }
+            break;
         default: break;
         }
     }
@@ -368,7 +376,7 @@ function setupSendMail(app, options, callback) {
     appdb.getAddonConfigByName(app.id, 'sendmail', 'MAIL_SMTP_PASSWORD', function (error, existingPassword) {
         if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
 
-        var password = error ? hat(4 * 128) : existingPassword;
+        var password = error ? hat(4 * 48) : existingPassword; // see box#565 for password length
 
         mailboxdb.getByOwnerId(app.id, function (error, results) {
             if (error) return callback(error);
@@ -410,7 +418,7 @@ function setupRecvMail(app, options, callback) {
     appdb.getAddonConfigByName(app.id, 'recvmail', 'MAIL_IMAP_PASSWORD', function (error, existingPassword) {
         if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
 
-        var password = error ? hat(4 * 128) : existingPassword;
+        var password = error ? hat(4 * 48) : existingPassword;  // see box#565 for password length
 
         mailboxdb.getByOwnerId(app.id, function (error, results) {
             if (error) return callback(error);
@@ -442,13 +450,12 @@ function teardownRecvMail(app, options, callback) {
     appdb.unsetAddonConfig(app.id, 'recvmail', callback);
 }
 
-function mysqlDatabaseName(appId, prefix) {
+function mysqlDatabaseName(appId) {
     assert.strictEqual(typeof appId, 'string');
 
     var md5sum = crypto.createHash('md5'); // get rid of "-"
     md5sum.update(appId);
-    var dbname = md5sum.digest('hex').substring(0, 16);  // max length of mysql usernames is 16
-    return prefix ? `${dbname}_` : dbname;
+    return md5sum.digest('hex').substring(0, 16);  // max length of mysql usernames is 16
 }
 
 function setupMySql(app, options, callback) {
@@ -461,7 +468,7 @@ function setupMySql(app, options, callback) {
     appdb.getAddonConfigByName(app.id, 'mysql', 'MYSQL_PASSWORD', function (error, existingPassword) {
         if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
 
-        const dbname = mysqlDatabaseName(app.id, options.multipleDatabases);
+        const dbname = mysqlDatabaseName(app.id);
         const password = error ? hat(4 * 48) : existingPassword; // see box#362 for password length
 
         var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'add-prefix' : 'add', dbname, password ];
@@ -477,7 +484,7 @@ function setupMySql(app, options, callback) {
             ];
 
             if (options.multipleDatabases) {
-                env = env.concat({ name: 'MYSQL_DATABASE_PREFIX', value: dbname });
+                env = env.concat({ name: 'MYSQL_DATABASE_PREFIX', value: `${dbname}_` });
             } else {
                 env = env.concat(
                     { name: 'MYSQL_URL', value: `mysql://${dbname}:${password}@mysql/${dbname}` },
@@ -496,7 +503,7 @@ function teardownMySql(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    const dbname = mysqlDatabaseName(app.id, options.multipleDatabases);
+    const dbname = mysqlDatabaseName(app.id);
     var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'remove-prefix' : 'remove', dbname ];
 
     debugApp(app, 'Tearing down mysql');
@@ -520,7 +527,7 @@ function backupMySql(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.APPS_DATA_DIR, app.id, 'mysqldump'));
     output.on('error', callback);
 
-    const dbname = mysqlDatabaseName(app.id, options.multipleDatabases);
+    const dbname = mysqlDatabaseName(app.id);
     var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', dbname ];
 
     docker.execContainer('mysql', cmd, { stdout: output }, callback);
@@ -541,7 +548,7 @@ function restoreMySql(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.APPS_DATA_DIR, app.id, 'mysqldump'));
         input.on('error', callback);
 
-        const dbname = mysqlDatabaseName(app.id, options.multipleDatabases);
+        const dbname = mysqlDatabaseName(app.id);
         var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', dbname ];
         docker.execContainer('mysql', cmd, { stdin: input }, callback);
     });
@@ -767,6 +774,10 @@ function setupRedis(app, options, callback) {
                     --label=location=${label} \
                     --net cloudron \
                     --net-alias ${redisName} \
+                    --log-driver syslog \
+                    --log-opt syslog-address=udp://127.0.0.1:2514 \
+                    --log-opt syslog-format=rfc5424 \
+                    --log-opt tag="${redisName}" \
                     -m ${memoryLimit/2} \
                     --memory-swap ${memoryLimit} \
                     --dns 172.18.0.1 \

src/appdb.js

@@ -24,6 +24,9 @@ exports = module.exports = {
     setRunCommand: setRunCommand,
     getAppStoreIds: getAppStoreIds,
 
+    setOwner: setOwner,
+    transferOwnership: transferOwnership,
+
     // installation codes (keep in sync in UI)
     ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
     ISTATE_PENDING_CLONE: 'pending_clone', // clone
@@ -47,6 +50,10 @@ exports = module.exports = {
     HEALTH_ERROR: 'error',
     HEALTH_DEAD: 'dead',
 
+    // subdomain table types
+    SUBDOMAIN_TYPE_PRIMARY: 'primary',
+    SUBDOMAIN_TYPE_REDIRECT: 'redirect',
+
     _clear: clear
 };
 
@@ -59,10 +66,10 @@ var assert = require('assert'),
     util = require('util');
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
-    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.domain', 'apps.dnsRecordId',
+    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'subdomains.subdomain AS location', 'subdomains.domain',
     'apps.accessRestrictionJson', 'apps.restoreConfigJson', 'apps.oldConfigJson', 'apps.updateConfigJson', 'apps.memoryLimit',
     'apps.xFrameOptions', 'apps.sso', 'apps.debugModeJson', 'apps.robotsTxt', 'apps.enableBackup',
-    'apps.creationTime', 'apps.updateTime' ].join(',');
+    'apps.creationTime', 'apps.updateTime', 'apps.ownerId', 'apps.ts' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -113,6 +120,12 @@ function postProcess(result) {
     assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
     result.debugMode = safe.JSON.parse(result.debugModeJson);
     delete result.debugModeJson;
+
+    result.alternateDomains = result.alternateDomains || [];
+    result.alternateDomains.forEach(function (d) {
+        delete d.appId;
+        delete d.type;
+    });
 }
 
 function get(id, callback) {
@@ -121,13 +134,22 @@ function get(id, callback) {
 
     database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
         + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE apps.id = ? GROUP BY apps.id', [ id ], function (error, result) {
+        + ' FROM apps'
+        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
+        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND type = ?'
+        + ' WHERE apps.id = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        postProcess(result[0]);
+        database.query('SELECT * FROM subdomains WHERE appId = ? AND type = ?', [ id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        callback(null, result[0]);
+            result[0].alternateDomains = alternateDomains;
+
+            postProcess(result[0]);
+
+            callback(null, result[0]);
+        })
     });
 }
 
@@ -137,13 +159,21 @@ function getByHttpPort(httpPort, callback) {
 
     database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
         + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE httpPort = ? GROUP BY apps.id', [ httpPort ], function (error, result) {
+        + ' FROM apps'
+        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
+        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND type = ?'
+        + ' WHERE httpPort = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, httpPort ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        postProcess(result[0]);
+        database.query('SELECT * FROM subdomains WHERE appId = ? AND type = ?', [ result[0].id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        callback(null, result[0]);
+            result[0].alternateDomains = alternateDomains;
+            postProcess(result[0]);
+
+            callback(null, result[0]);
+        });
     });
 }
 
@@ -153,13 +183,21 @@ function getByContainerId(containerId, callback) {
 
     database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
         + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE containerId = ? GROUP BY apps.id', [ containerId ], function (error, result) {
+        + ' FROM apps'
+        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
+        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND type = ?'
+        + ' WHERE containerId = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, containerId ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        postProcess(result[0]);
+        database.query('SELECT * FROM subdomains WHERE appId = ? AND type = ?', [ result[0].id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        callback(null, result[0]);
+            result[0].alternateDomains = alternateDomains;
+            postProcess(result[0]);
+
+            callback(null, result[0]);
+        });
     });
 }
 
@@ -168,23 +206,38 @@ function getAll(callback) {
 
     database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
         + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
-        + ' GROUP BY apps.id ORDER BY apps.id', function (error, results) {
+        + ' FROM apps'
+        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
+        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND type = ?'
+        + ' GROUP BY apps.id ORDER BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        results.forEach(postProcess);
+        database.query('SELECT * FROM subdomains WHERE type = ?', [ exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        callback(null, results);
+            alternateDomains.forEach(function (d) {
+                var domain = results.find(function (a) { return d.appId === a.id; });
+                if (!domain) return;
+
+                domain.alternateDomains = domain.alternateDomains || [];
+                domain.alternateDomains.push(d);
+            });
+
+            results.forEach(postProcess);
+
+            callback(null, results);
+        });
     });
 }
 
-function add(id, appStoreId, manifest, location, domain, portBindings, data, callback) {
+function add(id, appStoreId, manifest, location, domain, ownerId, portBindings, data, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof manifest.version, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof ownerId, 'string');
     assert.strictEqual(typeof portBindings, 'object');
     assert(data && typeof data === 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -204,10 +257,16 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
     var debugModeJson = data.debugMode ? JSON.stringify(data.debugMode) : null;
 
     var queries = [];
+
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, domain, accessRestrictionJson, memoryLimit, xFrameOptions, restoreConfigJson, sso, debugModeJson, robotsTxt) ' +
-            ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, installationState, location, domain, accessRestrictionJson, memoryLimit, xFrameOptions, restoreConfigJson, sso, debugModeJson, robotsTxt ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, accessRestrictionJson, memoryLimit, xFrameOptions, restoreConfigJson, sso, debugModeJson, robotsTxt, ownerId) ' +
+            ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, accessRestrictionJson, memoryLimit, xFrameOptions, restoreConfigJson, sso, debugModeJson, robotsTxt, ownerId ]
+    });
+
+    queries.push({
+        query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+        args: [ id, domain, location, exports.SUBDOMAIN_TYPE_PRIMARY ]
     });
 
     Object.keys(portBindings).forEach(function (env) {
@@ -225,6 +284,15 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
         });
     }
 
+    if (data.alternateDomains) {
+        data.alternateDomains.forEach(function (d) {
+            queries.push({
+                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]
+            });
+        });
+    }
+
     database.transaction(queries, function (error) {
         if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
         if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND, 'no such domain'));
@@ -278,6 +346,7 @@ function del(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var queries = [
+        { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM mailboxes WHERE ownerId=?', args: [ id ] },
         { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
         { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
@@ -285,7 +354,7 @@ function del(id, callback) {
 
     database.transaction(queries, function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (results[2].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (results[3].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         callback(null);
     });
@@ -295,6 +364,7 @@ function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     async.series([
+        database.query.bind(null, 'DELETE FROM subdomains'),
         database.query.bind(null, 'DELETE FROM appPortBindings'),
         database.query.bind(null, 'DELETE FROM appAddonConfigs'),
         database.query.bind(null, 'DELETE FROM apps')
@@ -315,6 +385,7 @@ function updateWithConstraints(id, app, constraints, callback) {
     assert.strictEqual(typeof callback, 'function');
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
     assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
+    assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
 
     var queries = [ ];
 
@@ -328,12 +399,27 @@ function updateWithConstraints(id, app, constraints, callback) {
         });
     }
 
+    if ('location' in app) {
+        queries.push({ query: 'UPDATE subdomains SET subdomain = ? WHERE appId = ? AND type = ?', args: [ app.location, id, exports.SUBDOMAIN_TYPE_PRIMARY ]});
+    }
+
+    if ('domain' in app) {
+        queries.push({ query: 'UPDATE subdomains SET domain = ? WHERE appId = ? AND type = ?', args: [ app.domain, id, exports.SUBDOMAIN_TYPE_PRIMARY ]});
+    }
+
+    if ('alternateDomains' in app) {
+        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ? AND type = ?', args: [ id, exports.SUBDOMAIN_TYPE_REDIRECT ]});
+        app.alternateDomains.forEach(function (d) {
+            queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
+        });
+    }
+
     var fields = [ ], values = [ ];
     for (var p in app) {
         if (p === 'manifest' || p === 'oldConfig' || p === 'updateConfig' || p === 'restoreConfig' || p === 'accessRestriction' || p === 'debugMode') {
             fields.push(`${p}Json = ?`);
             values.push(JSON.stringify(app[p]));
-        } else if (p !== 'portBindings') {
+        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains') {
             fields.push(p + ' = ?');
             values.push(app[p]);
         }
@@ -505,3 +591,31 @@ function getAddonConfigByName(appId, addonId, name, callback) {
         callback(null, results[0].value);
     });
 }
+
+function setOwner(appId, ownerId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('UPDATE apps SET ownerId=? WHERE appId=?', [ ownerId, appId ], function (error, results) {
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND, 'No such user'));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND, 'No such app'));
+
+        callback(null);
+    });
+}
+
+function transferOwnership(oldOwnerId, newOwnerId, callback) {
+    assert.strictEqual(typeof oldOwnerId, 'string');
+    assert.strictEqual(typeof newOwnerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('UPDATE apps SET ownerId=? WHERE ownerId=?', [ newOwnerId, oldOwnerId ], function (error, results) {
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND, 'No such user'));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}

src/apps.js

@@ -4,7 +4,8 @@ exports = module.exports = {
     AppsError: AppsError,
 
     hasAccessTo: hasAccessTo,
-    removeInternalAppFields: removeInternalAppFields,
+    removeInternalFields: removeInternalFields,
+    removeRestrictedFields: removeRestrictedFields,
 
     get: get,
     getByIpAddress: getByIpAddress,
@@ -41,14 +42,16 @@ exports = module.exports = {
     downloadFile: downloadFile,
     uploadFile: uploadFile,
 
+    setOwner: setOwner,
+    transferOwnership: transferOwnership,
+
     // exported for testing
     _validateHostname: validateHostname,
     _validatePortBindings: validatePortBindings,
     _validateAccessRestriction: validateAccessRestriction
 };
 
-var addons = require('./addons.js'),
-    appdb = require('./appdb.js'),
+var appdb = require('./appdb.js'),
     appstore = require('./appstore.js'),
     AppstoreError = require('./appstore.js').AppstoreError,
     assert = require('assert'),
@@ -65,7 +68,6 @@ var addons = require('./addons.js'),
     DomainsError = require('./domains.js').DomainsError,
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
-    groups = require('./groups.js'),
     mail = require('./mail.js'),
     mailboxdb = require('./mailboxdb.js'),
     manifestFormat = require('cloudron-manifestformat'),
@@ -147,9 +149,9 @@ function validateHostname(location, domain, hostname) {
 
     if (location) {
         // label validation
-        if (location.length > 63) return new AppsError(AppsError.BAD_FIELD, 'Subdomain exceeds 63 characters');
-        if (location.match(/^[A-Za-z0-9-]+$/) === null) return new AppsError(AppsError.BAD_FIELD, 'Subdomain can only contain alphanumerics and hyphen');
-        if (location.startsWith('-') || location.endsWith('-')) return new AppsError(AppsError.BAD_FIELD, 'Subdomain cannot start or end with hyphen');
+        if (location.split('.').some(function (p) { return p.length > 63 || p.length < 1; })) return new AppsError(AppsError.BAD_FIELD, 'Invalid subdomain length');
+        if (location.match(/^[A-Za-z0-9-.]+$/) === null) return new AppsError(AppsError.BAD_FIELD, 'Subdomain can only contain alphanumeric, hyphen and dot');
+        if (/^[-.]/.test(location)) return new AppsError(AppsError.BAD_FIELD, 'Subdomain cannot start or end with hyphen or dot');
     }
 
     return null;
@@ -159,7 +161,7 @@ function validateHostname(location, domain, hostname) {
 function validatePortBindings(portBindings, tcpPorts) {
     assert.strictEqual(typeof portBindings, 'object');
 
-    // keep the public ports in sync with firewall rules in scripts/initializeBaseUbuntuImage.sh
+    // keep the public ports in sync with firewall rules in setup/start/cloudron-firewall.sh
     // these ports are reserved even if we listen only on 127.0.0.1 because we setup HostIp to be 127.0.0.1
     // for custom tcp ports
     var RESERVED_PORTS = [
@@ -176,6 +178,7 @@ function validatePortBindings(portBindings, tcpPorts) {
         2003, /* graphite (lo) */
         2004, /* graphite (lo) */
         2020, /* mail server */
+        2514, /* cloudron-syslog (lo) */
         config.get('port'), /* app server (lo) */
         config.get('sysadminPort'), /* sysadmin app server (lo) */
         config.get('smtpPort'), /* internal smtp port (lo) */
@@ -320,16 +323,24 @@ function getAppConfig(app) {
         memoryLimit: app.memoryLimit,
         xFrameOptions: app.xFrameOptions || 'SAMEORIGIN',
         robotsTxt: app.robotsTxt,
-        sso: app.sso
+        sso: app.sso,
+        alternateDomains: app.alternateDomains || []
     };
 }
 
-function removeInternalAppFields(app) {
+function removeInternalFields(app) {
     return _.pick(app,
         'id', 'appStoreId', 'installationState', 'installationProgress', 'runState', 'health',
         'location', 'domain', 'fqdn', 'mailboxName',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'xFrameOptions',
-        'sso', 'debugMode', 'robotsTxt', 'enableBackup', 'creationTime', 'updateTime');
+        'sso', 'debugMode', 'robotsTxt', 'enableBackup', 'creationTime', 'updateTime', 'ts',
+        'alternateDomains', 'ownerId');
+}
+
+function removeRestrictedFields(app) {
+    return _.pick(app,
+        'id', 'appStoreId', 'installationState', 'installationProgress', 'runState', 'health', 'ownerId',
+        'location', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts');
 }
 
 function getIconUrlSync(app) {
@@ -347,20 +358,13 @@ function hasAccessTo(app, user, callback) {
     // check user access
     if (app.accessRestriction.users.some(function (e) { return e === user.id; })) return callback(null, true);
 
-    // check group access
-    groups.getGroups(user.id, function (error, groupIds) {
-        if (error) return callback(null, false);
+    if (user.admin) return callback(null, true); // admins can always access any app
 
-        const isAdmin = groupIds.indexOf(constants.ADMIN_GROUP_ID) !== -1;
+    if (!app.accessRestriction.groups) return callback(null, false);
 
-        if (isAdmin) return callback(null, true); // admins can always access any app
+    if (app.accessRestriction.groups.some(function (gid) { return user.groupIds.indexOf(gid) !== -1; })) return callback(null, true);
 
-        if (!app.accessRestriction.groups) return callback(null, false);
-
-        if (app.accessRestriction.groups.some(function (gid) { return groupIds.indexOf(gid) !== -1; })) return callback(null, true);
-
-        callback(null, false);
-    });
+    callback(null, false);
 }
 
 function get(appId, callback) {
@@ -479,6 +483,10 @@ function downloadManifest(appStoreId, manifest, callback) {
     });
 }
 
+function mailboxNameForLocation(location, manifest) {
+    return (location ? location : manifest.title.toLowerCase().replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+}
+
 function install(data, auditSource, callback) {
     assert(data && typeof data === 'object');
     assert.strictEqual(typeof auditSource, 'object');
@@ -498,7 +506,8 @@ function install(data, auditSource, callback) {
         robotsTxt = data.robotsTxt || null,
         enableBackup = 'enableBackup' in data ? data.enableBackup : true,
         backupId = data.backupId || null,
-        backupFormat = data.backupFormat || 'tgz';
+        backupFormat = data.backupFormat || 'tgz',
+        ownerId = data.ownerId;
 
     assert(data.appStoreId || data.manifest); // atleast one of them is required
 
@@ -568,13 +577,13 @@ function install(data, auditSource, callback) {
                 xFrameOptions: xFrameOptions,
                 sso: sso,
                 debugMode: debugMode,
-                mailboxName: (location ? location : manifest.title.toLowerCase().replace(/[^a-zA-Z0-9]/g, '')) + '.app',
+                mailboxName: mailboxNameForLocation(location, manifest),
                 restoreConfig: backupId ? { backupId: backupId, backupFormat: backupFormat } : null,
                 enableBackup: enableBackup,
                 robotsTxt: robotsTxt
             };
 
-            appdb.add(appId, appStoreId, manifest, location, domain, portBindings, data, function (error) {
+            appdb.add(appId, appStoreId, manifest, location, domain, ownerId, portBindings, data, function (error) {
                 if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, error.message));
                 if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
@@ -676,6 +685,11 @@ function configure(appId, data, auditSource, callback) {
             if (error) return callback(error);
         }
 
+        if ('alternateDomains' in data) {
+            // TODO validate all subdomains [{ domain: '', subdomain: ''}]
+            values.alternateDomains = data.alternateDomains;
+        }
+
         domains.get(domain, function (error, domainObject) {
             if (error && error.reason === DomainsError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such domain'));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Could not get domain info:' + error.message));
@@ -695,7 +709,7 @@ function configure(appId, data, auditSource, callback) {
                     if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, `${fqdn}.user.key`), data.key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
                 } else { // remove existing cert/key
                     if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, `${fqdn}.user.cert`))) debug('Error removing cert: ' + safe.error.message);
-                    if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, `${fqdn}..user.key`))) debug('Error removing key: ' + safe.error.message);
+                    if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, `${fqdn}.user.key`))) debug('Error removing key: ' + safe.error.message);
                 }
             }
 
@@ -705,9 +719,12 @@ function configure(appId, data, auditSource, callback) {
 
             debug('Will configure app with id:%s values:%j', appId, values);
 
+            // make the mailbox name follow the apps new location, if the user did not set it explicitly
             var oldName = app.mailboxName;
-            var newName = data.mailboxName || app.mailboxName;
+            var newName = data.mailboxName || (app.mailboxName.endsWith('.app') ? mailboxNameForLocation(location, app.manifest) : app.mailboxName);
             mailboxdb.updateName(oldName, values.oldConfig.domain, newName, domain, function (error) {
+                if (newName.endsWith('.app')) error = null; // ignore internal mailbox conflict errors since we want to show location conflict errors in the UI
+
                 if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new AppsError(AppsError.ALREADY_EXISTS, 'This mailbox is already taken'));
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE));
                 if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
@@ -803,12 +820,6 @@ function update(appId, data, auditSource, callback) {
     });
 }
 
-function appLogFilter(app) {
-    var names = [ app.id ].concat(addons.getContainerNamesSync(app, app.manifest.addons));
-
-    return names.map(function (name) { return 'CONTAINER_NAME=' + name; });
-}
-
 function getLogs(appId, options, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert(options && typeof options === 'object');
@@ -816,32 +827,38 @@ function getLogs(appId, options, callback) {
 
     debug('Getting logs for %s', appId);
 
-    get(appId, function (error, app) {
+    get(appId, function (error /*, app */) {
         if (error) return callback(error);
 
         var lines = options.lines || 100,
-            follow = !!options.follow,
-            format = options.format || 'json';
+            format = options.format || 'json',
+            follow = !!options.follow;
 
-        var args = [ '--no-pager', '--lines=' + lines ];
-        if (follow) args.push('--follow');
-        if (format == 'short') args.push('--output=short', '-a'); else args.push('--output=json');
-        args = args.concat(appLogFilter(app));
+        assert.strictEqual(typeof lines, 'number');
+        assert.strictEqual(typeof format, 'string');
 
-        var cp = spawn('/bin/journalctl', args);
+        var args = [ '--lines=' + lines ];
+        if (follow) args.push('--follow', '--retry', '--quiet'); // same as -F. to make it work if file doesn't exist, --quiet to not output file headers, which are no logs
+        args.push(path.join(paths.LOG_DIR, appId, 'apptask.log'));
+        args.push(path.join(paths.LOG_DIR, appId, 'app.log'));
+
+        var cp = spawn('/usr/bin/tail', args);
 
         var transformStream = split(function mapper(line) {
             if (format !== 'json') return line + '\n';
 
-            var obj = safe.JSON.parse(line);
-            if (!obj) return undefined;
+            var data = line.split(' '); // logs are <ISOtimestamp> <msg>
+            var timestamp = (new Date(data[0])).getTime();
+            if (isNaN(timestamp)) timestamp = 0;
+            var message = line.slice(data[0].length+1);
+
+            // ignore faulty empty logs
+            if (!timestamp && !message) return;
 
-            var source = obj.CONTAINER_NAME.slice(app.id.length + 1);
             return JSON.stringify({
-                realtimeTimestamp: obj.__REALTIME_TIMESTAMP,
-                monotonicTimestamp: obj.__MONOTONIC_TIMESTAMP,
-                message: obj.MESSAGE,
-                source: source || 'main'
+                realtimeTimestamp: timestamp * 1000,
+                message: message,
+                source: appId
             }) + '\n';
         });
 
@@ -910,12 +927,14 @@ function clone(appId, data, auditSource, callback) {
     var location = data.location.toLowerCase(),
         domain = data.domain.toLowerCase(),
         portBindings = data.portBindings || null,
-        backupId = data.backupId;
+        backupId = data.backupId,
+        ownerId = data.ownerId;
 
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof portBindings, 'object');
+    assert(ownerId === null || typeof ownerId === 'string');
 
     get(appId, function (error, app) {
         if (error) return callback(error);
@@ -955,7 +974,7 @@ function clone(appId, data, auditSource, callback) {
                     robotsTxt: app.robotsTxt
                 };
 
-                appdb.add(newAppId, app.appStoreId, manifest, location, domain, portBindings, data, function (error) {
+                appdb.add(newAppId, app.appStoreId, manifest, location, domain, ownerId, portBindings, data, function (error) {
                     if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
                     if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -1119,7 +1138,11 @@ function exec(appId, options, callback) {
                 if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
                 if (options.rows && options.columns) {
-                    exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
+                    // there is a race where resizing too early results in a 404 "no such exec"
+                    // https://git.cloudron.io/cloudron/box/issues/549
+                    setTimeout(function () {
+                        exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
+                    }, 2000);
                 }
 
                 return callback(null, stream);
@@ -1336,3 +1359,28 @@ function uploadFile(appId, sourceFilePath, destFilePath, callback) {
         callback(null);
     });
 }
+
+function setOwner(appId, ownerId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.setOwner(appId, ownerId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, error.message));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        callback();
+    });
+}
+
+function transferOwnership(oldOwnerId, newOwnerId, callback) {
+    assert.strictEqual(typeof oldOwnerId, 'string');
+    assert.strictEqual(typeof newOwnerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.transferOwnership(oldOwnerId, newOwnerId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, error.message));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        callback();
+    });
+}

src/appstore.js

@@ -265,17 +265,17 @@ function getBoxUpdate(callback) {
         var url = config.apiServerOrigin() + '/api/v1/users/' + appstoreConfig.userId + '/cloudrons/' + appstoreConfig.cloudronId + '/boxupdate';
 
         superagent.get(url).query({ accessToken: appstoreConfig.token, boxVersion: config.version() }).timeout(10 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new AppstoreError(AppstoreError.EXTERNAL_ERROR, error));
+            if (error && !error.response) return callback(new AppstoreError(AppstoreError.EXTERNAL_ERROR, error.message));
             if (result.statusCode === 204) return callback(null); // no update
             if (result.statusCode !== 200 || !result.body) return callback(new AppstoreError(AppstoreError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
             var updateInfo = result.body;
 
             if (!semver.valid(updateInfo.version) || semver.gt(config.version(), updateInfo.version)) {
-                return callback(new AppstoreError(AppstoreError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+                return callback(new AppstoreError(AppstoreError.EXTERNAL_ERROR, util.format('Invalid update version: %s %s', result.statusCode, result.text)));
             }
 
-            // { version, changelog, upgrade, sourceTarballUrl}
+            // updateInfo: { version, changelog, upgrade, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
             callback(null, updateInfo);
         });
     });

src/apptask.js

@@ -20,11 +20,6 @@ exports = module.exports = {
 
 require('supererror')({ splatchError: true });
 
-// remove timestamp from debug() based output
-require('debug').formatArgs = function formatArgs(args) {
-    args[0] = this.namespace + ' ' + args[0];
-};
-
 var addons = require('./addons.js'),
     appdb = require('./appdb.js'),
     apps = require('./apps.js'),
@@ -46,6 +41,7 @@ var addons = require('./addons.js'),
     path = require('path'),
     paths = require('./paths.js'),
     reverseProxy = require('./reverseproxy.js'),
+    rimraf = require('rimraf'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
     superagent = require('superagent'),
@@ -273,17 +269,15 @@ function registerSubdomain(app, overwrite, callback) {
                 // refuse to update any existing DNS record for custom domains that we did not create
                 if (values.length !== 0 && !overwrite) return retryCallback(null, new Error('DNS Record already exists'));
 
-                domains.upsertDnsRecords(app.location, app.domain, 'A', [ ip ], function (error, changeId) {
+                domains.upsertDnsRecords(app.location, app.domain, 'A', [ ip ], function (error) {
                     if (error && (error.reason === DomainsError.STILL_BUSY || error.reason === DomainsError.EXTERNAL_ERROR)) return retryCallback(error); // try again
 
-                    retryCallback(null, error || changeId);
+                    retryCallback(null, error);
                 });
             });
         }, function (error, result) {
             if (error || result instanceof Error) return callback(error || result);
-
-            // dnsRecordId tracks whether we created this DNS record so that we can unregister later
-            updateApp(app, { dnsRecordId: result }, callback);
+            callback(null);
         });
     });
 }
@@ -294,11 +288,6 @@ function unregisterSubdomain(app, location, domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!app.dnsRecordId) {
-        debugApp(app, 'Skip unregister of record not created by cloudron');
-        return callback(null);
-    }
-
     sysinfo.getPublicIp(function (error, ip) {
         if (error) return callback(error);
 
@@ -313,12 +302,76 @@ function unregisterSubdomain(app, location, domain, callback) {
             });
         }, function (error, result) {
             if (error || result instanceof Error) return callback(error || result);
-
-            updateApp(app, { dnsRecordId: null }, callback);
+            callback(null);
         });
     });
 }
 
+function registerAlternateDomains(app, overwrite, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof overwrite, 'boolean');
+    assert.strictEqual(typeof callback, 'function');
+
+    sysinfo.getPublicIp(function (error, ip) {
+        if (error) return callback(error);
+
+        async.eachSeries(app.alternateDomains, function (domain, callback) {
+            async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
+                debugApp(app, 'Registering alternate subdomain [%s] overwrite: %s', (domain.subdomain ? (domain.subdomain + '.') : '') + domain.domain, overwrite);
+
+                // get the current record before updating it
+                domains.getDnsRecords(domain.subdomain, domain.domain, 'A', function (error, values) {
+                    if (error) return retryCallback(error);
+
+                    // refuse to update any existing DNS record for custom domains that we did not create
+                    if (values.length !== 0 && !overwrite) return retryCallback(null, new Error('DNS Record already exists'));
+
+                    domains.upsertDnsRecords(domain.subdomain, domain.domain, 'A', [ ip ], function (error) {
+                        if (error && (error.reason === DomainsError.STILL_BUSY || error.reason === DomainsError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+
+                        retryCallback(null, error);
+                    });
+                });
+            }, function (error, result) {
+                if (error || result instanceof Error) return callback(error || result);
+                callback();
+            });
+        }, callback);
+    });
+}
+
+function unregisterAlternateDomains(app, all, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof all, 'boolean');
+    assert.strictEqual(typeof callback, 'function');
+
+    var obsoleteDomains
+    if (all) obsoleteDomains = app.alternateDomains;
+    else obsoleteDomains = app.oldConfig.alternateDomains.filter(function (o) { return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; }); });
+
+    if (obsoleteDomains.length === 0) return callback();
+
+    sysinfo.getPublicIp(function (error, ip) {
+        if (error) return callback(error);
+
+        async.eachSeries(obsoleteDomains, function (domain, callback) {
+            async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
+                debugApp(app, 'Unregistering subdomain: %s%s', domain.subdomain ? (domain.subdomain + '.') : '', domain.domain);
+
+                domains.removeDnsRecords(domain.subdomain, domain.domain, 'A', [ ip ], function (error) {
+                    if (error && error.reason === DomainsError.NOT_FOUND) return retryCallback(null, null);
+                    if (error && (error.reason === DomainsError.STILL_BUSY || error.reason === DomainsError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+
+                    retryCallback(null, error);
+                });
+            }, function (error, result) {
+                if (error || result instanceof Error) return callback(error || result);
+                callback();
+            });
+        }, callback);
+    });
+}
+
 function removeIcon(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -329,6 +382,16 @@ function removeIcon(app, callback) {
     });
 }
 
+function cleanupLogs(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    rimraf(path.join(paths.LOG_DIR, app.id), function (error) {
+        if (error) debugApp(app, 'cannot cleanup logs: %s', error);
+        callback(null);
+    });
+}
+
 function waitForDnsPropagation(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -341,7 +404,15 @@ function waitForDnsPropagation(app, callback) {
     sysinfo.getPublicIp(function (error, ip) {
         if (error) return callback(error);
 
-        domains.waitForDnsRecord(app.fqdn, app.domain, ip, { interval: 5000, times: 240 }, callback);
+        domains.waitForDnsRecord(app.fqdn, app.domain, ip, { interval: 5000, times: 240 }, function (error) {
+            if (error) return callback(error);
+
+            // now wait for alternateDomains, if any
+            async.eachSeries(app.alternateDomains, function (domain, callback) {
+                var fqdn = (domain.subdomain ? (domain.subdomain + '.') : '') + domain.domain;
+                domains.waitForDnsRecord(fqdn, domain.domain, ip, { interval: 5000, times: 240 }, callback);
+            }, callback);
+        });
     });
 }
 
@@ -487,20 +558,25 @@ function configure(app, callback) {
         removeLogrotateConfig.bind(null, app),
         stopApp.bind(null, app),
         deleteContainers.bind(null, app),
+        unregisterAlternateDomains.bind(null, app, false /* all */),
         function (next) {
             if (!locationChanged) return next();
 
             unregisterSubdomain(app, app.oldConfig.location, app.oldConfig.domain, next);
         },
 
+
         reserveHttpPort.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '20, Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
+        updateApp.bind(null, app, { installationProgress: '30, Registering subdomain' }),
         registerSubdomain.bind(null, app, !locationChanged /* overwrite */), // if location changed, do not overwrite to detect conflicts
 
+        updateApp.bind(null, app, { installationProgress: '35, Registering alternate domains'}),
+        registerAlternateDomains.bind(null, app, true /* overwrite */), // figure out when to overwrite
+
         updateApp.bind(null, app, { installationProgress: '40, Downloading image' }),
         docker.downloadImage.bind(null, app.manifest),
 
@@ -679,15 +755,19 @@ function uninstall(app, callback) {
         updateApp.bind(null, app, { installationProgress: '50, Deleting image' }),
         docker.deleteImage.bind(null, app.manifest),
 
-        updateApp.bind(null, app, { installationProgress: '60, Unregistering subdomain' }),
+        updateApp.bind(null, app, { installationProgress: '60, Unregistering domains' }),
+        unregisterAlternateDomains.bind(null, app, true /* all */),
         unregisterSubdomain.bind(null, app, app.location, app.domain),
 
-        updateApp.bind(null, app, { installationProgress: '80, Cleanup icon' }),
+        updateApp.bind(null, app, { installationProgress: '70, Cleanup icon' }),
         removeIcon.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Unconfiguring reverse proxy' }),
+        updateApp.bind(null, app, { installationProgress: '80, Unconfiguring reverse proxy' }),
         unconfigureReverseProxy.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '90, Cleanup logs' }),
+        cleanupLogs.bind(null, app),
+
         updateApp.bind(null, app, { installationProgress: '95, Remove app from database' }),
         appdb.del.bind(null, app.id)
     ], function seriesDone(error) {
@@ -776,6 +856,8 @@ function startTask(appId, callback) {
 if (require.main === module) {
     assert.strictEqual(process.argv.length, 3, 'Pass the appid as argument');
 
+    // add a separator for the log file
+    debug('------------------------------------------------------------');
     debug('Apptask for %s', process.argv[2]);
 
     process.on('SIGTERM', function () {

src/backups.js

@@ -58,6 +58,7 @@ var addons = require('./addons.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
     settings = require('./settings.js'),
+    superagent = require('superagent'),
     syncer = require('./syncer.js'),
     tar = require('tar-fs'),
     util = require('util'),
@@ -157,7 +158,7 @@ function get(backupId, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     backupdb.get(backupId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, error));
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND));
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
         callback(null, result);
@@ -182,6 +183,92 @@ function log(detail) {
     progress.setDetail(progress.BACKUP, detail);
 }
 
+function encryptFilePath(filePath, key) {
+    assert.strictEqual(typeof filePath, 'string');
+    assert.strictEqual(typeof key, 'string');
+
+    var encryptedParts = filePath.split('/').map(function (part) {
+        const cipher = crypto.createCipher('aes-256-cbc', key);
+        let crypt = cipher.update(part);
+        crypt = Buffer.concat([ crypt, cipher.final() ]);
+
+        return crypt.toString('base64')     // ensures path is valid
+            .replace(/\//g, '-')            // replace '/' of base64 since it conflicts with path separator
+            .replace(/=/g,'');              // strip trailing = padding. this is only needed if we concat base64 strings, which we don't
+    });
+
+    return encryptedParts.join('/');
+}
+
+function decryptFilePath(filePath, key) {
+    assert.strictEqual(typeof filePath, 'string');
+    assert.strictEqual(typeof key, 'string');
+
+    let decryptedParts = [];
+    for (let part of filePath.split('/')) {
+        part = part + Array(part.length % 4).join('='); // add back = padding
+        part = part.replace(/-/g, '/');                 // replace with '/'
+
+        try {
+            let decrypt = crypto.createDecipher('aes-256-cbc', key);
+            let text = decrypt.update(Buffer.from(part, 'base64'));
+            text = Buffer.concat([ text, decrypt.final() ]);
+            decryptedParts.push(text.toString('utf8'));
+        } catch (error) {
+            debug(`Error decrypting file ${filePath} part ${part}:`, error);
+            return null;
+        }
+    }
+
+    return decryptedParts.join('/');
+}
+
+function createReadStream(sourceFile, key) {
+    assert.strictEqual(typeof sourceFile, 'string');
+    assert(key === null || typeof key === 'string');
+
+    var stream = fs.createReadStream(sourceFile);
+    var ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+
+    stream.on('error', function (error) {
+        debug('createReadStream: tar stream error.', error);
+        ps.emit('error', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+    });
+
+    ps.on('progress', function(progress) {
+        debug('createReadStream: %s@%s (%s)', Math.round(progress.transferred/1024/1024) + 'M', Math.round(progress.speed/1024/1024) + 'Mbps', sourceFile);
+    });
+
+    if (key !== null) {
+        var encrypt = crypto.createCipher('aes-256-cbc', key);
+        encrypt.on('error', function (error) {
+            debug('createReadStream: encrypt stream error.', error);
+            ps.emit('error', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+        });
+        return stream.pipe(encrypt).pipe(ps);
+    } else {
+        return stream.pipe(ps);
+    }
+}
+
+function createWriteStream(destFile, key) {
+    assert.strictEqual(typeof destFile, 'string');
+    assert(key === null || typeof key === 'string');
+
+    var stream = fs.createWriteStream(destFile);
+
+    if (key !== null) {
+        var decrypt = crypto.createDecipher('aes-256-cbc', key);
+        decrypt.on('error', function (error) {
+            debug('createWriteStream: decrypt stream error.', error);
+        });
+        decrypt.pipe(stream);
+        return decrypt;
+    } else {
+        return stream;
+    }
+}
+
 function createTarPackStream(sourceDir, key) {
     assert.strictEqual(typeof sourceDir, 'string');
     assert(key === null || typeof key === 'string');
@@ -232,21 +319,23 @@ function sync(backupConfig, backupId, dataDir, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     function setBackupProgress(message) {
-        debug('%s: %s', (new Date()).toISOString(), message);
+        debug('%s', message);
         safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, message);
     }
 
     syncer.sync(dataDir, function processTask(task, iteratorCallback) {
         debug('sync: processing task: %j', task);
-        var backupFilePath = path.join(getBackupFilePath(backupConfig, backupId, backupConfig.format), task.path);
+        // the empty task.path is special to signify the directory
+        const destPath = task.path && backupConfig.key ? encryptFilePath(task.path, backupConfig.key) : task.path;
+        const backupFilePath = path.join(getBackupFilePath(backupConfig, backupId, backupConfig.format), destPath);
 
         if (task.operation === 'removedir') {
-            safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, `Removing directory ${task.path}`);
+            setBackupProgress(`Removing directory ${backupFilePath}`);
             return api(backupConfig.provider).removeDir(backupConfig, backupFilePath)
                 .on('progress', setBackupProgress)
                 .on('done', iteratorCallback);
         } else if (task.operation === 'remove') {
-            setBackupProgress(`Removing ${task.path}`);
+            setBackupProgress(`Removing ${backupFilePath}`);
             return api(backupConfig.provider).remove(backupConfig, backupFilePath, iteratorCallback);
         }
 
@@ -258,7 +347,7 @@ function sync(backupConfig, backupId, dataDir, callback) {
             debug(`${task.operation} ${task.path} try ${retryCount}`);
             if (task.operation === 'add') {
                 setBackupProgress(`Adding ${task.path} position ${task.position} try ${retryCount}`);
-                var stream = fs.createReadStream(path.join(dataDir, task.path));
+                var stream = createReadStream(path.join(dataDir, task.path), backupConfig.key || null);
                 stream.on('error', function (error) {
                     setBackupProgress(`read stream error for ${task.path}: ${error.message}`);
                     retryCallback();
@@ -366,7 +455,7 @@ function tarExtract(inStream, destination, key, callback) {
         var decrypt = crypto.createDecipher('aes-256-cbc', key);
         decrypt.on('error', function (error) {
             debug('tarExtract: decrypt stream error.', error);
-            callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+            callback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`));
         });
         inStream.pipe(ps).pipe(decrypt).pipe(gunzip).pipe(extract);
     } else {
@@ -400,6 +489,45 @@ function restoreFsMetadata(appDataDir, callback) {
     });
 }
 
+function downloadDir(backupConfig, backupFilePath, destDir, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert.strictEqual(typeof destDir, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`downloadDir: ${backupFilePath} to ${destDir}`);
+
+    function downloadFile(entry, callback) {
+        let relativePath = path.relative(backupFilePath, entry.fullPath);
+        if (backupConfig.key) {
+            relativePath = decryptFilePath(relativePath, backupConfig.key);
+            if (!relativePath) return callback(new BackupsError(BackupsError.BAD_STATE, 'Unable to decrypt file'));
+        }
+        const destFilePath = path.join(destDir, relativePath);
+
+        mkdirp(path.dirname(destFilePath), function (error) {
+            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+
+            api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) {
+                if (error) return callback(error);
+
+                sourceStream.on('error', callback);
+
+                let destStream = createWriteStream(destFilePath, backupConfig.key || null);
+                destStream.on('error', callback);
+
+                debug(`downloadDir: Copying ${entry.fullPath} to ${destFilePath}`);
+
+                sourceStream.pipe(destStream, { end: true }).on('finish', callback);
+            });
+        });
+    }
+
+    api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, done) {
+        async.each(entries, downloadFile, done);
+    }, callback);
+}
+
 function download(backupConfig, backupId, format, dataDir, callback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof backupId, 'string');
@@ -418,9 +546,7 @@ function download(backupConfig, backupId, format, dataDir, callback) {
             tarExtract(sourceStream, dataDir, backupConfig.key || null, callback);
         });
     } else {
-        var events = api(backupConfig.provider).downloadDir(backupConfig, getBackupFilePath(backupConfig, backupId, format), dataDir);
-        events.on('progress', log);
-        events.on('done', function (error) {
+        downloadDir(backupConfig, getBackupFilePath(backupConfig, backupId, format), dataDir, function (error) {
             if (error) return callback(error);
 
             restoreFsMetadata(dataDir, callback);
@@ -562,6 +688,34 @@ function uploadBoxSnapshot(backupConfig, callback) {
     });
 }
 
+
+function backupDone(apiConfig, backupId, appBackupIds, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof backupId, 'string');
+    assert(Array.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    if (apiConfig.provider !== 'caas') return callback();
+
+    debug('[%s] backupDone: %s apps %j', backupId, backupId, appBackupIds);
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + apiConfig.fqdn + '/backupDone';
+    var data = {
+        boxVersion: config.version(),
+        backupId: backupId,
+        appId: null,        // now unused
+        appVersion: null,   // now unused
+        appBackupIds: appBackupIds
+    };
+
+    superagent.post(url).send(data).query({ token: apiConfig.token }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+        if (result.statusCode !== 200) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
+
+        return callback(null);
+    });
+}
+
 function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof timestamp, 'string');
@@ -591,8 +745,7 @@ function rotateBoxBackup(backupConfig, timestamp, appBackupIds, callback) {
 
                 log(`Rotated box backup successfully as id ${backupId}`);
 
-                // FIXME this is only needed for caas, hopefully we can remove that in the future
-                api(backupConfig.provider).backupDone(backupConfig, backupId, appBackupIds, function (error) {
+                backupDone(backupConfig, backupId, appBackupIds, function (error) {
                     if (error) return callback(error);
 
                     callback(null, backupId);

src/clients.js

@@ -33,10 +33,11 @@ var apps = require('./apps.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:clients'),
     eventlog = require('./eventlog.js'),
-    hat = require('hat'),
+    hat = require('./hat.js'),
     accesscontrol = require('./accesscontrol.js'),
     tokendb = require('./tokendb.js'),
     users = require('./users.js'),
+    UsersError = users.UsersError,
     util = require('util'),
     uuid = require('uuid');
 
@@ -85,7 +86,7 @@ function add(appId, type, redirectURI, scope, callback) {
     assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var error = accesscontrol.validateScope(scope);
+    var error = accesscontrol.validateScopeString(scope);
     if (error) return callback(new ClientsError(ClientsError.INVALID_SCOPE, error.message));
 
     error = validateName(appId);
@@ -252,18 +253,29 @@ function addTokenByUserId(clientId, userId, expiresAt, callback) {
     get(clientId, function (error, result) {
         if (error) return callback(error);
 
-        var token = tokendb.generateToken();
-        var scope = accesscontrol.canonicalScope(result.scope);
-
-        tokendb.add(token, userId, result.id, expiresAt, scope, function (error) {
+        users.get(userId, function (error, user) {
+            if (error && error.reason === UsersError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such user'));
             if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
 
-            callback(null, {
-                accessToken: token,
-                identifier: userId,
-                clientId: result.id,
-                scope: result.scope,
-                expires: expiresAt
+            accesscontrol.scopesForUser(user, function (error, userScopes) {
+                if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+                var scope = accesscontrol.canonicalScopeString(result.scope);
+                var authorizedScopes = accesscontrol.intersectScopes(userScopes, scope.split(','));
+
+                var token = tokendb.generateToken();
+
+                tokendb.add(token, userId, result.id, expiresAt, authorizedScopes.join(','), function (error) {
+                    if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+                    callback(null, {
+                        accessToken: token,
+                        tokenScopes: authorizedScopes,
+                        identifier: userId,
+                        clientId: result.id,
+                        expires: expiresAt
+                    });
+                });
             });
         });
     });

src/cloudron.js

@@ -9,7 +9,6 @@ exports = module.exports = {
     getDisks: getDisks,
     getLogs: getLogs,
 
-    updateToLatest: updateToLatest,
     reboot: reboot,
 
     onActivated: onActivated,
@@ -19,14 +18,10 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
-    backups = require('./backups.js'),
-    caas = require('./caas.js'),
     config = require('./config.js'),
     cron = require('./cron.js'),
     debug = require('debug')('box:cloudron'),
     df = require('@sindresorhus/df'),
-    eventlog = require('./eventlog.js'),
-    locker = require('./locker.js'),
     mailer = require('./mailer.js'),
     os = require('os'),
     path = require('path'),
@@ -39,13 +34,10 @@ var assert = require('assert'),
     shell = require('./shell.js'),
     spawn = require('child_process').spawn,
     split = require('split'),
-    updateChecker = require('./updatechecker.js'),
     users = require('./users.js'),
-    util = require('util'),
-    _ = require('underscore');
+    util = require('util');
 
-var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
-    UPDATE_CMD = path.join(__dirname, 'scripts/update.sh');
+var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');
 
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
@@ -140,47 +132,23 @@ function getDisks(callback) {
 function getConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    // result to not depend on the appstore
-    const BOX_AND_USER_TEMPLATE = {
-        box: {
-            region: null,
-            size: null,
-            plan: 'Custom Plan'
-        },
-        user: {
-            billing: false,
-            currency: ''
-        }
-    };
+    settings.getAll(function (error, allSettings) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-    caas.getBoxAndUserDetails(function (error, result) {
-        if (error) debug('Failed to fetch cloudron details.', error.reason, error.message);
-
-        result = _.extend(BOX_AND_USER_TEMPLATE, result || {});
-
-        settings.getCloudronName(function (error, cloudronName) {
-            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-            callback(null, {
-                apiServerOrigin: config.apiServerOrigin(),
-                webServerOrigin: config.webServerOrigin(),
-                adminDomain: config.adminDomain(),
-                adminLocation: config.adminLocation(),
-                adminFqdn: config.adminFqdn(),
-                mailFqdn: config.mailFqdn(),
-                version: config.version(),
-                update: updateChecker.getUpdateInfo(),
-                progress: progress.getAll(),
-                isDemo: config.isDemo(),
-                region: result.box.region,
-                size: result.box.size,
-                billing: !!result.user.billing,
-                plan: result.box.plan,
-                currency: result.user.currency,
-                memory: os.totalmem(),
-                provider: config.provider(),
-                cloudronName: cloudronName
-            });
+        // be picky about what we send out here since this is sent for 'normal' users as well
+        callback(null, {
+            apiServerOrigin: config.apiServerOrigin(),
+            webServerOrigin: config.webServerOrigin(),
+            adminDomain: config.adminDomain(),
+            adminFqdn: config.adminFqdn(),
+            mailFqdn: config.mailFqdn(),
+            version: config.version(),
+            progress: progress.getAll(),
+            isDemo: config.isDemo(),
+            memory: os.totalmem(),
+            provider: config.provider(),
+            cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
+            spaces: allSettings[settings.SPACES_CONFIG_KEY] // here because settings route cannot be accessed by spaces users
         });
     });
 }
@@ -189,102 +157,6 @@ function reboot(callback) {
     shell.sudo('reboot', [ REBOOT_CMD ], callback);
 }
 
-function update(boxUpdateInfo, auditSource, callback) {
-    assert.strictEqual(typeof boxUpdateInfo, 'object');
-    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!boxUpdateInfo) return callback(null);
-
-    var error = locker.lock(locker.OP_BOX_UPDATE);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
-
-    // ensure tools can 'wait' on progress
-    progress.set(progress.UPDATE, 0, 'Starting');
-
-    // initiate the update/upgrade but do not wait for it
-    if (boxUpdateInfo.upgrade) {
-        debug('Starting upgrade');
-        caas.upgrade(boxUpdateInfo, function (error) {
-            if (error) {
-                debug('Upgrade failed with error:', error);
-                locker.unlock(locker.OP_BOX_UPDATE);
-            }
-        });
-    } else {
-        debug('Starting update');
-        doUpdate(boxUpdateInfo, function (error) {
-            if (error) {
-                debug('Update failed with error:', error);
-                locker.unlock(locker.OP_BOX_UPDATE);
-            }
-        });
-    }
-
-    callback(null);
-}
-
-function updateToLatest(auditSource, callback) {
-    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var boxUpdateInfo = updateChecker.getUpdateInfo().box;
-    if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
-    if (!boxUpdateInfo.sourceTarballUrl) return callback(new CloudronError(CloudronError.BAD_STATE, 'No automatic update available'));
-
-    if (boxUpdateInfo.upgrade && config.provider() !== 'caas') return callback(new CloudronError(CloudronError.SELF_UPGRADE_NOT_SUPPORTED));
-
-    update(boxUpdateInfo, auditSource, callback);
-}
-
-function doUpdate(boxUpdateInfo, callback) {
-    assert(boxUpdateInfo && typeof boxUpdateInfo === 'object');
-
-    function updateError(e) {
-        progress.set(progress.UPDATE, -1, e.message);
-        callback(e);
-    }
-
-    progress.set(progress.UPDATE, 5, 'Backing up for update');
-
-    backups.backupBoxAndApps({ userId: null, username: 'updater' }, function (error) {
-        if (error) return updateError(error);
-
-        // NOTE: this data is opaque and will be passed through the installer.sh
-        var data= {
-            provider: config.provider(),
-            apiServerOrigin: config.apiServerOrigin(),
-            webServerOrigin: config.webServerOrigin(),
-            adminDomain: config.adminDomain(),
-            adminFqdn: config.adminFqdn(),
-            adminLocation: config.adminLocation(),
-            isDemo: config.isDemo(),
-
-            appstore: {
-                apiServerOrigin: config.apiServerOrigin()
-            },
-            caas: {
-                apiServerOrigin: config.apiServerOrigin(),
-                webServerOrigin: config.webServerOrigin()
-            },
-
-            version: boxUpdateInfo.version
-        };
-
-        debug('updating box %s %j', boxUpdateInfo.sourceTarballUrl, _.omit(data, 'tlsCert', 'tlsKey', 'token', 'appstore', 'caas'));
-
-        progress.set(progress.UPDATE, 5, 'Downloading and installing new version');
-
-        shell.sudo('update', [ UPDATE_CMD, boxUpdateInfo.sourceTarballUrl, JSON.stringify(data) ], function (error) {
-            if (error) return updateError(error);
-
-            // Do not add any code here. The installer script will stop the box code any instant
-        });
-    });
-}
-
 function checkDiskSpace(callback) {
     callback = callback || NOOP_CALLBACK;
 
@@ -327,44 +199,66 @@ function checkDiskSpace(callback) {
     });
 }
 
-function getLogs(options, callback) {
+function getLogs(unit, options, callback) {
+    assert.strictEqual(typeof unit, 'string');
     assert(options && typeof options === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var units = options.units || [],
-        lines = options.lines || 100,
+    var lines = options.lines || 100,
         format = options.format || 'json',
         follow = !!options.follow;
 
-    assert(Array.isArray(units));
     assert.strictEqual(typeof lines, 'number');
     assert.strictEqual(typeof format, 'string');
 
-    debug('Getting logs for %j', units);
+    assert.strictEqual(typeof lines, 'number');
+    assert.strictEqual(typeof format, 'string');
 
-    var args = [ '--no-pager', '--lines=' + lines ];
-    units.forEach(function (u) {
-        if (u === 'box') args.push('--unit=box');
-        else if (u === 'mail') args.push('CONTAINER_NAME=mail');
-    });
-    if (format === 'short') args.push('--output=short', '-a'); else args.push('--output=json');
-    if (follow) args.push('--follow');
+    debug('Getting logs for %s as %s', unit, format);
 
-    var cp = spawn('/bin/journalctl', args);
+    var cp, transformStream;
+    if (unit === 'box') {
+        let args = [ '--no-pager', `--lines=${lines}` ];
+        if (format === 'short') args.push('--output=short', '-a'); else args.push('--output=json');
+        if (follow) args.push('--follow');
+        args.push('--unit=box');
+        args.push('--unit=cloudron-updater');
+        cp = spawn('/bin/journalctl', args);
 
-    var transformStream = split(function mapper(line) {
-        if (format !== 'json') return line + '\n';
+        transformStream = split(function mapper(line) {
+            if (format !== 'json') return line + '\n';
 
-        var obj = safe.JSON.parse(line);
-        if (!obj) return undefined;
+            var obj = safe.JSON.parse(line);
+            if (!obj) return undefined;
 
-        return JSON.stringify({
-            realtimeTimestamp: obj.__REALTIME_TIMESTAMP,
-            monotonicTimestamp: obj.__MONOTONIC_TIMESTAMP,
-            message: obj.MESSAGE,
-            source: obj.SYSLOG_IDENTIFIER || ''
-        }) + '\n';
-    });
+            return JSON.stringify({
+                realtimeTimestamp: obj.__REALTIME_TIMESTAMP,
+                monotonicTimestamp: obj.__MONOTONIC_TIMESTAMP,
+                message: obj.MESSAGE,
+                source: obj.SYSLOG_IDENTIFIER || ''
+            }) + '\n';
+        });
+    } else { // mail, mongodb, mysql, postgresql, backup
+        let args = [ '--lines=' + lines ];
+        if (follow) args.push('--follow');
+        args.push(path.join(paths.LOG_DIR, unit, 'app.log'));
+
+        cp = spawn('/usr/bin/tail', args);
+
+        transformStream = split(function mapper(line) {
+            if (format !== 'json') return line + '\n';
+
+            var data = line.split(' '); // logs are <ISOtimestamp> <msg>
+            var timestamp = (new Date(data[0])).getTime();
+            if (isNaN(timestamp)) timestamp = 0;
+
+            return JSON.stringify({
+                realtimeTimestamp: timestamp * 1000,
+                message: line.slice(data[0].length+1),
+                source: unit
+            }) + '\n';
+        });
+    }
 
     transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process
 

src/constants.js

@@ -21,9 +21,6 @@ exports = module.exports = {
 
     ADMIN_CLIENT_ID: 'webadmin', // oauth client id
 
-    ADMIN_GROUP_NAME: 'admin',
-    ADMIN_GROUP_ID: 'admin',
-
     NGINX_ADMIN_CONFIG_FILE_NAME: 'admin.conf',
 
     GHOST_USER_FILE: '/tmp/cloudron_ghost.json',

src/cron.js

@@ -22,6 +22,7 @@ var apps = require('./apps.js'),
     reverseProxy = require('./reverseproxy.js'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
+    updater = require('./updater.js'),
     updateChecker = require('./updatechecker.js');
 
 var gJobs = {
@@ -207,7 +208,7 @@ function boxAutoupdatePatternChanged(pattern) {
             var updateInfo = updateChecker.getUpdateInfo();
             if (updateInfo.box) {
                 debug('Starting autoupdate to %j', updateInfo.box);
-                cloudron.updateToLatest(AUDIT_SOURCE, NOOP_CALLBACK);
+                updater.updateToLatest(AUDIT_SOURCE, NOOP_CALLBACK);
             } else {
                 debug('No box auto updates available');
             }

src/database.js

@@ -91,8 +91,7 @@ function clear(callback) {
 
     async.series([
         child_process.exec.bind(null, cmd),
-        require('./clientdb.js')._addDefaultClients,
-        require('./groupdb.js')._addDefaultGroups
+        require('./clientdb.js')._addDefaultClients
     ], callback);
 }
 

src/digest.js

@@ -28,42 +28,36 @@ function maybeSend(callback) {
         var pendingAppUpdates = updateInfo.apps || {};
         pendingAppUpdates = Object.keys(pendingAppUpdates).map(function (key) { return pendingAppUpdates[key]; });
 
-        appstore.getSubscription(function (error, subscription) {
-            if (error) debug('Error getting subscription:', error);
+        eventlog.getByCreationTime(new Date(new Date() - 7*86400000), function (error, events) {
+            if (error) return callback(error);
 
-            eventlog.getByCreationTime(new Date(new Date() - 7*86400000), function (error, events) {
-                if (error) return callback(error);
+            var appUpdates = events.filter(function (e) { return e.action === eventlog.ACTION_APP_UPDATE; }).map(function (e) { return e.data; });
+            var boxUpdates = events.filter(function (e) { return e.action === eventlog.ACTION_UPDATE; }).map(function (e) { return e.data; });
+            var certRenewals = events.filter(function (e) { return e.action === eventlog.ACTION_CERTIFICATE_RENEWAL; }).map(function (e) { return e.data; });
+            var usersAdded = events.filter(function (e) { return e.action === eventlog.ACTION_USER_ADD; }).map(function (e) { return e.data; });
+            var usersRemoved = events.filter(function (e) { return e.action === eventlog.ACTION_USER_REMOVE; }).map(function (e) { return e.data; });
+            var finishedBackups = events.filter(function (e) { return e.action === eventlog.ACTION_BACKUP_FINISH && !e.errorMessage; }).map(function (e) { return e.data; });
 
-                var appUpdates = events.filter(function (e) { return e.action === eventlog.ACTION_APP_UPDATE; }).map(function (e) { return e.data; });
-                var boxUpdates = events.filter(function (e) { return e.action === eventlog.ACTION_UPDATE; }).map(function (e) { return e.data; });
-                var certRenewals = events.filter(function (e) { return e.action === eventlog.ACTION_CERTIFICATE_RENEWAL; }).map(function (e) { return e.data; });
-                var usersAdded = events.filter(function (e) { return e.action === eventlog.ACTION_USER_ADD; }).map(function (e) { return e.data; });
-                var usersRemoved = events.filter(function (e) { return e.action === eventlog.ACTION_USER_REMOVE; }).map(function (e) { return e.data; });
-                var finishedBackups = events.filter(function (e) { return e.action === eventlog.ACTION_BACKUP_FINISH && !e.errorMessage; }).map(function (e) { return e.data; });
+            if (error) return callback(error);
 
-                if (error) return callback(error);
+            var info = {
+                pendingAppUpdates: pendingAppUpdates,
+                pendingBoxUpdate: updateInfo.box || null,
 
-                var info = {
-                    hasSubscription: appstore.isFreePlan(subscription),
+                finishedAppUpdates: appUpdates,
+                finishedBoxUpdates: boxUpdates,
 
-                    pendingAppUpdates: pendingAppUpdates,
-                    pendingBoxUpdate: updateInfo.box || null,
+                certRenewals: certRenewals,
+                finishedBackups: finishedBackups, // only the successful backups
+                usersAdded: usersAdded,
+                usersRemoved: usersRemoved // unused because we don't have username to work with
+            };
 
-                    finishedAppUpdates: appUpdates,
-                    finishedBoxUpdates: boxUpdates,
+            // always send digest for backup failure notification
+            debug('maybeSend: sending digest email', info);
+            mailer.sendDigest(info);
 
-                    certRenewals: certRenewals,
-                    finishedBackups: finishedBackups, // only the successful backups
-                    usersAdded: usersAdded,
-                    usersRemoved: usersRemoved // unused because we don't have username to work with
-                };
-
-                // always send digest for backup failure notification
-                debug('maybeSend: sending digest email', info);
-                mailer.sendDigest(info);
-
-                callback();
-            });
+            callback();
         });
     });
 }

src/dns/caas.js

@@ -50,7 +50,7 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
             if (result.statusCode === 420) return callback(new DomainsError(DomainsError.STILL_BUSY));
             if (result.statusCode !== 201) return callback(new DomainsError(DomainsError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
 
-            return callback(null, result.body.changeId);
+            return callback(null);
         });
 }
 
@@ -126,10 +126,26 @@ function verifyDnsConfig(dnsConfig, domain, zoneName, ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'token must be a non-empty string'));
+
     var credentials = {
         token: dnsConfig.token,
         fqdn: domain
     };
 
-    return callback(null, credentials);
+    const testSubdomain = 'cloudrontestdns';
+
+    upsert(credentials, zoneName, testSubdomain, 'A', [ ip ], function (error, changeId) {
+        if (error) return callback(error);
+
+        debug('verifyDnsConfig: Test A record added with change id %s', changeId);
+
+        del(credentials, zoneName, testSubdomain, 'A', [ ip ], function (error) {
+            if (error) return callback(error);
+
+            debug('verifyDnsConfig: Test A record removed again');
+
+            callback(null, credentials);
+        });
+    });
 }

src/dns/cloudflare.js

@@ -152,11 +152,7 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
                             callback(null);
                         });
                 }
-            }, function (error) {
-                if (error) return callback(error);
-
-                callback(null, 'unused');
-            });
+            }, callback);
         });
     });
 }

src/dns/digitalocean.js

@@ -129,11 +129,7 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
                         return iteratorCallback(null);
                     });
             }
-        }, function (error) {
-            if (error) return callback(error);
-
-            callback(null, '' + recordIds[0]); // DO ids are integers
-        });
+        }, callback);
     });
 }
 
@@ -204,6 +200,8 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'token must be a non-empty string'));
+
     var credentials = {
         token: dnsConfig.token
     };

src/dns/gandi.js

@@ -48,7 +48,7 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
             if (result.statusCode === 400) return callback(new DomainsError(DomainsError.BAD_FIELD, formatError(result)));
             if (result.statusCode !== 201) return callback(new DomainsError(DomainsError.EXTERNAL_ERROR, formatError(result)));
 
-            return callback(null, 'unused-id');
+            return callback(null);
         });
 }
 
@@ -112,6 +112,8 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'token must be a non-empty string'));
+
     var credentials = {
         token: dnsConfig.token
     };

src/dns/gcdns.js

@@ -19,19 +19,13 @@ var assert = require('assert'),
 function getDnsCredentials(dnsConfig) {
     assert.strictEqual(typeof dnsConfig, 'object');
 
-    var config = {
+    return {
         projectId: dnsConfig.projectId,
-        keyFilename: dnsConfig.keyFilename,
-        email: dnsConfig.email
-    };
-
-    if (dnsConfig.credentials) {
-        config.credentials = {
+        credentials: {
             client_email: dnsConfig.credentials.client_email,
             private_key: dnsConfig.credentials.private_key
-        };
-    }
-    return config;
+        }
+    };
 }
 
 function getZoneByName(dnsConfig, zoneName, callback) {
@@ -97,7 +91,7 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
                     return callback(new DomainsError(DomainsError.EXTERNAL_ERROR, error.message));
                 }
 
-                callback(null, change.id);
+                callback(null);
             });
         });
     });
@@ -169,6 +163,11 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    if (typeof dnsConfig.projectId !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'projectId must be a string'));
+    if (!dnsConfig.credentials || typeof dnsConfig.credentials !== 'object') return callback(new DomainsError(DomainsError.BAD_FIELD, 'credentials must be an object'));
+    if (typeof dnsConfig.credentials.client_email !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'credentials.client_email must be a string'));
+    if (typeof dnsConfig.credentials.private_key !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'credentials.private_key must be a string'));
+
     var credentials = getDnsCredentials(dnsConfig);
     if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
 

src/dns/godaddy.js

@@ -63,7 +63,7 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
             if (result.statusCode === 422) return callback(new DomainsError(DomainsError.BAD_FIELD, formatError(result))); // conflict
             if (result.statusCode !== 200) return callback(new DomainsError(DomainsError.EXTERNAL_ERROR, formatError(result)));
 
-            return callback(null, 'unused-id');
+            return callback(null);
         });
 }
 
@@ -146,6 +146,9 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    if (!dnsConfig.apiKey || typeof dnsConfig.apiKey !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'apiKey must be a non-empty string'));
+    if (!dnsConfig.apiSecret || typeof dnsConfig.apiSecret !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'apiSecret must be a non-empty string'));
+
     var credentials = {
         apiKey: dnsConfig.apiKey,
         apiSecret: dnsConfig.apiSecret

src/dns/interface.js

@@ -26,7 +26,7 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    // Result: backend specific change id, to be passed into getChangeStatus()
+    // Result: none
 
     callback(new Error('not implemented'));
 }

src/dns/manual.js

@@ -24,7 +24,7 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
 
     debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    return callback(null, 'noop-record-id');
+    return callback(null);
 }
 
 function get(dnsConfig, zoneName, subdomain, type, callback) {

src/dns/namecom.js

@@ -90,7 +90,7 @@ function updateRecord(dnsConfig, zoneName, recordId, subdomain, type, values, ca
             if (result.statusCode === 403) return callback(new DomainsError(DomainsError.ACCESS_DENIED, formatError(result)));
             if (result.statusCode !== 200) return callback(new DomainsError(DomainsError.EXTERNAL_ERROR, formatError(result)));
 
-            return callback(null, 'unused-id');
+            return callback(null);
         });
 }
 

src/dns/noop.js

@@ -22,7 +22,7 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
 
     debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    return callback(null, 'noop-record-id');
+    return callback(null);
 }
 
 function get(dnsConfig, zoneName, subdomain, type, callback) {

src/dns/route53.js

@@ -121,7 +121,7 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
             if (error && error.code === 'InvalidChangeBatch') return callback(new DomainsError(DomainsError.BAD_FIELD, error.message));
             if (error) return callback(new DomainsError(DomainsError.EXTERNAL_ERROR, error.message));
 
-            callback(null, result.ChangeInfo.Id);
+            callback(null);
         });
     });
 }
@@ -233,6 +233,9 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    if (!dnsConfig.accessKeyId || typeof dnsConfig.accessKeyId !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'accessKeyId must be a non-empty string'));
+    if (!dnsConfig.secretAccessKey || typeof dnsConfig.secretAccessKey !== 'string') return callback(new DomainsError(DomainsError.BAD_FIELD, 'secretAccessKey must be a non-empty string'));
+
     var credentials = {
         accessKeyId: dnsConfig.accessKeyId,
         secretAccessKey: dnsConfig.secretAccessKey,

src/docker.js

@@ -179,6 +179,14 @@ function createSubcontainer(app, name, cmd, options, callback) {
             },
             HostConfig: {
                 Binds: addons.getBindsSync(app, app.manifest.addons),
+                LogConfig: {
+                    Type: 'syslog',
+                    Config: {
+                        'tag': app.id,
+                        'syslog-address': 'udp://127.0.0.1:2514', // see apps.js:validatePortBindings()
+                        'syslog-format': 'rfc5424'
+                    }
+                },
                 Memory: memoryLimit / 2,
                 MemorySwap: memoryLimit, // Memory + Swap
                 PortBindings: isAppContainer ? dockerPortBindings : { },

src/domains.js

@@ -17,6 +17,7 @@ module.exports = exports = {
     waitForDnsRecord: waitForDnsRecord,
 
     removePrivateFields: removePrivateFields,
+    removeRestrictedFields: removeRestrictedFields,
 
     DomainsError: DomainsError
 };
@@ -114,9 +115,11 @@ function add(domain, zoneName, provider, config, fallbackCertificate, tlsConfig,
     assert.strictEqual(typeof callback, 'function');
 
     if (!tld.isValid(domain)) return callback(new DomainsError(DomainsError.BAD_FIELD, 'Invalid domain'));
+    if (domain.endsWith('.')) return callback(new DomainsError(DomainsError.BAD_FIELD, 'Invalid domain'));
 
     if (zoneName) {
         if (!tld.isValid(zoneName)) return callback(new DomainsError(DomainsError.BAD_FIELD, 'Invalid zoneName'));
+        if (zoneName.endsWith('.')) return callback(new DomainsError(DomainsError.BAD_FIELD, 'Invalid zoneName'));
     } else {
         zoneName = tld.getDomain(domain) || domain;
     }
@@ -298,10 +301,10 @@ function upsertDnsRecords(subdomain, domain, type, values, callback) {
     get(domain, function (error, result) {
         if (error) return callback(new DomainsError(DomainsError.INTERNAL_ERROR, error));
 
-        api(result.provider).upsert(result.config, result.zoneName, getName(result, subdomain), type, values, function (error, changeId) {
+        api(result.provider).upsert(result.config, result.zoneName, getName(result, subdomain), type, values, function (error) {
             if (error) return callback(error);
 
-            callback(null, changeId);
+            callback(null);
         });
     });
 }
@@ -370,8 +373,15 @@ function fqdn(location, domain, provider) {
     return location + (location ? (provider !== 'caas' ? '.' : '-') : '') + domain;
 }
 
+// removes all fields that are strictly private and should never be returned by API calls
 function removePrivateFields(domain) {
     var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate');
     if (result.fallbackCertificate) delete result.fallbackCertificate.key;  // do not return the 'key'. in caas, this is private
     return result;
 }
+
+// removes all fields that are not accessible by a normal user
+function removeRestrictedFields(domain) {
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider');
+    return result;
+}

src/eventlog.js

@@ -27,7 +27,8 @@ exports = module.exports = {
     ACTION_USER_ADD: 'user.add',
     ACTION_USER_LOGIN: 'user.login',
     ACTION_USER_REMOVE: 'user.remove',
-    ACTION_USER_UPDATE: 'user.update'
+    ACTION_USER_UPDATE: 'user.update',
+    ACTION_USER_TRANSFER: 'user.transfer',
 };
 
 var assert = require('assert'),

src/groupdb.js

@@ -6,6 +6,7 @@ exports = module.exports = {
     getAll: getAll,
     getAllWithMembers: getAllWithMembers,
     add: add,
+    update: update,
     del: del,
     count: count,
 
@@ -15,15 +16,15 @@ exports = module.exports = {
     setMembers: setMembers,
     isMember: isMember,
 
-    getGroups: getGroups,
-    setGroups: setGroups,
+    getMembership: getMembership,
+    setMembership: setMembership,
 
-    _clear: clear,
-    _addDefaultGroups: addDefaultGroups
+    getGroups: getGroups,
+
+    _clear: clear
 };
 
 var assert = require('assert'),
-    constants = require('./constants.js'),
     database = require('./database.js'),
     DatabaseError = require('./databaseerror');
 
@@ -62,10 +63,10 @@ function getWithMembers(groupId, callback) {
 function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups', function (error, result) {
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups', function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        callback(null, result);
+        callback(null, results);
     });
 }
 
@@ -95,6 +96,31 @@ function add(id, name, callback) {
     });
 }
 
+function update(id, data, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var args = [ ];
+    var fields = [ ];
+    for (var k in data) {
+        if (k === 'name') {
+            assert.strictEqual(typeof data.name, 'string');
+            fields.push(k + ' = ?');
+            args.push(data.name);
+        }
+    }
+    args.push(id);
+
+    database.query('UPDATE groups SET ' + fields.join(', ') + ' WHERE id = ?', args, function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('groups_name') !== -1) return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, 'name already exists'));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null);
+    });
+}
+
 function del(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -126,7 +152,7 @@ function clear(callback) {
     database.query('DELETE FROM groupMembers', function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        database.query('DELETE FROM groups WHERE id != ?', [ 'admin' ], function (error) {
+        database.query('DELETE FROM groups', function (error) {
             if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
             callback(error);
@@ -165,7 +191,7 @@ function setMembers(groupId, userIds, callback) {
     });
 }
 
-function getGroups(userId, callback) {
+function getMembership(userId, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
@@ -177,7 +203,7 @@ function getGroups(userId, callback) {
     });
 }
 
-function setGroups(userId, groupIds, callback) {
+function setMembership(userId, groupIds, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert(Array.isArray(groupIds));
     assert.strictEqual(typeof callback, 'function');
@@ -235,6 +261,14 @@ function isMember(groupId, userId, callback) {
     });
 }
 
-function addDefaultGroups(callback) {
-    add(constants.ADMIN_GROUP_ID, 'admin', callback);
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' ' +
+        ' FROM groups INNER JOIN groupMembers ON groups.id = groupMembers.groupId AND groupMembers.userId = ?', [ userId ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
 }

src/groups.js

@@ -6,6 +6,7 @@ exports = module.exports = {
     create: create,
     remove: remove,
     get: get,
+    update: update,
     getWithMembers: getWithMembers,
     getAll: getAll,
     getAllWithMembers: getAllWithMembers,
@@ -17,7 +18,9 @@ exports = module.exports = {
     isMember: isMember,
 
     getGroups: getGroups,
-    setGroups: setGroups
+
+    setMembership: setMembership,
+    getMembership: getMembership
 };
 
 var assert = require('assert'),
@@ -25,7 +28,8 @@ var assert = require('assert'),
     DatabaseError = require('./databaseerror.js'),
     groupdb = require('./groupdb.js'),
     util = require('util'),
-    uuid = require('uuid');
+    uuid = require('uuid'),
+    _ = require('underscore');
 
 // http://dustinsenos.com/articles/customErrorsInNode
 // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
@@ -64,11 +68,8 @@ function validateGroupname(name) {
 
     if (constants.RESERVED_NAMES.indexOf(name) !== -1) return new GroupsError(GroupsError.BAD_FIELD, 'name is reserved');
 
-    // +/- can be tricky in emails. also need to consider valid LDAP characters here (e.g '+' is reserved)
-    if (/[^a-zA-Z0-9.]/.test(name)) return new GroupsError(GroupsError.BAD_FIELD, 'name can only contain alphanumerals and dot');
-
-    // app emails are sent using the .app suffix
-    if (name.indexOf('.app') !== -1) return new GroupsError(GroupsError.BAD_FIELD, 'name pattern is reserved for apps');
+    // need to consider valid LDAP characters here (e.g '+' is reserved)
+    if (/[^a-zA-Z0-9.-]/.test(name)) return new GroupsError(GroupsError.BAD_FIELD, 'name can only contain alphanumerals, hyphen and dot');
 
     return null;
 }
@@ -96,9 +97,6 @@ function remove(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    // never allow admin group to be deleted
-    if (id === constants.ADMIN_GROUP_ID) return callback(new GroupsError(GroupsError.NOT_ALLOWED));
-
     groupdb.del(id, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupsError(GroupsError.NOT_FOUND));
         if (error) return callback(new GroupsError(GroupsError.INTERNAL_ERROR, error));
@@ -163,11 +161,11 @@ function getMembers(groupId, callback) {
     });
 }
 
-function getGroups(userId, callback) {
+function getMembership(userId, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    groupdb.getGroups(userId, function (error, result) {
+    groupdb.getMembership(userId, function (error, result) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupsError(GroupsError.NOT_FOUND));
         if (error) return callback(new GroupsError(GroupsError.INTERNAL_ERROR, error));
 
@@ -175,12 +173,12 @@ function getGroups(userId, callback) {
     });
 }
 
-function setGroups(userId, groupIds, callback) {
+function setMembership(userId, groupIds, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert(Array.isArray(groupIds));
     assert.strictEqual(typeof callback, 'function');
 
-    groupdb.setGroups(userId, groupIds, function (error) {
+    groupdb.setMembership(userId, groupIds, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupsError(GroupsError.NOT_FOUND));
         if (error) return callback(new GroupsError(GroupsError.INTERNAL_ERROR, error));
 
@@ -239,3 +237,34 @@ function isMember(groupId, userId, callback) {
         return callback(null, result);
     });
 }
+
+function update(groupId, data, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    let error;
+    if ('name' in data) {
+        assert.strictEqual(typeof data.name, 'string');
+        error = validateGroupname(data.name);
+        if (error) return callback(error);
+    }
+
+    groupdb.update(groupId, _.pick(data, 'name'), function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupsError(GroupsError.NOT_FOUND));
+        if (error) return callback(new GroupsError(GroupsError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getGroups(userId, function (error, results) {
+        if (error) return callback(new GroupsError(GroupsError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}

src/hat.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports = module.exports = hat;
+
+var crypto = require('crypto');
+
+function hat (bits) {
+    return crypto.randomBytes(bits / 8).toString('hex');
+}

src/infra_version.js

@@ -7,7 +7,7 @@
 exports = module.exports = {
     // a major version makes all apps restore from backup. #451 must be fixed before we do this.
     // a minor version makes all apps re-configure themselves
-    'version': '48.10.0',
+    'version': '48.11.0',
 
     'baseImages': [ 'cloudron/base:0.10.0' ],
 
@@ -18,7 +18,7 @@ exports = module.exports = {
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:1.1.0' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:1.1.0' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:1.0.0' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:1.3.0' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:1.4.0' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:1.0.0' }
     }
 };

src/ldap.js

@@ -10,6 +10,7 @@ var assert = require('assert'),
     apps = require('./apps.js'),
     async = require('async'),
     config = require('./config.js'),
+    constants = require('./constants.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:ldap'),
     eventlog = require('./eventlog.js'),
@@ -51,13 +52,13 @@ function getUsersWithAccessToApp(req, callback) {
     getAppByRequest(req, function (error, app) {
         if (error) return callback(error);
 
-        users.list(function (error, result){
+        users.list(function (error, result) {
             if (error) return callback(new ldap.OperationsError(error.toString()));
 
-            async.filter(result, apps.hasAccessTo.bind(null, app), function (error, result) {
+            async.filter(result, apps.hasAccessTo.bind(null, app), function (error, allowedUsers) {
                 if (error) return callback(new ldap.OperationsError(error.toString()));
 
-                callback(null, result);
+                callback(null, allowedUsers);
             });
         });
     });

src/mail.js

@@ -8,9 +8,8 @@ exports = module.exports = {
     getDomain: getDomain,
     addDomain: addDomain,
     removeDomain: removeDomain,
-    updateDomain: updateDomain,
 
-    addDnsRecords: addDnsRecords,
+    setDnsRecords: setDnsRecords,
 
     validateName: validateName,
 
@@ -47,8 +46,6 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
-    appstore = require('./appstore.js'),
-    AppstoreError = appstore.AppstoreError,
     config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:mail'),
@@ -108,8 +105,8 @@ function validateName(name) {
     if (name.length < 1) return new MailError(MailError.BAD_FIELD, 'mailbox name must be atleast 1 char');
     if (name.length >= 200) return new MailError(MailError.BAD_FIELD, 'mailbox name too long');
 
-    // +/- can be tricky in emails. also need to consider valid LDAP characters here (e.g '+' is reserved)
-    if (/[^a-zA-Z0-9.]/.test(name)) return new MailError(MailError.BAD_FIELD, 'mailbox name can only contain alphanumerals and dot');
+    // also need to consider valid LDAP characters here (e.g '+' is reserved)
+    if (/[^a-zA-Z0-9.-]/.test(name)) return new MailError(MailError.BAD_FIELD, 'mailbox name can only contain alphanumerals and dot');
 
     // app emails are sent using the .app suffix
     if (name.indexOf('.app') !== -1) return new MailError(MailError.BAD_FIELD, 'mailbox name pattern is reserved for apps');
@@ -164,6 +161,8 @@ function checkSmtpRelay(relay, callback) {
     };
 
     var transporter = nodemailer.createTransport(smtpTransport({
+        connectionTimeout: 5000,
+        greetingTimeout: 5000,
         host: relay.host,
         port: relay.port,
         auth: {
@@ -201,6 +200,7 @@ function verifyRelay(relay, callback) {
 function checkDkim(domain, callback) {
     var dkim = {
         domain: config.dkimSelector() + '._domainkey.' + domain,
+        name: config.dkimSelector() + '._domainkey',
         type: 'TXT',
         expected: null,
         value: null,
@@ -227,6 +227,7 @@ function checkDkim(domain, callback) {
 function checkSpf(domain, callback) {
     var spf = {
         domain: domain,
+        name: '@',
         type: 'TXT',
         value: null,
         expected: 'v=spf1 a:' + config.mailFqdn() + ' ~all',
@@ -258,6 +259,7 @@ function checkSpf(domain, callback) {
 function checkMx(domain, callback) {
     var mx = {
         domain: domain,
+        name: '@',
         type: 'MX',
         value: null,
         expected: '10 ' + config.mailFqdn() + '.',
@@ -279,6 +281,7 @@ function checkMx(domain, callback) {
 function checkDmarc(domain, callback) {
     var dmarc = {
         domain: '_dmarc.' + domain,
+        name: '_dmarc',
         type: 'TXT',
         value: null,
         expected: 'v=DMARC1; p=reject; pct=100',
@@ -562,6 +565,10 @@ function restartMail(callback) {
             const cmd = `docker run --restart=always -d --name="mail" \
                         --net cloudron \
                         --net-alias mail \
+                        --log-driver syslog \
+                        --log-opt syslog-address=udp://127.0.0.1:2514 \
+                        --log-opt syslog-format=rfc5424 \
+                        --log-opt tag=mail \
                         -m ${memoryLimit}m \
                         --memory-swap ${memoryLimit * 2}m \
                         --dns 172.18.0.1 \
@@ -685,40 +692,49 @@ function readDkimPublicKeySync(domain) {
     return publicKey;
 }
 
-function addDnsRecords(domain, callback) {
+function setDnsRecords(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var error = ensureDkimKeySync(domain);
-    if (error) return callback(error);
+    maildb.get(domain, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND));
+        if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
-    if (process.env.BOX_ENV === 'test') return callback();
-
-    var dkimKey = readDkimPublicKeySync(domain);
-    if (!dkimKey) return callback(new MailError(MailError.INTERNAL_ERROR, new Error('Failed to read dkim public key')));
-
-    // t=s limits the domainkey to this domain and not it's subdomains
-    var dkimRecord = { subdomain: config.dkimSelector() + '._domainkey', domain: domain, type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
-
-    var records = [ ];
-    records.push(dkimRecord);
-
-    debug('addDnsRecords: %j', records);
-
-    txtRecordsWithSpf(domain, function (error, txtRecords) {
+        error = ensureDkimKeySync(domain);
         if (error) return callback(error);
 
-        if (txtRecords) records.push({ subdomain: '', domain: domain, type: 'TXT', values: txtRecords });
+        if (process.env.BOX_ENV === 'test') return callback();
 
-        debug('addDnsRecords: will update %j', records);
+        var dkimKey = readDkimPublicKeySync(domain);
+        if (!dkimKey) return callback(new MailError(MailError.INTERNAL_ERROR, new Error('Failed to read dkim public key')));
 
-        async.mapSeries(records, function (record, iteratorCallback) {
-            domains.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values, iteratorCallback);
-        }, function (error, changeIds) {
-            if (error) debug('addDnsRecords: failed to update : %s. will retry', error);
-            else debug('addDnsRecords: records %j added with changeIds %j', records, changeIds);
+        // t=s limits the domainkey to this domain and not it's subdomains
+        var dkimRecord = { subdomain: config.dkimSelector() + '._domainkey', domain: domain, type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
 
-            callback(error);
+        var records = [ ];
+        records.push(dkimRecord);
+        if (result.enabled) {
+            records.push({ subdomain: '_dmarc', domain: domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] });
+            records.push({ subdomain: '', domain: domain, type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] });
+        }
+
+        debug('addDnsRecords: %j', records);
+
+        txtRecordsWithSpf(domain, function (error, txtRecords) {
+            if (error) return callback(error);
+
+            if (txtRecords) records.push({ subdomain: '', domain: domain, type: 'TXT', values: txtRecords });
+
+            debug('addDnsRecords: will update %j', records);
+
+            async.mapSeries(records, function (record, iteratorCallback) {
+                domains.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values, iteratorCallback);
+            }, function (error, changeIds) {
+                if (error) debug('addDnsRecords: failed to update : %s. will retry', error);
+                else debug('addDnsRecords: records %j added with changeIds %j', records, changeIds);
+
+                callback(error);
+            });
         });
     });
 }
@@ -733,7 +749,7 @@ function addDomain(domain, callback) {
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
         async.series([
-            addDnsRecords.bind(null, domain), // do this first to ensure DKIM keys
+            setDnsRecords.bind(null, domain), // do this first to ensure DKIM keys
             restartMail
         ], NOOP_CALLBACK); // do these asynchronously
 
@@ -741,20 +757,6 @@ function addDomain(domain, callback) {
     });
 }
 
-// this is just a way to resync the mail "dns" records via the UI
-function updateDomain(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    getDomain(domain, function (error) {
-        if (error) return callback(error);
-
-        addDnsRecords(domain, NOOP_CALLBACK);
-
-        callback();
-    });
-}
-
 function removeDomain(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -809,6 +811,7 @@ function setMailRelay(domain, relay, callback) {
         if (error) return callback(error);
 
         maildb.update(domain, { relay: relay }, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND));
             if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
             restartMail(NOOP_CALLBACK);
@@ -823,33 +826,13 @@ function setMailEnabled(domain, enabled, callback) {
     assert.strictEqual(typeof enabled, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
-    appstore.getSubscription(function (error, result) {
-        if (error && error.reason === AppstoreError.BILLING_REQUIRED) return callback(new MailError(MailError.BILLING_REQUIRED));
-        if (error) return callback(new MailError(MailError.EXTERNAL_ERROR, error));
+    maildb.update(domain, { enabled: enabled }, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND));
+        if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
-        // we only allow enabling email on a paid plan
-        if (enabled && appstore.isFreePlan(result)) return callback(new MailError(MailError.BILLING_REQUIRED));
+        restartMail(NOOP_CALLBACK);
 
-        maildb.update(domain, { enabled: enabled }, function (error) {
-            if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-
-            restartMail(NOOP_CALLBACK);
-
-            if (!enabled || process.env.BOX_ENV === 'test') return callback(null);
-
-            // Add MX and DMARC record. Note that DMARC policy depends on DKIM signing and thus works
-            // only if we use our internal mail server.
-            var records = [
-                { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] },
-                { subdomain: '', type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] }
-            ];
-
-            async.mapSeries(records, function (record, iteratorCallback) {
-                domains.upsertDnsRecords(record.subdomain, domain, record.type, record.values, iteratorCallback);
-            }, NOOP_CALLBACK);
-
-            callback(null);
-        });
+        callback(null);
     });
 }
 
@@ -997,7 +980,7 @@ function setAliases(name, domain, aliases, callback) {
 
     mailboxdb.setAliasesForName(name, domain, aliases, function (error) {
         if (error && error.reason === DatabaseError.ALREADY_EXISTS && error.message.indexOf('mailboxes_name_domain_unique_index') !== -1) {
-            var aliasMatch = error.message.match(new RegExp(`^ER_DUP_ENTRY: Duplicate entry '(.*)-${domain}' for key 'mailboxes_name_domain_unique_index'$`))
+            var aliasMatch = error.message.match(new RegExp(`^ER_DUP_ENTRY: Duplicate entry '(.*)-${domain}' for key 'mailboxes_name_domain_unique_index'$`));
             if (!aliasMatch) return callback(new MailError(MailError.ALREADY_EXISTS, error.message));
             return callback(new MailError(MailError.ALREADY_EXISTS, `Mailbox, mailinglist or alias for ${aliasMatch[1]} already exists`));
         }

src/mail_templates/digest.ejs

@@ -53,11 +53,6 @@ Last successful backup: <%- info.finishedBackups[0].backupId || info.finishedBac
 This Cloudron did **not** backup successfully in the last week!
 <% } -%>
 
-<% if (!info.hasSubscription) { -%>
-
-*Keep your Cloudron automatically up-to-date and secure by upgrading to a paid plan at* <%= webadminUrl %>/#/settings
-<% } -%>
-
 Powered by https://cloudron.io
 
 Sent at: <%= new Date().toUTCString() %>
@@ -156,12 +151,6 @@ Sent at: <%= new Date().toUTCString() %>
     <% } %>
 
         <br/>
-
-    <% if (!info.hasSubscription) { %>
-        Keep your Cloudron automatically up-to-date and secure by upgrading to a <a href="<%= webadminUrl %>/#/settings">paid plan</a>.
-    <% } %>
-
-        <br/>
         <br/>
         <br/>
 

src/mailer.js

@@ -202,7 +202,7 @@ function sendInvite(user, invitor) {
         var templateData = {
             user: user,
             webadminUrl: config.adminOrigin(),
-            setupLink: config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken,
+            setupLink: `${config.adminOrigin()}/api/v1/session/account/setup.html?reset_token=${user.resetToken}&email=${encodeURIComponent(user.email)}`,
             invitor: invitor,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: config.adminOrigin() + '/api/v1/cloudron/avatar'
@@ -239,7 +239,7 @@ function userAdded(user, inviteSent) {
 
         var templateData = {
             user: user,
-            inviteLink: inviteSent ? null : config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken,
+            inviteLink: inviteSent ? null : `${config.adminOrigin()}/api/v1/session/account/setup.html?reset_token=${user.resetToken}&email=${encodeURIComponent(user.email)}`,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: config.adminOrigin() + '/api/v1/cloudron/avatar'
         };
@@ -289,7 +289,7 @@ function passwordReset(user) {
 
         var templateData = {
             user: user,
-            resetLink: config.adminOrigin() + '/api/v1/session/password/reset.html?reset_token=' + user.resetToken,
+            resetLink: `${config.adminOrigin()}/api/v1/session/password/reset.html?reset_token=${user.resetToken}&email=${encodeURIComponent(user.email)}`,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: config.adminOrigin() + '/api/v1/cloudron/avatar'
         };
@@ -429,7 +429,7 @@ function sendDigest(info) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailConfig.adminEmails.join(', '),
-            subject: util.format('[%s] Cloudron - Weekly activity digest', mailConfig.cloudronName),
+            subject: util.format('[%s] Weekly activity digest', mailConfig.cloudronName),
             text: render('digest.ejs', templateDataText),
             html: render('digest.ejs', templateDataHTML)
         };

src/oauth2views/account_setup.ejs

@@ -19,8 +19,8 @@ app.controller('Controller', ['$scope', function ($scope) {
 
   <center>
     <br/>
-    <h4>Hello <%= (user && user.email) ? user.email : '' %>, welcome to <%= cloudronName %>.</h4>
-    <h2>Setup your account and password.</h2>
+    <h4>Hello <%= (user && user.email) ? user.email : '' %>, welcome to <%= cloudronName %>!</h4>
+    <h2>Setup your account and password</h2>
   </center>
 
   <div class="container" ng-app="Application" ng-controller="Controller">
@@ -29,6 +29,7 @@ app.controller('Controller', ['$scope', function ($scope) {
         <form action="/api/v1/session/account/setup" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
           <input type="password" style="display: none;">
           <input type="hidden" name="_csrf" value="<%= csrf %>"/>
+          <input type="hidden" name="email" value="<%= email %>"/>
           <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
 
           <center><p class="has-error"><%= error %></p></center>
@@ -51,16 +52,16 @@ app.controller('Controller', ['$scope', function ($scope) {
 <% } %>
 
           <div class="form-group">
-            <label class="control-label">Display Name</label>
+            <label class="control-label">Full Name</label>
             <input type="displayName" class="form-control" ng-model="displayName" name="displayName" required>
           </div>
 
           <div class="form-group" ng-class="{ 'has-error': (setupForm.password.$dirty && setupForm.password.$invalid) }">
             <label class="control-label">New Password</label>
             <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
-              <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
+              <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be atleast 8 characters</small>
             </div>
-            <input type="password" class="form-control" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" required>
+            <input type="password" class="form-control" ng-model="password" name="password" ng-pattern="/^.{8,30}$/" required>
           </div>
 
           <div class="form-group" ng-class="{ 'has-error': (setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)) }">
@@ -71,7 +72,7 @@ app.controller('Controller', ['$scope', function ($scope) {
             <input type="password" class="form-control" ng-model="passwordRepeat" name="passwordRepeat" required>
           </div>
 
-          <input class="btn btn-primary btn-outline pull-right" type="submit" value="Create" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/>
+          <center><input class="btn btn-primary btn-outline" type="submit" value="Setup" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/></center>
         </form>
       </div>
     </div>

src/oauth2views/password_reset.ejs

@@ -26,14 +26,15 @@ app.controller('Controller', [function () {}]);
         <form action="/api/v1/session/password/reset" method="post" name="resetForm" autocomplete="off" role="form" novalidate>
           <input type="password" style="display: none;">
           <input type="hidden" name="_csrf" value="<%= csrf %>"/>
+          <input type="hidden" name="email" value="<%= email %>"/>
           <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
 
           <div class="form-group" ng-class="{ 'has-error': resetForm.password.$dirty && resetForm.password.$invalid }">
             <label class="control-label" for="inputPassword">New Password</label>
             <div class="control-label" ng-show="resetForm.password.$dirty && resetForm.password.$invalid">
-              <small ng-show="resetForm.password.$dirty && resetForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
+              <small ng-show="resetForm.password.$dirty && resetForm.password.$invalid">Password must be atleast 8 characters</small>
             </div>
-            <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" autofocus required>
+            <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^.{8,30}$/" autofocus required>
           </div>
           <div class="form-group" ng-class="{ 'has-error': resetForm.passwordRepeat.$dirty && (password !== passwordRepeat) }">
             <label class="control-label" for="inputPasswordRepeat">Repeat Password</label>

src/password.js

@@ -1,47 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-// From https://www.npmjs.com/package/password-generator
-
-exports = module.exports = {
-    generate: generate,
-    validate: validate
-};
-
-var assert = require('assert'),
-    generatePassword = require('password-generator');
-
-// http://www.w3resource.com/javascript/form/example4-javascript-form-validation-password.html
-// WARNING!!! if this is changed, the UI parts in the setup and account view have to be adjusted!
-var gPasswordTestRegExp = /^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/;
-
-var UPPERCASE_RE = /([A-Z])/g;
-var LOWERCASE_RE = /([a-z])/g;
-var NUMBER_RE = /([\d])/g;
-var SPECIAL_CHAR_RE = /([\?\-])/g;
-
-function isStrongEnough(password) {
-    var uc = password.match(UPPERCASE_RE);
-    var lc = password.match(LOWERCASE_RE);
-    var n = password.match(NUMBER_RE);
-    var sc = password.match(SPECIAL_CHAR_RE);
-
-    return uc && lc && n && sc;
-}
-
-function generate() {
-    var password = '';
-
-    while (!isStrongEnough(password)) password = generatePassword(8, false, /[\w\d\?\-]/);
-
-    return password;
-}
-
-function validate(password) {
-    assert.strictEqual(typeof password, 'string');
-
-    if (!password.match(gPasswordTestRegExp)) return new Error('Password must be 8-30 character with at least one uppercase, one numeric and one special character');
-
-    return null;
-}

src/paths.js

@@ -8,7 +8,6 @@ exports = module.exports = {
     CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
     INFRA_VERSION_FILE: path.join(config.baseDir(), 'platformdata/INFRA_VERSION'),
     BACKUP_RESULT_FILE: path.join(config.baseDir(), 'platformdata/backup/result.txt'),
-    BACKUP_LOG_FILE: path.join(config.baseDir(), 'platformdata/backup/logs.txt'),
 
     OLD_DATA_DIR: path.join(config.baseDir(), 'data'),
     PLATFORM_DATA_DIR: path.join(config.baseDir(), 'platformdata'),
@@ -23,6 +22,7 @@ exports = module.exports = {
     NGINX_APPCONFIG_DIR: path.join(config.baseDir(), 'platformdata/nginx/applications'),
     NGINX_CERT_DIR: path.join(config.baseDir(), 'platformdata/nginx/cert'),
     BACKUP_INFO_DIR: path.join(config.baseDir(), 'platformdata/backup'),
+    UPDATE_DIR: path.join(config.baseDir(), 'platformdata/update'),
     SNAPSHOT_INFO_FILE: path.join(config.baseDir(), 'platformdata/backup/snapshot-info.json'),
 
     // this is not part of appdata because an icon may be set before install
@@ -33,5 +33,9 @@ exports = module.exports = {
     CLOUDRON_AVATAR_FILE: path.join(config.baseDir(), 'boxdata/avatar.png'),
     UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'boxdata/updatechecker.json'),
 
-    AUTO_PROVISION_FILE: path.join(config.baseDir(), 'configs/autoprovision.json')
+    AUTO_PROVISION_FILE: path.join(config.baseDir(), 'configs/autoprovision.json'),
+
+    LOG_DIR: path.join(config.baseDir(), 'platformdata/logs'),
+    // this pattern is for the cloudron logs API route to work
+    BACKUP_LOG_FILE: path.join(config.baseDir(), 'platformdata/logs/backup/app.log'),
 };

src/platform.js

@@ -13,7 +13,7 @@ var apps = require('./apps.js'),
     config = require('./config.js'),
     debug = require('debug')('box:platform'),
     fs = require('fs'),
-    hat = require('hat'),
+    hat = require('./hat.js'),
     infra = require('./infra_version.js'),
     locker = require('./locker.js'),
     mail = require('./mail.js'),
@@ -45,18 +45,15 @@ function start(callback) {
         if (!existingInfra) existingInfra = { version: 'corrupt' };
     }
 
+    settings.events.on(settings.PLATFORM_CONFIG_KEY, updateAddons);
+
     // short-circuit for the restart case
     if (_.isEqual(infra, existingInfra)) {
         debug('platform is uptodate at version %s', infra.version);
 
-        updateAddons(function (error) {
-            if (error) return callback(error);
+        emitPlatformReady();
 
-            emitPlatformReady();
-
-            callback();
-        });
-        return;
+        return callback();
     }
 
     debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version);
@@ -69,8 +66,7 @@ function start(callback) {
         startAddons.bind(null, existingInfra),
         removeOldImages,
         startApps.bind(null, existingInfra),
-        fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra, null, 4)),
-        updateAddons
+        fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra, null, 4))
     ], function (error) {
         if (error) return callback(error);
 
@@ -89,22 +85,19 @@ function stop(callback) {
     taskmanager.pauseTasks(callback);
 }
 
-function updateAddons(callback) {
-    settings.getPlatformConfig(function (error, platformConfig) {
-        if (error) return callback(error);
+function updateAddons(platformConfig, callback) {
+    callback = callback || NOOP_CALLBACK;
 
-        for (var containerName of [ 'mysql', 'postgresql', 'mail', 'mongodb' ]) {
-            const containerConfig = platformConfig[containerName];
-            if (!containerConfig) continue;
+    // TODO: this should possibly also rollback memory to default
+    async.eachSeries([ 'mysql', 'postgresql', 'mail', 'mongodb' ], function iterator(containerName, iteratorCallback) {
+        const containerConfig = platformConfig[containerName];
+        if (!containerConfig) return iteratorCallback();
 
-            if (!containerConfig.memory || !containerConfig.memorySwap) continue;
+        if (!containerConfig.memory || !containerConfig.memorySwap) return iteratorCallback();
 
-            const cmd = `docker update --memory ${containerConfig.memory} --memory-swap ${containerConfig.memorySwap} ${containerName}`;
-            shell.execSync(`update${containerName}`, cmd);
-        }
-
-        callback();
-    });
+        const args = `update --memory ${containerConfig.memory} --memory-swap ${containerConfig.memorySwap} ${containerName}`.split(' ');
+        shell.exec(`update${containerName}`, '/usr/bin/docker', args, { }, iteratorCallback);
+    }, callback);
 }
 
 function emitPlatformReady() {
@@ -162,6 +155,10 @@ function startGraphite(callback) {
     const cmd = `docker run --restart=always -d --name="graphite" \
                 --net cloudron \
                 --net-alias graphite \
+                --log-driver syslog \
+                --log-opt syslog-address=udp://127.0.0.1:2514 \
+                --log-opt syslog-format=rfc5424 \
+                --log-opt tag=graphite \
                 -m 75m \
                 --memory-swap 150m \
                 --dns 172.18.0.1 \
@@ -191,6 +188,10 @@ function startMysql(callback) {
     const cmd = `docker run --restart=always -d --name="mysql" \
                 --net cloudron \
                 --net-alias mysql \
+                --log-driver syslog \
+                --log-opt syslog-address=udp://127.0.0.1:2514 \
+                --log-opt syslog-format=rfc5424 \
+                --log-opt tag=mysql \
                 -m ${memoryLimit}m \
                 --memory-swap ${memoryLimit * 2}m \
                 --dns 172.18.0.1 \
@@ -217,6 +218,10 @@ function startPostgresql(callback) {
     const cmd = `docker run --restart=always -d --name="postgresql" \
                 --net cloudron \
                 --net-alias postgresql \
+                --log-driver syslog \
+                --log-opt syslog-address=udp://127.0.0.1:2514 \
+                --log-opt syslog-format=rfc5424 \
+                --log-opt tag=postgresql \
                 -m ${memoryLimit}m \
                 --memory-swap ${memoryLimit * 2}m \
                 --dns 172.18.0.1 \
@@ -243,6 +248,10 @@ function startMongodb(callback) {
     const cmd = `docker run --restart=always -d --name="mongodb" \
                 --net cloudron \
                 --net-alias mongodb \
+                --log-driver syslog \
+                --log-opt syslog-address=udp://127.0.0.1:2514 \
+                --log-opt syslog-format=rfc5424 \
+                --log-opt tag=mongodb \
                 -m ${memoryLimit}m \
                 --memory-swap ${memoryLimit * 2}m \
                 --dns 172.18.0.1 \
@@ -275,7 +284,15 @@ function startAddons(existingInfra, callback) {
         debug('startAddons: existing infra. incremental addon create %j', startFuncs.map(function (f) { return f.name; }));
     }
 
-    async.series(startFuncs, callback);
+    async.series(startFuncs, function (error) {
+        if (error) return callback(error);
+
+        settings.getPlatformConfig(function (error, platformConfig) {
+            if (error) return callback(error);
+
+            updateAddons(platformConfig, callback);
+        });
+    });
 }
 
 function startApps(existingInfra, callback) {

src/reverseproxy.js

@@ -44,6 +44,7 @@ var acme = require('./cert/acme.js'),
     path = require('path'),
     paths = require('./paths.js'),
     platform = require('./platform.js'),
+    rimraf = require('rimraf'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
     users = require('./users.js'),
@@ -348,15 +349,59 @@ function configureAppInternal(app, bundle, callback) {
     reload(callback);
 }
 
+function configureAppRedirect(app, fqdn, bundle, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof fqdn, 'string');
+    assert.strictEqual(typeof bundle, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = {
+        sourceDir: path.resolve(__dirname, '..'),
+        vhost: fqdn,
+        redirectTo: app.fqdn,
+        hasIPv6: config.hasIPv6(),
+        endpoint: 'redirect',
+        certFilePath: bundle.certFilePath,
+        keyFilePath: bundle.keyFilePath,
+        robotsTxtQuoted: null,
+        xFrameOptions: 'SAMEORIGIN'
+    };
+    var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
+
+    // if we change the filename, also change it in unconfigureApp()
+    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-redirect-${fqdn}.conf`);
+    debug('writing config for "%s" redirecting to "%s" to %s with options %j', app.fqdn, fqdn, nginxConfigFilename, data);
+
+    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
+        debug('Error creating nginx redirect config for "%s" : %s', app.fqdn, safe.error.message);
+        return callback(safe.error);
+    }
+
+    reload(callback);
+}
+
 function configureApp(app, auditSource, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    ensureCertificate(app, auditSource, function (error, bundle) {
+    ensureCertificate({ fqdn: app.fqdn, domain: app.domain }, auditSource, function (error, bundle) {
         if (error) return callback(error);
 
-        configureAppInternal(app, bundle, callback);
+        configureAppInternal(app, bundle, function (error) {
+            if (error) return callback(error);
+
+            // now setup alternateDomain redirects if any
+            async.eachSeries(app.alternateDomains, function (domain, callback) {
+                var fqdn = (domain.subdomain ? (domain.subdomain + '.') : '') + domain.domain;
+
+                ensureCertificate({ fqdn: fqdn, domain: domain.domain }, auditSource, function (error, bundle) {
+                    if (error) return callback(error);
+
+                    configureAppRedirect(app, fqdn, bundle, callback);
+                });
+            }, callback);
+        });
     });
 }
 
@@ -364,13 +409,12 @@ function unconfigureApp(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-    if (!safe.fs.unlinkSync(nginxConfigFilename)) {
-        if (safe.error.code !== 'ENOENT') debug('Error removing nginx configuration of "%s": %s', app.fqdn, safe.error.message);
-        return callback(null);
-    }
+    // we use globbing to find all nginx configs for an app
+    rimraf(path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}*.conf`), function (error) {
+        if (error) debug('Error removing nginx configurations of "%s":', app.fqdn, error);
 
-    reload(callback);
+        reload(callback);
+    });
 }
 
 function renewAll(auditSource, callback) {
@@ -386,6 +430,7 @@ function renewAll(auditSource, callback) {
 
         async.eachSeries(allApps, function (app, iteratorCallback) {
             ensureCertificate(app, auditSource, function (error, bundle) {
+                if (error) return iteratorCallback(error); // this can happen if cloudron is not setup yet
                 if (bundle.reason !== 'new-le' && bundle.reason !== 'fallback') return iteratorCallback();
 
                 // reconfigure for the case where we got a renewed cert after fallback

src/routes/accesscontrol.js

@@ -1,14 +1,101 @@
 'use strict';
 
 exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize,
+
     scope: scope,
-    websocketAuth: websocketAuth
+    websocketAuth: websocketAuth,
+    verifyAppOwnership: verifyAppOwnership
 };
 
 var accesscontrol = require('../accesscontrol.js'),
+    apps = require('../apps.js'),
+    AppsError = apps.AppsError,
     assert = require('assert'),
+    BasicStrategy = require('passport-http').BasicStrategy,
+    BearerStrategy = require('passport-http-bearer').Strategy,
+    clients = require('../clients.js'),
+    ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    ClientsError = clients.ClientsError,
     HttpError = require('connect-lastmile').HttpError,
-    passport = require('passport');
+    LocalStrategy = require('passport-local').Strategy,
+    passport = require('passport'),
+    settings = require('../settings.js'),
+    users = require('../users.js'),
+    UsersError = users.UsersError;
+
+function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // serialize user into session
+    passport.serializeUser(function (user, callback) {
+        callback(null, user.id);
+    });
+
+    // deserialize user from session
+    passport.deserializeUser(function(userId, callback) {
+        users.get(userId, function (error, result) {
+            if (error) return callback(error);
+
+            callback(null, result);
+        });
+    });
+
+    // used when username/password is sent in request body. used in CLI tool login route
+    passport.use(new LocalStrategy(function (username, password, callback) {
+        if (username.indexOf('@') === -1) {
+            users.verifyWithUsername(username, password, function (error, result) {
+                if (error && error.reason === UsersError.NOT_FOUND) return callback(null, false);
+                if (error && error.reason === UsersError.WRONG_PASSWORD) return callback(null, false);
+                if (error) return callback(error);
+                if (!result) return callback(null, false);
+                callback(null, result);
+            });
+        } else {
+            users.verifyWithEmail(username, password, function (error, result) {
+                if (error && error.reason === UsersError.NOT_FOUND) return callback(null, false);
+                if (error && error.reason === UsersError.WRONG_PASSWORD) return callback(null, false);
+                if (error) return callback(error);
+                if (!result) return callback(null, false);
+                callback(null, result);
+            });
+        }
+    }));
+
+    // Used to authenticate a OAuth2 client which uses clientId and clientSecret in the Authorization header
+    passport.use(new BasicStrategy(function (clientId, clientSecret, callback) {
+        clients.get(clientId, function (error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
+            if (error) return callback(error);
+            if (client.clientSecret !== clientSecret) return callback(null, false);
+            callback(null, client);
+        });
+    }));
+
+    // Used to authenticate a OAuth2 client which uses clientId and clientSecret in the request body (client_id, client_secret)
+    passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
+        clients.get(clientId, function(error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
+            if (error) { return callback(error); }
+            if (client.clientSecret !== clientSecret) { return callback(null, false); }
+            callback(null, client);
+        });
+    }));
+
+    // used for "Authorization: Bearer token" or access_token query param authentication
+    passport.use(new BearerStrategy(function (token, callback) {
+        accesscontrol.validateToken(token, callback);
+    }));
+
+    callback(null);
+}
+
+function uninitialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    callback(null);
+}
 
 //  The scope middleware provides an auth middleware for routes.
 //
@@ -19,16 +106,18 @@ var accesscontrol = require('../accesscontrol.js'),
 //  See server.js:
 //    var profileScope = routes.oauth2.scope('profile');
 //
-function scope(requestedScope) {
-    assert.strictEqual(typeof requestedScope, 'string');
+function scope(requiredScope) {
+    assert.strictEqual(typeof requiredScope, 'string');
 
-    var requestedScopes = requestedScope.split(',');
+    var requiredScopes = requiredScope.split(',');
 
     return [
         passport.authenticate(['bearer'], { session: false }),
 
         function (req, res, next) {
-            var error = accesscontrol.validateRequestedScopes(req.authInfo || null, requestedScopes);
+            assert(req.authInfo && typeof req.authInfo === 'object');
+
+            var error = accesscontrol.hasScopes(req.authInfo.authorizedScopes, requiredScopes);
             if (error) return next(new HttpError(403, error.message));
 
             next();
@@ -36,21 +125,42 @@ function scope(requestedScope) {
     ];
 }
 
-function websocketAuth(requestedScopes, req, res, next) {
-    assert(Array.isArray(requestedScopes));
+function websocketAuth(requiredScopes, req, res, next) {
+    assert(Array.isArray(requiredScopes));
 
     if (typeof req.query.access_token !== 'string') return next(new HttpError(401, 'Unauthorized'));
 
-    accesscontrol.accessTokenAuth(req.query.access_token, function (error, user, info) {
+    accesscontrol.validateToken(req.query.access_token, function (error, user, info) {
         if (error) return next(new HttpError(500, error.message));
         if (!user) return next(new HttpError(401, 'Unauthorized'));
 
         req.user = user;
-        req.authInfo = info;
 
-        var e = accesscontrol.validateRequestedScopes(req.authInfo, requestedScopes);
-        if (e) return next(new HttpError(401, e.message));
+        var e = accesscontrol.hasScopes(info.authorizedScopes, requiredScopes);
+        if (e) return next(new HttpError(403, e.message));
 
         next();
     });
 }
+
+function verifyAppOwnership(req, res, next) {
+    if (req.user.admin) return next();
+
+    const appCreate = !('id' in req.params);
+
+    settings.getSpacesConfig(function (error, spaces) {
+        if (error) return next(new HttpError(500, error));
+        if (!spaces.enabled) return next();
+
+        if (appCreate) return next(); // ok to install app
+
+        apps.get(req.params.id, function (error, app) {
+            if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
+            if (error) return next(new HttpError(500, error));
+
+            if (app.ownerId !== req.user.id) return next(new HttpError(401, 'Unauthorized'));
+
+            next();
+        });
+    });
+}

src/routes/apps.js

@@ -3,7 +3,6 @@
 exports = module.exports = {
     getApp: getApp,
     getApps: getApps,
-    getAllByUser: getAllByUser,
     getAppIcon: getAppIcon,
     installApp: installApp,
     configureApp: configureApp,
@@ -22,6 +21,8 @@ exports = module.exports = {
 
     cloneApp: cloneApp,
 
+    setOwner: setOwner,
+
     uploadFile: uploadFile,
     downloadFile: downloadFile
 };
@@ -50,7 +51,7 @@ function getApp(req, res, next) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(200, apps.removeInternalAppFields(app)));
+        next(new HttpSuccess(200, apps.removeInternalFields(app)));
     });
 }
 
@@ -60,19 +61,7 @@ function getApps(req, res, next) {
     apps.getAll(function (error, allApps) {
         if (error) return next(new HttpError(500, error));
 
-        allApps = allApps.map(apps.removeInternalAppFields);
-
-        next(new HttpSuccess(200, { apps: allApps }));
-    });
-}
-
-function getAllByUser(req, res, next) {
-    assert.strictEqual(typeof req.user, 'object');
-
-    apps.getAllByUser(req.user, function (error, allApps) {
-        if (error) return next(new HttpError(500, error));
-
-        allApps = allApps.map(apps.removeInternalAppFields);
+        allApps = allApps.map(apps.removeRestrictedFields);
 
         next(new HttpSuccess(200, { apps: allApps }));
     });
@@ -92,6 +81,7 @@ function installApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
     var data = req.body;
+    data.ownerId = req.user.id;
 
     // atleast one
     if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
@@ -172,6 +162,11 @@ function configureApp(req, res, next) {
 
     if ('mailboxName' in data && typeof data.mailboxName !== 'string') return next(new HttpError(400, 'mailboxName must be a string'));
 
+    if ('alternateDomains' in data) {
+        if (!Array.isArray(data.alternateDomains)) return next(new HttpError(400, 'alternateDomains must be an array'));
+        if (data.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
+    }
+
     debug('Configuring app id:%s data:%j', req.params.id, data);
 
     apps.configure(req.params.id, data, auditSource(req), function (error) {
@@ -215,6 +210,7 @@ function cloneApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
     var data = req.body;
+    data.ownerId = req.user.id;
 
     debug('Clone app id:%s', req.params.id);
 
@@ -565,3 +561,17 @@ function downloadFile(req, res, next) {
         stream.pipe(res);
     });
 }
+
+function setOwner(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.ownerId !== 'string') return next(new HttpError(400, 'ownerId must be a string'));
+
+    apps.setOwner(req.params.id, req.body.ownerId, function (error) {
+        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { }));
+    });
+}

src/routes/caas.js

@@ -1,6 +1,7 @@
 'use strict';
 
 exports = module.exports = {
+    getConfig: getConfig,
     changePlan: changePlan
 };
 
@@ -12,8 +13,25 @@ var caas = require('../caas.js'),
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     _ = require('underscore');
 
+function getConfig(req, res, next) {
+    if (config.provider() !== 'caas') return next(new HttpError(422, 'Cannot use this API with this provider'));
+
+    caas.getBoxAndUserDetails(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        // the result is { box: { region, size, plan }, user: { billing, currency } }
+        next(new HttpSuccess(200, {
+            region: result.box.region,
+            size: result.box.size,
+            billing: !!result.user.billing,
+            plan: result.box.plan,
+            currency: result.user.currency
+        }));
+    });
+}
+
 function changePlan(req, res, next) {
-    if (config.provider() !== 'caas') return next(new HttpError(422, 'Cannot use migrate API with this provider'));
+    if (config.provider() !== 'caas') return next(new HttpError(422, 'Cannot use this API with this provider'));
 
     if ('size' in req.body && typeof req.body.size !== 'string') return next(new HttpError(400, 'size must be string'));
     if ('region' in req.body && typeof req.body.region !== 'string') return next(new HttpError(400, 'region must be string'));

src/routes/cloudron.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     getProgress: getProgress,
     getConfig: getConfig,
     getDisks: getDisks,
+    getUpdateInfo: getUpdateInfo,
     update: update,
     feedback: feedback,
     checkForUpdates: checkForUpdates,
@@ -21,7 +22,9 @@ var appstore = require('../appstore.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     progress = require('../progress.js'),
+    updater = require('../updater.js'),
     updateChecker = require('../updatechecker.js'),
+    UpdaterError = require('../updater.js').UpdaterError,
     _ = require('underscore');
 
 function auditSource(req) {
@@ -57,16 +60,20 @@ function getDisks(req, res, next) {
 
 function update(req, res, next) {
     // this only initiates the update, progress can be checked via the progress route
-    cloudron.updateToLatest(auditSource(req), function (error) {
-        if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
-        if (error && error.reason === CloudronError.SELF_UPGRADE_NOT_SUPPORTED) return next(new HttpError(412, error.message));
+    updater.updateToLatest(auditSource(req), function (error) {
+        if (error && error.reason === UpdaterError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
+        if (error && error.reason === UpdaterError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === UpdaterError.SELF_UPGRADE_NOT_SUPPORTED) return next(new HttpError(412, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
     });
 }
 
+function getUpdateInfo(req, res, next) {
+    next(new HttpSuccess(200, { update: updateChecker.getUpdateInfo() }));
+}
+
 function checkForUpdates(req, res, next) {
     async.series([
         updateChecker.checkAppUpdates,
@@ -97,19 +104,18 @@ function feedback(req, res, next) {
 }
 
 function getLogs(req, res, next) {
+    assert.strictEqual(typeof req.params.unit, 'string');
+
     var lines = req.query.lines ? parseInt(req.query.lines, 10) : 100;
     if (isNaN(lines)) return next(new HttpError(400, 'lines must be a number'));
 
-    var units = req.query.units || 'all';
-
     var options = {
         lines: lines,
         follow: false,
-        units: units.split(','),
         format: req.query.format
     };
 
-    cloudron.getLogs(options, function (error, logStream) {
+    cloudron.getLogs(req.params.unit, options, function (error, logStream) {
         if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(404, 'Invalid type'));
         if (error) return next(new HttpError(500, error));
 
@@ -124,11 +130,11 @@ function getLogs(req, res, next) {
 }
 
 function getLogStream(req, res, next) {
+    assert.strictEqual(typeof req.params.unit, 'string');
+
     var lines = req.query.lines ? parseInt(req.query.lines, 10) : -10; // we ignore last-event-id
     if (isNaN(lines)) return next(new HttpError(400, 'lines must be a valid number'));
 
-    var units = req.query.units || 'all';
-
     function sse(id, data) { return 'id: ' + id + '\ndata: ' + data + '\n\n'; }
 
     if (req.headers.accept !== 'text/event-stream') return next(new HttpError(400, 'This API call requires EventStream'));
@@ -136,11 +142,10 @@ function getLogStream(req, res, next) {
     var options = {
         lines: lines,
         follow: true,
-        units: units.split(','),
         format: req.query.format
     };
 
-    cloudron.getLogs(options, function (error, logStream) {
+    cloudron.getLogs(req.params.unit, options, function (error, logStream) {
         if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(404, 'Invalid type'));
         if (error) return next(new HttpError(500, error));
 

src/routes/domains.js

@@ -55,7 +55,7 @@ function getAll(req, res, next) {
     domains.getAll(function (error, result) {
         if (error) return next(new HttpError(500, error));
 
-        result = result.map(domains.removePrivateFields);
+        result = result.map(domains.removeRestrictedFields);
 
         next(new HttpSuccess(200, { domains: result }));
     });
@@ -98,4 +98,3 @@ function del(req, res, next) {
         next(new HttpSuccess(204));
     });
 }
-

src/routes/groups.js

@@ -4,6 +4,7 @@ exports = module.exports = {
     get: get,
     list: list,
     create: create,
+    update: update,
     remove: remove,
     updateMembers: updateMembers
 };
@@ -12,7 +13,6 @@ var assert = require('assert'),
     groups = require('../groups.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
-    groups = require('../groups.js'),
     GroupsError = groups.GroupsError;
 
 function create(req, res, next) {
@@ -45,6 +45,20 @@ function get(req, res, next) {
     });
 }
 
+function update(req, res, next) {
+    assert.strictEqual(typeof req.params.groupId, 'string');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if ('name' in req.body && typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be a string'));
+
+    groups.update(req.params.groupId, req.body, function (error) {
+        if (error && error.reason === GroupsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { }));
+    });
+}
+
 function updateMembers(req, res, next) {
     assert.strictEqual(typeof req.params.groupId, 'string');
 
@@ -55,12 +69,12 @@ function updateMembers(req, res, next) {
         if (error && error.reason === GroupsError.NOT_FOUND) return next(new HttpError(404, 'Invalid group or user id'));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(200));
+        next(new HttpSuccess(200, { }));
     });
 }
 
 function list(req, res, next) {
-    groups.getAllWithMembers(function (error, result) {
+    groups.getAll(function (error, result) {
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(200, { groups: result }));

src/routes/index.js

@@ -19,6 +19,5 @@ exports = module.exports = {
     sysadmin: require('./sysadmin.js'),
     settings: require('./settings.js'),
     ssh: require('./ssh.js'),
-    user: require('./user.js'),
     users: require('./users.js')
 };

src/routes/mail.js

@@ -4,9 +4,10 @@ exports = module.exports = {
     getDomain: getDomain,
     addDomain: addDomain,
     getDomainStats: getDomainStats,
-    updateDomain: updateDomain,
     removeDomain: removeDomain,
 
+    setDnsRecords: setDnsRecords,
+
     getStatus: getStatus,
 
     setMailFromValidation: setMailFromValidation,
@@ -81,15 +82,15 @@ function getDomainStats(req, res, next) {
     mailProxy(req, res, next);
 }
 
-function updateDomain(req, res, next) {
+function setDnsRecords(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.domain, 'string');
 
-    mail.updateDomain(req.params.domain, function (error) {
+    mail.setDnsRecords(req.params.domain, function (error) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(202));
+        next(new HttpSuccess(201));
     });
 }
 

src/routes/oauth2.js

@@ -19,8 +19,7 @@ exports = module.exports = {
     csrf: csrf
 };
 
-var accesscontrol = require('../accesscontrol.js'),
-    apps = require('../apps.js'),
+var apps = require('../apps.js'),
     assert = require('assert'),
     authcodedb = require('../authcodedb.js'),
     clients = require('../clients'),
@@ -30,7 +29,7 @@ var accesscontrol = require('../accesscontrol.js'),
     DatabaseError = require('../databaseerror.js'),
     debug = require('debug')('box:routes/oauth2'),
     eventlog = require('../eventlog.js'),
-    hat = require('hat'),
+    hat = require('../hat.js'),
     HttpError = require('connect-lastmile').HttpError,
     middleware = require('../middleware/index.js'),
     oauth2orize = require('oauth2orize'),
@@ -39,7 +38,6 @@ var accesscontrol = require('../accesscontrol.js'),
     session = require('connect-ensure-login'),
     settings = require('../settings.js'),
     speakeasy = require('speakeasy'),
-    tokendb = require('../tokendb.js'),
     url = require('url'),
     users = require('../users.js'),
     UsersError = users.UsersError,
@@ -52,19 +50,13 @@ function auditSource(req, appId, appObject) {
     return { authType: 'oauth', ip: ip, appId: appId, app: appObject };
 }
 
-// create OAuth 2.0 server
 var gServer = null;
 
 function initialize() {
-    assert(gServer === null);
+    assert.strictEqual(gServer, null);
 
     gServer = oauth2orize.createServer();
 
-    // Register serialialization and deserialization functions.
-    //
-    // The client id is stored in the session and can thus be retrieved for each
-    // step in the oauth flow transaction, which involves multiple http requests.
-
     gServer.serializeClient(function (client, callback) {
         return callback(null, client.id);
     });
@@ -73,16 +65,7 @@ function initialize() {
         clients.get(id, callback);
     });
 
-
-    // Register supported grant types.
-
-    // Grant authorization codes.  The callback takes the `client` requesting
-    // authorization, the `redirectURI` (which is used as a verifier in the
-    // subsequent exchange), the authenticated `user` granting access, and
-    // their response, which contains approved scope, duration, etc. as parsed by
-    // the application.  The application issues a code, which is bound to these
-    // values, and will be exchanged for an access token.
-
+    // grant authorization code that can be exchanged for access tokens. this is used by external oauth clients
     gServer.grant(oauth2orize.grant.code({ scopeSeparator: ',' }, function (client, redirectURI, user, ares, callback) {
         debug('grant code:', client.id, redirectURI, user.id, ares);
 
@@ -98,33 +81,8 @@ function initialize() {
         });
     }));
 
-
-    gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client, user, ares, callback) {
-        debug('grant token:', client.id, user.id, ares);
-
-        var token = tokendb.generateToken();
-        var expires = Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
-        var scope = accesscontrol.normalizeScope(user.scope, client.scope);
-
-        tokendb.add(token, user.id, client.id, expires, scope, function (error) {
-            if (error) return callback(error);
-
-            debug('grant token: new access token for client %s token %s', client.id, token);
-
-            callback(null, token);
-        });
-    }));
-
-
-    // Exchange authorization codes for access tokens.  The callback accepts the
-    // `client`, which is exchanging `code` and any `redirectURI` from the
-    // authorization request for verification.  If these values are validated, the
-    // application issues an access token on behalf of the user who authorized the
-    // code.
-
+    // exchange authorization codes for access tokens. this is used by external oauth clients
     gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI, callback) {
-        debug('exchange:', client, code, redirectURI);
-
         authcodedb.get(code, function (error, authCode) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
             if (error) return callback(error);
@@ -133,21 +91,28 @@ function initialize() {
             authcodedb.del(code, function (error) {
                 if(error) return callback(error);
 
-                var token = tokendb.generateToken();
-                var expires = Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
-                var scope = accesscontrol.canonicalScope(client.scope);
-
-                tokendb.add(token, authCode.userId, authCode.clientId, expires, client.scope, function (error) {
+                clients.addTokenByUserId(client.id, authCode.userId, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, function (error, result) {
                     if (error) return callback(error);
 
-                    debug('exchange: new access token for client %s token %s', client.id, token);
+                    debug('exchange: new access token for client %s user %s token %s', client.id, authCode.userId, result.accessToken.slice(0, 6)); // partial token for security
 
-                    callback(null, token);
+                    callback(null, result.accessToken);
                 });
             });
         });
     }));
 
+    // implicit token grant that skips issuing auth codes. this is used by our webadmin
+    gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client, user, ares, callback) {
+        clients.addTokenByUserId(client.id, user.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, function (error, result) {
+            if (error) return callback(error);
+
+            debug('grant token: new access token for client %s user %s token %s', client.id, user.id, result.accessToken.slice(0, 6)); // partial token for security
+
+            callback(null, result.accessToken);
+        });
+    }));
+
     // overwrite the session.ensureLoggedIn to not use res.redirect() due to a chrome bug not sending cookies on redirects
     session.ensureLoggedIn = function (redirectTo) {
         assert.strictEqual(typeof redirectTo, 'string');
@@ -327,7 +292,7 @@ function passwordResetRequestSite(req, res) {
 function passwordResetRequest(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (typeof req.body.identifier !== 'string') return next(new HttpError(400, 'Missing identifier'));
+    if (typeof req.body.identifier !== 'string') return next(new HttpError(400, 'Missing identifier')); // email or username
 
     debug('passwordResetRequest: email or username %s.', req.body.identifier);
 
@@ -352,16 +317,18 @@ function renderAccountSetupSite(res, req, userObject, error) {
         error: error,
         csrf: req.csrfToken(),
         resetToken: req.query.reset_token || req.body.resetToken,
-        title: 'Password Setup'
+        email: req.query.email || req.body.email,
+        title: 'Account Setup'
     });
 }
 
 // -> GET /api/v1/session/account/setup.html
 function accountSetupSite(req, res) {
     if (!req.query.reset_token) return sendError(req, res, 'Missing Reset Token');
+    if (!req.query.email) return sendError(req, res, 'Missing Email');
 
-    users.getByResetToken(req.query.reset_token, function (error, userObject) {
-        if (error) return sendError(req, res, 'Invalid Reset Token');
+    users.getByResetToken(req.query.email, req.query.reset_token, function (error, userObject) {
+        if (error) return sendError(req, res, 'Invalid Email or Reset Token');
 
         renderAccountSetupSite(res, req, userObject, '');
     });
@@ -371,14 +338,15 @@ function accountSetupSite(req, res) {
 function accountSetup(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
+    if (typeof req.body.email !== 'string') return next(new HttpError(400, 'Missing email'));
     if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
     if (typeof req.body.username !== 'string') return next(new HttpError(400, 'Missing username'));
     if (typeof req.body.displayName !== 'string') return next(new HttpError(400, 'Missing displayName'));
 
-    debug('acountSetup: with token %s.', req.body.resetToken);
+    debug(`acountSetup: for email ${req.body.email} with token ${req.body.resetToken}`);
 
-    users.getByResetToken(req.body.resetToken, function (error, userObject) {
+    users.getByResetToken(req.body.email, req.body.resetToken, function (error, userObject) {
         if (error) return sendError(req, res, 'Invalid Reset Token');
 
         var data = _.pick(req.body, 'username', 'displayName');
@@ -392,7 +360,7 @@ function accountSetup(req, res, next) {
             userObject.displayName = req.body.displayName;
 
             // setPassword clears the resetToken
-            users.setPassword(userObject.id, req.body.password, function (error, result) {
+            users.setPassword(userObject.id, req.body.password, function (error) {
                 if (error && error.reason === UsersError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
 
                 if (error) return next(new HttpError(500, error));
@@ -405,15 +373,17 @@ function accountSetup(req, res, next) {
 
 // -> GET /api/v1/session/password/reset.html
 function passwordResetSite(req, res, next) {
+    if (!req.query.email) return next(new HttpError(400, 'Missing email'));
     if (!req.query.reset_token) return next(new HttpError(400, 'Missing reset_token'));
 
-    users.getByResetToken(req.query.reset_token, function (error, user) {
-        if (error) return next(new HttpError(401, 'Invalid reset_token'));
+    users.getByResetToken(req.query.email, req.query.reset_token, function (error, user) {
+        if (error) return next(new HttpError(401, 'Invalid email or reset token'));
 
         renderTemplate(res, 'password_reset', {
             user: user,
             csrf: req.csrfToken(),
             resetToken: req.query.reset_token,
+            email: req.query.email,
             title: 'Password Reset'
         });
     });
@@ -423,13 +393,14 @@ function passwordResetSite(req, res, next) {
 function passwordReset(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
+    if (typeof req.body.email !== 'string') return next(new HttpError(400, 'Missing email'));
     if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
 
-    debug('passwordReset: with token %s.', req.body.resetToken);
+    debug(`passwordReset: for ${req.body.email} with token ${req.body.resetToken}`);
 
-    users.getByResetToken(req.body.resetToken, function (error, userObject) {
-        if (error) return next(new HttpError(401, 'Invalid resetToken'));
+    users.getByResetToken(req.body.email, req.body.resetToken, function (error, userObject) {
+        if (error) return next(new HttpError(401, 'Invalid email or resetToken'));
 
         if (!userObject.username) return next(new HttpError(401, 'No username set'));
 
@@ -526,7 +497,7 @@ function authorization() {
 function token() {
     return [
         passport.authenticate(['basic', 'oauth2-client-password'], { session: false }),
-        gServer.token(),
+        gServer.token(), // will call the token grant callback registered in initialize()
         gServer.errorHandler()
     ];
 }

src/routes/profile.js

@@ -9,8 +9,7 @@ exports = module.exports = {
     disableTwoFactorAuthentication: disableTwoFactorAuthentication
 };
 
-var accesscontrol = require('../accesscontrol.js'),
-    assert = require('assert'),
+var assert = require('assert'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     users = require('../users.js'),
@@ -30,10 +29,9 @@ function get(req, res, next) {
         username: req.user.username,
         email: req.user.email,
         fallbackEmail: req.user.fallbackEmail,
-        admin: req.user.admin,
-        scope: accesscontrol.canonicalScope(req.authInfo.scope), // this returns the token scope and not the user's scope
         displayName: req.user.displayName,
-        twoFactorAuthenticationEnabled: req.user.twoFactorAuthenticationEnabled
+        twoFactorAuthenticationEnabled: req.user.twoFactorAuthenticationEnabled,
+        admin: req.user.admin
     }));
 }
 
@@ -65,7 +63,7 @@ function changePassword(req, res, next) {
 
     users.setPassword(req.user.id, req.body.newPassword, function (error) {
         if (error && error.reason === UsersError.BAD_FIELD) return next(new HttpError(400, error.message));
-        if (error && error.reason === UsersError.NOT_FOUND) return next(new HttpError(403, 'Wrong password'));
+        if (error && error.reason === UsersError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(204));
@@ -91,7 +89,7 @@ function enableTwoFactorAuthentication(req, res, next) {
 
     users.enableTwoFactorAuthentication(req.user.id, req.body.totpToken, function (error) {
         if (error && error.reason === UsersError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
-        if (error && error.reason === UsersError.BAD_TOKEN) return next(new HttpError(403, 'Invalid token'));
+        if (error && error.reason === UsersError.BAD_TOKEN) return next(new HttpError(401, 'Invalid token'));
         if (error && error.reason === UsersError.ALREADY_EXISTS) return next(new HttpError(409, 'TwoFactor Authentication is already enabled'));
         if (error) return next(new HttpError(500, error));
 

src/routes/settings.js

@@ -20,7 +20,13 @@ exports = module.exports = {
     setTimeZone: setTimeZone,
 
     getAppstoreConfig: getAppstoreConfig,
-    setAppstoreConfig: setAppstoreConfig
+    setAppstoreConfig: setAppstoreConfig,
+
+    getPlatformConfig: getPlatformConfig,
+    setPlatformConfig: setPlatformConfig,
+
+    setSpacesConfig: setSpacesConfig,
+    getSpacesConfig: getSpacesConfig
 };
 
 var assert = require('assert'),
@@ -47,7 +53,7 @@ function setAppAutoupdatePattern(req, res, next) {
         if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(200));
+        next(new HttpSuccess(200, {}));
     });
 }
 
@@ -68,7 +74,7 @@ function setBoxAutoupdatePattern(req, res, next) {
         if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(200));
+        next(new HttpSuccess(200, {}));
     });
 }
 
@@ -81,7 +87,7 @@ function setCloudronName(req, res, next) {
         if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(202));
+        next(new HttpSuccess(202, {}));
     });
 }
 
@@ -110,7 +116,7 @@ function setTimeZone(req, res, next) {
         if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(200));
+        next(new HttpSuccess(200, {}));
     });
 }
 
@@ -123,7 +129,7 @@ function setCloudronAvatar(req, res, next) {
     settings.setCloudronAvatar(avatar, function (error) {
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(202));
+        next(new HttpSuccess(202, {}));
     });
 }
 
@@ -160,12 +166,63 @@ function setBackupConfig(req, res, next) {
     if (typeof req.body.format !== 'string') return next(new HttpError(400, 'format must be a string'));
     if ('acceptSelfSignedCerts' in req.body && typeof req.body.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
 
+    // testing the backup using put/del takes a bit of time at times
+    req.clearTimeout();
+
     settings.setBackupConfig(req.body, function (error) {
         if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === SettingsError.EXTERNAL_ERROR) return next(new HttpError(402, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(200));
+        next(new HttpSuccess(200, {}));
+    });
+}
+
+function getPlatformConfig(req, res, next) {
+    settings.getPlatformConfig(function (error, config) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, config));
+    });
+}
+
+function setPlatformConfig(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    for (let addon of [ 'mysql', 'postgresql', 'mail', 'mongodb' ]) {
+        if (!(addon in req.body)) continue;
+        if (typeof req.body[addon] !== 'object') return next(new HttpError(400, 'addon config must be an object'));
+
+        if (typeof req.body[addon].memory !== 'number') return next(new HttpError(400, 'memory must be a number'));
+        if (typeof req.body[addon].memorySwap !== 'number') return next(new HttpError(400, 'memorySwap must be a number'));
+    }
+
+    settings.setPlatformConfig(req.body, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === SettingsError.EXTERNAL_ERROR) return next(new HttpError(402, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, {}));
+    });
+}
+
+function getSpacesConfig(req, res, next) {
+    settings.getSpacesConfig(function (error, config) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, config));
+    });
+}
+
+function setSpacesConfig(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    settings.setSpacesConfig(req.body, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === SettingsError.EXTERNAL_ERROR) return next(new HttpError(402, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, {}));
     });
 }
 

src/routes/setup.js

@@ -35,7 +35,7 @@ function providerTokenAuth(req, res, next) {
             if (error && !error.response) return next(new HttpError(500, error));
             if (result.statusCode !== 200) return next(new HttpError(500, 'Unable to get meta data'));
 
-            if (result.text !== req.body.providerToken) return next(new HttpError(403, 'Invalid providerToken'));
+            if (result.text !== req.body.providerToken) return next(new HttpError(401, 'Invalid providerToken'));
 
             next();
         });
@@ -53,7 +53,7 @@ function setupTokenAuth(req, res, next) {
 
     caas.verifySetupToken(req.query.setupToken, function (error) {
         if (error && error.reason === CaasError.BAD_STATE) return next(new HttpError(409, 'Already setup'));
-        if (error && error.reason === CaasError.INVALID_TOKEN) return next(new HttpError(403, 'Invalid token'));
+        if (error && error.reason === CaasError.INVALID_TOKEN) return next(new HttpError(401, 'Invalid token'));
         if (error && error.reason === CaasError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
 
         if (error) return next(new HttpError(500, error));
@@ -78,6 +78,7 @@ function dnsSetup(req, res, next) {
     setup.dnsSetup(req.body.adminFqdn.toLowerCase(), req.body.domain.toLowerCase(), req.body.zoneName || '', req.body.provider, req.body.config, req.body.tlsConfig || { provider: 'letsencrypt-prod' }, function (error) {
         if (error && error.reason === SetupError.ALREADY_SETUP) return next(new HttpError(409, error.message));
         if (error && error.reason === SetupError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === SetupError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(200));
@@ -118,7 +119,7 @@ function activate(req, res, next) {
 
         caas.setupDone(req.query.setupToken, function (error) {
             if (error && error.reason === CaasError.BAD_STATE) return next(new HttpError(409, 'Already setup'));
-            if (error && error.reason === CaasError.INVALID_TOKEN) return next(new HttpError(403, 'Invalid token'));
+            if (error && error.reason === CaasError.INVALID_TOKEN) return next(new HttpError(401, 'Invalid token'));
             if (error && error.reason === CaasError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
 
             if (error) return next(new HttpError(500, error));

src/routes/sysadmin.js

@@ -9,10 +9,11 @@ exports = module.exports = {
 var backups = require('../backups.js'),
     BackupsError = require('../backups.js').BackupsError,
     cloudron = require('../cloudron.js'),
-    CloudronError = require('../cloudron.js').CloudronError,
     debug = require('debug')('box:routes/sysadmin'),
     HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess;
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    updater = require('../updater.js'),
+    UpdaterError = require('../updater.js').UpdaterError;
 
 function backup(req, res, next) {
     debug('triggering backup');
@@ -33,10 +34,10 @@ function update(req, res, next) {
 
     // this only initiates the update, progress can be checked via the progress route
     var auditSource = { userId: null, username: 'sysadmin' };
-    cloudron.updateToLatest(auditSource, function (error) {
-        if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
-        if (error && error.reason === CloudronError.SELF_UPGRADE_NOT_SUPPORTED) return next(new HttpError(412, error.message));
+    updater.updateToLatest(auditSource, function (error) {
+        if (error && error.reason === UpdaterError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
+        if (error && error.reason === UpdaterError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === UpdaterError.SELF_UPGRADE_NOT_SUPPORTED) return next(new HttpError(412, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));

src/routes/test/accesscontrol-test.js

@@ -25,29 +25,9 @@ describe('scopes middleware', function () {
         passport.authenticate = passportAuthenticateSave;
     });
 
-    it('fails due to missing authInfo', function (done) {
+    it('fails due to empty scope in request', function (done) {
         var mw = accesscontrol.scope('admin')[1];
-        var req = {};
-
-        mw(req, null, function (error) {
-            expect(error).to.be.a(HttpError);
-            done();
-        });
-    });
-
-    it('fails due to missing scope property in authInfo', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: {} };
-
-        mw(req, null, function (error) {
-            expect(error).to.be.a(HttpError);
-            done();
-        });
-    });
-
-    it('fails due to missing scope in request', function (done) {
-        var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { scope: '' } };
+        var req = { authInfo: { authorizedScopes: [ ] } };
 
         mw(req, null, function (error) {
             expect(error).to.be.a(HttpError);
@@ -57,7 +37,7 @@ describe('scopes middleware', function () {
 
     it('fails due to wrong scope in request', function (done) {
         var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { scope: 'foobar,something' } };
+        var req = { authInfo: { authorizedScopes: [ 'foobar', 'something' ] } };
 
         mw(req, null, function (error) {
             expect(error).to.be.a(HttpError);
@@ -67,7 +47,7 @@ describe('scopes middleware', function () {
 
     it('fails due to wrong scope in request', function (done) {
         var mw = accesscontrol.scope('admin,users')[1];
-        var req = { authInfo: { scope: 'foobar,admin' } };
+        var req = { authInfo: { authorizedScopes: [ 'foobar', 'admin' ] } };
 
         mw(req, null, function (error) {
             expect(error).to.be.a(HttpError);
@@ -77,7 +57,7 @@ describe('scopes middleware', function () {
 
     it('succeeds with one requested scope and one provided scope', function (done) {
         var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { scope: 'admin' } };
+        var req = { authInfo: { authorizedScopes: [ 'admin' ] } };
 
         mw(req, null, function (error) {
             expect(error).to.not.be.ok();
@@ -87,7 +67,7 @@ describe('scopes middleware', function () {
 
     it('succeeds with one requested scope and two provided scopes', function (done) {
         var mw = accesscontrol.scope('admin')[1];
-        var req = { authInfo: { scope: 'foobar,admin' } };
+        var req = { authInfo: { authorizedScopes: [ 'foobar', 'admin' ] } };
 
         mw(req, null, function (error) {
             expect(error).to.not.be.ok();
@@ -97,7 +77,7 @@ describe('scopes middleware', function () {
 
     it('succeeds with two requested scope and two provided scopes', function (done) {
         var mw = accesscontrol.scope('admin,foobar')[1];
-        var req = { authInfo: { scope: 'foobar,admin' } };
+        var req = { authInfo: { authorizedScopes: [ 'foobar', 'admin' ] } };
 
         mw(req, null, function (error) {
             expect(error).to.not.be.ok();
@@ -107,7 +87,7 @@ describe('scopes middleware', function () {
 
     it('succeeds with two requested scope and provided wildcard scope', function (done) {
         var mw = accesscontrol.scope('admin,foobar')[1];
-        var req = { authInfo: { scope: '*' } };
+        var req = { authInfo: { authorizedScopes: [ '*' ] } };
 
         mw(req, null, function (error) {
             expect(error).to.not.be.ok();

src/routes/test/backups-test.js

@@ -29,7 +29,7 @@ const DOMAIN_0 = {
     tlsConfig: { provider: 'fallback' }
 };
 
-var token = null;
+var token = null, ownerId = null;
 
 function setup(done) {
     nock.cleanAll();
@@ -49,6 +49,7 @@ function setup(done) {
                     expect(result.statusCode).to.eql(201);
 
                     // stash token for further use
+                    ownerId = result.body.userId;
                     token = result.body.token;
 
                     callback();
@@ -57,7 +58,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', DOMAIN_0.domain, [ ] /* portBindings */, { }, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', DOMAIN_0.domain, ownerId, [ ] /* portBindings */, { }, callback);
         },
 
         function createSettings(callback) {

src/routes/test/caas-test.js

@@ -35,7 +35,7 @@ const DOMAIN_0 = {
     tlsConfig: { provider: 'fallback' }
 };
 
-var token = null;
+var token = null, ownerId = null;
 var gSudoOriginal = null;
 function injectShellMock() {
     gSudoOriginal = shell.sudo;
@@ -74,6 +74,7 @@ function setup(done) {
                     expect(scope2.isDone()).to.be.ok();
 
                     // stash token for further use
+                    ownerId = result.body.userId;
                     token = result.body.token;
 
                     callback();
@@ -82,7 +83,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', DOMAIN_0.domain, [ ] /* portBindings */, { }, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', DOMAIN_0.domain, ownerId, [ ] /* portBindings */, { }, callback);
         },
 
         function createSettings(callback) {
@@ -153,21 +154,12 @@ describe('Caas', function () {
                 .get('/api/v1/boxes/BOX_ID?token=ACCESS_TOKEN2')
                 .reply(200, { box: { region: 'sfo', size: '1gb' }, user: { }});
 
-            superagent.get(SERVER_URL + '/api/v1/cloudron/config')
+            superagent.get(SERVER_URL + '/api/v1/caas/config')
                 .query({ access_token: token })
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(200);
-                    expect(result.body.apiServerOrigin).to.eql('http://localhost:6060');
-                    expect(result.body.webServerOrigin).to.eql(null);
-                    expect(result.body.adminFqdn).to.eql(config.adminFqdn());
-                    expect(result.body.progress).to.be.an('object');
-                    expect(result.body.update).to.be.an('object');
-                    expect(result.body.version).to.eql(config.version());
                     expect(result.body.size).to.eql('1gb');
                     expect(result.body.region).to.eql('sfo');
-                    expect(result.body.memory).to.eql(os.totalmem());
-                    expect(result.body.cloudronName).to.be.a('string');
-                    expect(result.body.provider).to.be.a('string');
 
                     expect(scope.isDone()).to.be.ok();
 

src/routes/test/clients-test.js

@@ -13,7 +13,7 @@ var accesscontrol = require('../../accesscontrol.js'),
     oauth2 = require('../oauth2.js'),
     expect = require('expect.js'),
     uuid = require('uuid'),
-    hat = require('hat'),
+    hat = require('../../hat.js'),
     superagent = require('superagent'),
     server = require('../../server.js');
 
@@ -483,6 +483,8 @@ describe('Clients', function () {
     });
 
     describe('delete tokens by client', function () {
+        this.timeout(5000);
+
         before(setup2);
         after(cleanup);
 

src/routes/test/cloudron-test.js

@@ -167,7 +167,7 @@ describe('Cloudron', function () {
                             userId_1 = result.body.id;
 
                             // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                            tokendb.add(token_1, userId_1, 'test-client-id',  Date.now() + 100000, accesscontrol.SCOPE_ANY, callback);
+                            tokendb.add(token_1, userId_1, 'test-client-id',  Date.now() + 100000, 'cloudron', callback);
                         });
                 }
             ], done);
@@ -176,7 +176,7 @@ describe('Cloudron', function () {
         after(cleanup);
 
         it('cannot get without token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/cloudron/config')
+            superagent.get(SERVER_URL + '/api/v1/config')
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(401);
                     done();
@@ -184,7 +184,7 @@ describe('Cloudron', function () {
         });
 
         it('succeeds (admin)', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/cloudron/config')
+            superagent.get(SERVER_URL + '/api/v1/config')
                 .query({ access_token: token })
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(200);
@@ -192,10 +192,7 @@ describe('Cloudron', function () {
                     expect(result.body.webServerOrigin).to.eql(null);
                     expect(result.body.adminFqdn).to.eql(config.adminFqdn());
                     expect(result.body.progress).to.be.an('object');
-                    expect(result.body.update).to.be.an('object');
                     expect(result.body.version).to.eql(config.version());
-                    expect(result.body.size).to.eql(null);
-                    expect(result.body.region).to.eql(null);
                     expect(result.body.memory).to.eql(os.totalmem());
                     expect(result.body.cloudronName).to.be.a('string');
 
@@ -204,7 +201,7 @@ describe('Cloudron', function () {
         });
 
         it('fails (non-admin)', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/cloudron/config')
+            superagent.get(SERVER_URL + '/api/v1/config')
                 .query({ access_token: token_1 })
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(403);
@@ -372,7 +369,7 @@ describe('Cloudron', function () {
         after(cleanup);
 
         it('logStream - requires event-stream accept header', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/cloudron/logstream')
+            superagent.get(SERVER_URL + '/api/v1/cloudron/logstream/box')
                 .query({ access_token: token, fromLine: 0 })
                 .end(function (err, res) {
                     expect(res.statusCode).to.be(400);
@@ -382,7 +379,9 @@ describe('Cloudron', function () {
 
         it('logStream - stream logs', function (done) {
             var options = {
-                port: config.get('port'), host: 'localhost', path: '/api/v1/cloudron/logstream?units=all&lines=10&access_token=' + token,
+                host: 'localhost',
+                port: config.get('port'),
+                path: '/api/v1/cloudron/logstream/mail?units=all&lines=10&access_token=' + token,
                 headers: { 'Accept': 'text/event-stream', 'Connection': 'keep-alive' }
             };
 

src/routes/test/groups-test.js

@@ -24,7 +24,7 @@ var token, token_1 = null;
 var userId, userId_1 = null;
 
 var GROUP_NAME = 'externals';
-var groupObject;
+var groupObject, group1Object;
 
 function setup(done) {
     config._reset();
@@ -88,220 +88,183 @@ describe('Groups API', function () {
     before(setup);
     after(cleanup);
 
-    describe('list', function () {
-        it('cannot get groups without token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/groups')
-                .end(function (err, res) {
-                    expect(res.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('cannot get groups as normal user', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/groups')
-                .query({ access_token: token_1 })
-                .end(function (err, res) {
-                    expect(res.statusCode).to.equal(403);
-                    done();
-                });
-        });
-
-        it('can get groups', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/groups')
-                .query({ access_token: token })
-                .end(function (err, res) {
-                    expect(res.statusCode).to.equal(200);
-                    expect(res.body.groups).to.be.an(Array);
-                    expect(res.body.groups.length).to.be(1);
-                    expect(res.body.groups[0].name).to.eql('admin');
-                    expect(res.body.groups[0].userIds).to.be.an(Array);
-                    expect(res.body.groups[0].userIds.length).to.be(1);
-                    expect(res.body.groups[0].userIds[0]).to.be(userId);
-                    done();
-                });
-        });
-    });
-
-    describe('create', function () {
-        it('fails due to mising token', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/groups')
-                .send({ name: GROUP_NAME})
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/groups')
-                .query({ access_token: token })
-                .send({ name: GROUP_NAME})
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(201);
-                    groupObject = result.body;
-                    done();
-                });
-        });
-
-        it('fails for already exists', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/groups')
-                .query({ access_token: token })
-                .send({ name: GROUP_NAME})
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(409);
-                    done();
-                });
-        });
-    });
-
-    describe('get', function () {
-        it('cannot get non-existing group', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/groups/nope')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('cannot get existing group with normal user', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/groups/admin')
-                .query({ access_token: token_1 })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(403);
-                    done();
-                });
-        });
-
-        it('can get existing group', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/groups/admin')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-                    expect(result.body.name).to.be('admin');
-                    expect(result.body.userIds.length).to.be(1);
-                    expect(result.body.userIds[0]).to.be(userId);
-                    done();
-                });
-        });
-    });
-
-    describe('remove', function () {
-        it('cannot remove without token', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/groups/externals')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('can remove empty group', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/groups/' + groupObject.id)
-                .send({ password: PASSWORD })
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(204);
-                    done();
-                });
-        });
-
-        it('cannot remove non-empty group', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/groups/admin')
-                .send({ password: PASSWORD })
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(409);
-                    done();
-                });
-        });
-    });
-
-    describe('Set groups', function () {
-        var group0Object, group1Object;
-        before(function (done) {
-            groups.create('group0', function (e, r) {
-                group0Object = r;
-                groups.create('group1', function (e, r) {
-                    group1Object = r;
-                    done();
-                });
+    it('create fails due to mising token', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/groups')
+            .send({ name: GROUP_NAME})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
             });
-        });
+    });
 
-        it('cannot add user to invalid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
-                .query({ access_token: token })
-                .send({ groupIds: [ 'admin', 'something' ]})
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(404);
-                    done();
-                });
-        });
+    it('create succeeds', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/groups')
+            .query({ access_token: token })
+            .send({ name: GROUP_NAME})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(201);
+                groupObject = result.body;
+                done();
+            });
+    });
 
-        it('can add user to valid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
-                .query({ access_token: token })
-                .send({ groupIds: [ 'admin', group0Object.id, group1Object.id ]})
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(204);
-                    done();
-                });
-        });
+    it('create fails for already exists', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/groups')
+            .query({ access_token: token })
+            .send({ name: GROUP_NAME})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(409);
+                done();
+            });
+    });
 
-        it('cannot remove self from admin', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
-                .query({ access_token: token })
-                .send({ groupIds: [ group0Object.id, group1Object.id ]})
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(403); // not allowed
-                    done();
-                });
-        });
+    it('can create another group', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/groups')
+            .query({ access_token: token })
+            .send({ name: 'group1'})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(201);
+                group1Object = result.body;
+                done();
+            });
+    })
 
-        it('can add another user to admin', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId_1 + '/groups')
-                .query({ access_token: token })
-                .send({ groupIds: [ 'admin' ]})
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(204);
-                    done();
-                });
-        });
+    it('cannot add user to invalid group', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
+            .query({ access_token: token })
+            .send({ groupIds: [ groupObject.id, 'something' ]})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+    });
 
-        it('lists members of admin group', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/groups/admin')
-                .query({ access_token: token })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-                    expect(result.body.userIds.length).to.be(2);
-                    expect(result.body.userIds[0]).to.be(userId);
-                    expect(result.body.userIds[1]).to.be(userId_1);
-                    done();
-                });
-        });
+    it('can set groups of a user', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
+            .query({ access_token: token })
+            .send({ groupIds: [ groupObject.id, group1Object.id ]})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+    });
 
-        it('can add user_1 to admin', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId_1 + '/groups')
-                .query({ access_token: token })
-                .send({ groupIds: [ 'admin' ]})
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(204);
+    it('can set users of a group', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/groups/' + groupObject.id + '/members')
+            .query({ access_token: token })
+            .send({ userIds: [ userId, userId_1 ]})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                done();
+            });
+    });
 
-                    token_1 = tokendb.generateToken();
+    it('cannot get non-existing group', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/groups/nope')
+            .query({ access_token: token })
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+    });
 
-                    // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                    tokendb.add(token_1, userId_1, 'test-client-id',  Date.now() + 100000, accesscontrol.SCOPE_ANY, done);
-                });
-        });
+    it('cannot get existing group with normal user', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/groups/' + groupObject.id)
+            .query({ access_token: token_1 })
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(403);
+                done();
+            });
+    });
 
-        it('remove activation user from admin', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
-                .query({ access_token: token_1 })
-                .send({ groupIds: [ group1Object.id, group0Object.id ]})
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(204); // user_1 is still admin, so we can remove the other person
-                    done();
-                });
-        });
+    it('can get existing group', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/groups/' + groupObject.id)
+            .query({ access_token: token })
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.name).to.be(groupObject.name);
+                expect(result.body.userIds.length).to.be(2);
+                expect(result.body.userIds[0]).to.be(userId);
+                expect(result.body.userIds[1]).to.be(userId_1);
+                done();
+            });
+    });
+
+    it('cannot list groups without token', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/groups')
+            .end(function (err, res) {
+                expect(res.statusCode).to.equal(401);
+                done();
+            });
+    });
+
+    it('cannot list groups as normal user', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/groups')
+            .query({ access_token: token_1 })
+            .end(function (err, res) {
+                expect(res.statusCode).to.equal(403);
+                done();
+            });
+    });
+
+    it('can list groups', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/groups')
+            .query({ access_token: token })
+            .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.groups).to.be.an(Array);
+                expect(res.body.groups.length).to.be(2);
+                expect(res.body.groups[0].name).to.eql(groupObject.name);
+                expect(res.body.groups[1].name).to.eql(group1Object.name);
+                done();
+            });
+    });
+
+    it('remove user from group', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
+            .query({ access_token: token })
+            .send({ groupIds: [ groupObject.id ]})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+    });
+
+    it('cannot remove without token', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/groups/externals')
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+    });
+
+    it('can clear users of a group', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/groups/' + group1Object.id + '/members')
+            .query({ access_token: token })
+            .send({ userIds: [ ]})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                done();
+            });
+    });
+
+    it('can remove empty group', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/groups/' + group1Object.id)
+            .send({ password: PASSWORD })
+            .query({ access_token: token })
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+    });
+
+    it('can remove non-empty group', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/groups/' + groupObject.id)
+            .send({ password: PASSWORD })
+            .query({ access_token: token })
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
     });
 });

src/routes/test/oauth2-test.js

@@ -15,7 +15,7 @@ var accesscontrol = require('../../accesscontrol.js'),
     database = require('../../database.js'),
     domains = require('../../domains.js'),
     expect = require('expect.js'),
-    hat = require('hat'),
+    hat = require('../../hat.js'),
     nock = require('nock'),
     oauth2 = require('../oauth2.js'),
     querystring = require('querystring'),
@@ -62,7 +62,8 @@ describe('OAuth2', function () {
             domain: DOMAIN_0.domain,
             portBindings: {},
             accessRestriction: null,
-            memoryLimit: 0
+            memoryLimit: 0,
+            ownerId: USER_0.id
         };
 
         var APP_1 = {
@@ -73,7 +74,8 @@ describe('OAuth2', function () {
             domain: DOMAIN_0.domain,
             portBindings: {},
             accessRestriction: { users: [ 'foobar' ] },
-            memoryLimit: 0
+            memoryLimit: 0,
+            ownerId: USER_0.id
         };
 
         var APP_2 = {
@@ -84,7 +86,8 @@ describe('OAuth2', function () {
             domain: DOMAIN_0.domain,
             portBindings: {},
             accessRestriction: { users: [ USER_0.id ] },
-            memoryLimit: 0
+            memoryLimit: 0,
+            ownerId: USER_0.id
         };
 
         var APP_3 = {
@@ -95,7 +98,8 @@ describe('OAuth2', function () {
             domain: DOMAIN_0.domain,
             portBindings: {},
             accessRestriction: { groups: [ 'someothergroup', 'admin', 'anothergroup' ] },
-            memoryLimit: 0
+            memoryLimit: 0,
+            ownerId: USER_0.id
         };
 
         // unknown app
@@ -214,19 +218,22 @@ describe('OAuth2', function () {
                 clientdb.add.bind(null, CLIENT_6.id, CLIENT_6.appId, CLIENT_6.type, CLIENT_6.clientSecret, CLIENT_6.redirectURI, CLIENT_6.scope),
                 clientdb.add.bind(null, CLIENT_7.id, CLIENT_7.appId, CLIENT_7.type, CLIENT_7.clientSecret, CLIENT_7.redirectURI, CLIENT_7.scope),
                 clientdb.add.bind(null, CLIENT_9.id, CLIENT_9.appId, CLIENT_9.type, CLIENT_9.clientSecret, CLIENT_9.redirectURI, CLIENT_9.scope),
-                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.domain, APP_0.portBindings, APP_0),
-                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.domain, APP_1.portBindings, APP_1),
-                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.domain, APP_2.portBindings, APP_2),
-                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.domain, APP_3.portBindings, APP_3),
                 function (callback) {
-                    users.create(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, null /* source */, function (error, userObject) {
+                    users.create(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, { }, null /* source */, function (error, userObject) {
                         expect(error).to.not.be.ok();
 
                         // update the global objects to reflect the new user id
-                        USER_0.id = userObject.id;
+                        USER_0.id = APP_0.ownerId = APP_1.ownerId = APP_2.ownerId = APP_3.ownerId = userObject.id;
                         APP_2.accessRestriction = { users: [ 'foobar', userObject.id ] };
 
-                        appdb.update(APP_2.id, APP_2, callback);
+                        async.series([
+                            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.domain, APP_0.ownerId, APP_0.portBindings, APP_0),
+                            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.domain, APP_1.ownerId, APP_1.portBindings, APP_1),
+                            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.domain, APP_2.ownerId, APP_2.portBindings, APP_2),
+                            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.domain, APP_3.ownerId, APP_3.portBindings, APP_3),
+
+                            appdb.update.bind(null, APP_2.id, APP_2)
+                        ], callback);
                     });
                 },
             ], done);
@@ -1341,9 +1348,19 @@ describe('Password', function () {
                 });
         });
 
-        it('setup succeeds', function (done) {
+        it('setup fails without email', function (done) {
             superagent.get(SERVER_URL + '/api/v1/session/account/setup.html')
                 .query({ reset_token: USER_0.resetToken })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(result.text.indexOf('<!-- error tester -->')).to.not.equal(-1);
+                    done();
+                });
+        });
+
+        it('setup succeeds', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/session/account/setup.html')
+                .query({ email: USER_0.email, reset_token: USER_0.resetToken })
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(200);
                     expect(result.text.indexOf('<!-- tester -->')).to.not.equal(-1);
@@ -1361,7 +1378,16 @@ describe('Password', function () {
 
         it('reset fails due to invalid reset_token', function (done) {
             superagent.get(SERVER_URL + '/api/v1/session/password/reset.html')
-                .query({ reset_token: hat(256) })
+                .query({ email: USER_0.email, reset_token: hat(256) })
+                .end(function (error, result) {
+                    expect(result.statusCode).to.equal(401);
+                    done();
+                });
+        });
+
+        it('reset fails due to invalid email', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/session/password/reset.html')
+                .query({ email: USER_0.email + 'x', reset_token: hat(256) })
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(401);
                     done();
@@ -1370,7 +1396,7 @@ describe('Password', function () {
 
         it('reset succeeds', function (done) {
             superagent.get(SERVER_URL + '/api/v1/session/password/reset.html')
-                .query({ reset_token: USER_0.resetToken })
+                .query({ email: USER_0.email, reset_token: USER_0.resetToken })
                 .end(function (error, result) {
                     expect(result.text.indexOf('<!-- tester -->')).to.not.equal(-1);
                     expect(result.statusCode).to.equal(200);
@@ -1427,7 +1453,7 @@ describe('Password', function () {
 
         it('fails due to empty password', function (done) {
             superagent.post(SERVER_URL + '/api/v1/session/password/reset')
-                .send({ password: '', resetToken: hat(256) })
+                .send({ password: '', email: USER_0.email, resetToken: hat(256) })
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(401);
                     done();
@@ -1436,7 +1462,7 @@ describe('Password', function () {
 
         it('fails due to empty resetToken', function (done) {
             superagent.post(SERVER_URL + '/api/v1/session/password/reset')
-                .send({ password: '', resetToken: '' })
+                .send({ password: '', email: USER_0.email, resetToken: '' })
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(401);
                     done();
@@ -1445,7 +1471,7 @@ describe('Password', function () {
 
         it('fails due to weak password', function (done) {
             superagent.post(SERVER_URL + '/api/v1/session/password/reset')
-                .send({ password: 'foobar', resetToken: USER_0.resetToken })
+                .send({ password: 'foobar', email: USER_0.email, resetToken: USER_0.resetToken })
                 .end(function (error, result) {
                     expect(result.statusCode).to.equal(406);
                     done();
@@ -1460,7 +1486,7 @@ describe('Password', function () {
                 .get('/').reply(200, {});
 
             superagent.post(SERVER_URL + '/api/v1/session/password/reset')
-                .send({ password: 'ASF23$%somepassword', resetToken: USER_0.resetToken })
+                .send({ password: '12345678', email: USER_0.email, resetToken: USER_0.resetToken })
                 .end(function (error, result) {
                     expect(scope.isDone()).to.be.ok();
                     expect(result.statusCode).to.equal(200);

src/routes/test/profile-test.js

@@ -6,8 +6,7 @@
 
 'use strict';
 
-var accesscontrol = require('../../accesscontrol.js'),
-    config = require('../../config.js'),
+var config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
     mailer = require('../../mailer.js'),
@@ -102,7 +101,6 @@ describe('Profile API', function () {
                 expect(result.body.username).to.equal(USERNAME_0.toLowerCase());
                 expect(result.body.email).to.equal(EMAIL_0.toLowerCase());
                 expect(result.body.fallbackEmail).to.equal(EMAIL_0.toLowerCase());
-                expect(result.body.admin).to.be.ok();
                 expect(result.body.displayName).to.be.a('string');
                 expect(result.body.password).to.not.be.ok();
                 expect(result.body.salt).to.not.be.ok();
@@ -117,7 +115,7 @@ describe('Profile API', function () {
             var token = tokendb.generateToken();
             var expires = Date.now() - 2000; // 1 sec
 
-            tokendb.add(token, user_0.id, null, expires, accesscontrol.SCOPE_ANY, function (error) {
+            tokendb.add(token, user_0.id, null, expires, 'profile', function (error) {
                 expect(error).to.not.be.ok();
 
                 superagent.get(SERVER_URL + '/api/v1/profile').query({ access_token: token }).end(function (error, result) {
@@ -140,7 +138,6 @@ describe('Profile API', function () {
                 expect(result.statusCode).to.equal(200);
                 expect(result.body.username).to.equal(USERNAME_0.toLowerCase());
                 expect(result.body.email).to.equal(EMAIL_0.toLowerCase());
-                expect(result.body.admin).to.be.ok();
                 expect(result.body.displayName).to.be.a('string');
                 expect(result.body.password).to.not.be.ok();
                 expect(result.body.salt).to.not.be.ok();
@@ -196,7 +193,6 @@ describe('Profile API', function () {
                             expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
                             expect(res.body.email).to.equal(EMAIL_0_NEW.toLowerCase());
                             expect(res.body.fallbackEmail).to.equal(EMAIL_0_NEW_FALLBACK.toLowerCase());
-                            expect(res.body.admin).to.equal(true);
                             expect(res.body.displayName).to.equal('');
 
                             done();
@@ -217,7 +213,6 @@ describe('Profile API', function () {
                             expect(res.statusCode).to.equal(200);
                             expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
                             expect(res.body.email).to.equal(EMAIL_0_NEW.toLowerCase());
-                            expect(res.body.admin).to.be.ok();
                             expect(res.body.displayName).to.equal(DISPLAY_NAME_0_NEW);
 
                             done();

src/routes/test/sysadmin-test.js

@@ -5,25 +5,16 @@
 
 'use strict';
 
-var appdb = require('../../appdb.js'),
-    async = require('async'),
+var async = require('async'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     domains = require('../../domains.js'),
+    eventlog = require('../../eventlog.js'),
     expect = require('expect.js'),
-    hock = require('hock'),
-    http = require('http'),
-    MockS3 = require('mock-aws-s3'),
-    os = require('os'),
-    path = require('path'),
-    rimraf = require('rimraf'),
-    s3 = require('../../storage/s3.js'),
-    safe = require('safetydance'),
     server = require('../../server.js'),
     settings = require('../../settings.js'),
     settingsdb = require('../../settingsdb.js'),
-    superagent = require('superagent'),
-    url = require('url');
+    superagent = require('superagent');
 
 const SERVER_URL = 'http://localhost:' + config.get('port');
 
@@ -55,73 +46,47 @@ function setup(done) {
                 .end(function (error, result) {
                     expect(result).to.be.ok();
                     expect(result.statusCode).to.eql(201);
-
                     callback();
                 });
         },
 
-        function addApp(callback) {
-            var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', DOMAIN_0.domain, [ ] /* portBindings */, { }, callback);
-        },
-
         function createSettings(callback) {
-            MockS3.config.basePath = path.join(os.tmpdir(), 's3-sysadmin-test-buckets/');
-
-            s3._mockInject(MockS3);
-
-            safe.fs.mkdirSync('/tmp/box-sysadmin-test');
             settingsdb.set(settings.BACKUP_CONFIG_KEY, JSON.stringify({ provider: 'caas', token: 'BACKUP_TOKEN', fqdn: DOMAIN_0.domain, key: 'key', prefix: 'boxid', format: 'tgz'}), callback);
         }
     ], done);
 }
 
 function cleanup(done) {
-    s3._mockRestore();
-    rimraf.sync(MockS3.config.basePath);
-
     database._clear(function (error) {
         expect(!error).to.be.ok();
-
         server.stop(done);
     });
 }
 
 describe('Internal API', function () {
+    this.timeout(5000);
+
     before(setup);
     after(cleanup);
 
-    var apiHockInstance = hock.createHock({ throwOnUnmatched: false }), apiHockServer;
-
-    before(function (done) {
-        apiHockInstance
-            .post('/api/v1/boxes/' + config.adminDomain() + '/awscredentials?token=BACKUP_TOKEN')
-            .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey' } });
-        var port = parseInt(url.parse(config.apiServerOrigin()).port, 10);
-        apiHockServer = http.createServer(apiHockInstance.handler).listen(port, done);
-    });
-
-    after(function (done) {
-        apiHockServer.close();
-        done();
-    });
-
     describe('backup', function () {
         it('succeeds', function (done) {
             superagent.post(config.sysadminOrigin() + '/api/v1/backup')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(202);
+              .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
 
-                    function checkAppstoreServerCalled() {
-                        apiHockInstance.done(function (error) {
-                            if (!error) return done();
+                function checkBackupStartEvent() {
+                    eventlog.getAllPaged([ eventlog.ACTION_BACKUP_START ], '', 1, 100, function (error, result) {
+                        expect(error).to.equal(null);
 
-                            setTimeout(checkAppstoreServerCalled, 100);
-                        });
-                    }
+                        if (result.length === 0) return setTimeout(checkBackupStartEvent, 1000);
 
-                    checkAppstoreServerCalled();
-                });
+                        done();
+                    });
+                }
+
+                checkBackupStartEvent();
+            });
         });
     });
-});
+});

src/routes/test/user-test.js

@@ -1,121 +0,0 @@
-'use strict';
-
-/* global it:false */
-/* global describe:false */
-/* global before:false */
-/* global after:false */
-
-var accesscontrol = require('../../accesscontrol.js'),
-    async = require('async'),
-    config = require('../../config.js'),
-    database = require('../../database.js'),
-    expect = require('expect.js'),
-    nock = require('nock'),
-    superagent = require('superagent'),
-    server = require('../../server.js'),
-    settings = require('../../settings.js'),
-    settingsdb = require('../../settingsdb.js'),
-    tokendb = require('../../tokendb.js');
-
-var SERVER_URL = 'http://localhost:' + config.get('port');
-
-var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
-var token = null; // authentication token
-var USERNAME_1 = 'userTheFirst', EMAIL_1 = 'taO@zen.mac', userId_1, token_1;
-
-function setup(done) {
-    nock.cleanAll();
-    config._reset();
-    config.setFqdn('example-cloudron-test.com');
-    config.setAdminFqdn('my.example-cloudron-test.com');
-
-    async.series([
-        server.start.bind(server),
-        database._clear,
-        settings.setBackupConfig.bind(null, { provider: 'filesystem', backupFolder: '/tmp', format: 'tgz' }),
-        settingsdb.set.bind(null, settings.APPSTORE_CONFIG_KEY, JSON.stringify({ userId: 'USER_ID', cloudronId: 'CLOUDRON_ID', token: 'ACCESS_TOKEN' }))
-    ], done);
-}
-
-function cleanup(done) {
-    database._clear(function (error) {
-        expect(error).to.not.be.ok();
-
-        config._reset();
-
-        server.stop(done);
-    });
-}
-
-describe('User test', function () {
-    describe('get config', function () {
-        before(function (done) {
-            async.series([
-                setup,
-
-                function (callback) {
-                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
-                        .query({ setupToken: 'somesetuptoken' })
-                        .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
-                        .end(function (error, result) {
-                            expect(result).to.be.ok();
-
-                            // stash token for further use
-                            token = result.body.token;
-
-                            callback();
-                        });
-                },
-
-                function (callback) {
-                    superagent.post(SERVER_URL + '/api/v1/users')
-                        .query({ access_token: token })
-                        .send({ username: USERNAME_1, email: EMAIL_1, invite: false })
-                        .end(function (error, result) {
-                            expect(result).to.be.ok();
-                            expect(result.statusCode).to.eql(201);
-
-                            token_1 = tokendb.generateToken();
-                            userId_1 = result.body.id;
-
-                            // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                            tokendb.add(token_1, userId_1, 'test-client-id',  Date.now() + 100000, accesscontrol.SCOPE_ANY, callback);
-                        });
-                }
-            ], done);
-        });
-
-        after(cleanup);
-
-        it('cannot get without token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/user/cloudron_config')
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-        });
-
-        it('succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/user/cloudron_config')
-                .query({ access_token: token_1 })
-                .end(function (error, result) {
-                    expect(result.statusCode).to.equal(200);
-
-                    expect(result.body.apiServerOrigin).to.eql('http://localhost:6060');
-                    expect(result.body.webServerOrigin).to.eql(null);
-                    expect(result.body.adminFqdn).to.eql(config.adminFqdn());
-                    expect(result.body.progress).to.be.an('object');
-                    expect(result.body.version).to.eql(config.version());
-                    expect(result.body.cloudronName).to.be.a('string');
-                    expect(result.body.provider).to.be.a('string');
-
-                    expect(result.body.update).to.be(undefined);
-                    expect(result.body.size).to.be(undefined);
-                    expect(result.body.region).to.be(undefined);
-                    expect(result.body.memory).to.be(undefined);
-
-                    done();
-                });
-        });
-    });
-});

src/routes/test/users-test.js

@@ -165,7 +165,8 @@ describe('Users API', function () {
                 expect(res.statusCode).to.equal(200);
                 expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
                 expect(res.body.email).to.equal(EMAIL_0.toLowerCase());
-                expect(res.body.admin).to.be.ok();
+                expect(res.body.groupIds).to.eql([]);
+                expect(res.body.admin).to.be(true);
 
                 done();
             });
@@ -196,7 +197,9 @@ describe('Users API', function () {
                 expect(res.statusCode).to.equal(200);
                 expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
                 expect(res.body.email).to.equal(EMAIL_0.toLowerCase());
-                expect(res.body.admin).to.be.ok();
+                expect(res.body.groupIds).to.eql([]);
+                expect(res.body.admin).to.be(true);
+
                 done();
             });
     });
@@ -235,7 +238,8 @@ describe('Users API', function () {
                 expect(res.statusCode).to.equal(200);
                 expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
                 expect(res.body.email).to.equal(EMAIL_0.toLowerCase());
-                expect(res.body.admin).to.be.ok();
+                expect(res.body.groupIds).to.eql([]);
+                expect(res.body.admin).to.be(true);
                 expect(res.body.displayName).to.be.a('string');
                 expect(res.body.password).to.not.be.ok();
                 expect(res.body.salt).to.not.be.ok();
@@ -320,9 +324,9 @@ describe('Users API', function () {
     });
 
     it('set second user as admin succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/groups')
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_1.id)
             .query({ access_token: token })
-            .send({ groupIds: [ constants.ADMIN_GROUP_ID ] })
+            .send({ admin: true })
             .end(function (err, res) {
                 expect(res.statusCode).to.equal(204);
 
@@ -330,14 +334,14 @@ describe('Users API', function () {
                     .query({ access_token: token })
                     .end(function (err, res) {
                         expect(res.statusCode).to.equal(200);
-                        expect(res.body.admin).to.equal(true);
+                        expect(res.body.admin).to.be(true);
 
                         done();
                     });
             });
     });
 
-    it('list groupIds when listing users', function (done) {
+    it('does not list groupIds when listing users', function (done) {
         superagent.get(SERVER_URL + '/api/v1/users')
             .query({ access_token: token })
             .end(function (error, res) {
@@ -346,19 +350,18 @@ describe('Users API', function () {
                 expect(res.body.users).to.be.an('array');
 
                 res.body.users.forEach(function (user) {
-                    expect(user.admin).to.be(true);
-                    expect(user.groupIds).to.eql([ constants.ADMIN_GROUP_ID ]);
+                    expect('groupIds' in user).to.be(false);
                 });
                 done();
             });
     });
 
-    it('remove itself from admins fails', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id + '/groups')
+    it('remove self as admin fails', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_0.id)
             .query({ access_token: token })
-            .send({ groupIds: [ groupObject.id ] })
+            .send({ admin: false })
             .end(function (err, res) {
-                expect(res.statusCode).to.equal(403);
+                expect(res.statusCode).to.equal(409);
                 done();
             });
     });
@@ -374,7 +377,7 @@ describe('Users API', function () {
                     .query({ access_token: token })
                     .end(function (err, res) {
                         expect(res.statusCode).to.equal(200);
-                        expect(res.body.admin).to.equal(false);
+                        expect(res.body.groupIds).to.eql([ groupObject.id ]);
 
                         done();
                     });
@@ -461,7 +464,7 @@ describe('Users API', function () {
                 expect(result.statusCode).to.equal(200);
                 expect(result.body.username).to.equal(USERNAME_2.toLowerCase());
                 expect(result.body.email).to.equal(EMAIL_2.toLowerCase());
-                expect(result.body.admin).to.not.be.ok();
+                expect(result.body.groupIds).to.eql([]);
 
                 done();
             });
@@ -502,8 +505,7 @@ describe('Users API', function () {
                     expect(user.email).to.be.ok();
                     expect(user.password).to.not.be.ok();
                     expect(user.salt).to.not.be.ok();
-                    expect(user.groupIds).to.be.an(Array);
-                    expect(user.admin).to.be.a('boolean');
+                    expect(user.groupIds).to.not.be.ok();
                 });
 
                 done();
@@ -525,7 +527,7 @@ describe('Users API', function () {
             .query({ access_token: token })
             .send({ password: PASSWORD })
             .end(function (err, res) {
-                expect(res.statusCode).to.equal(403);
+                expect(res.statusCode).to.equal(409);
                 done();
             });
     });
@@ -574,7 +576,7 @@ describe('Users API', function () {
             .query({ access_token: token })
             .send({ password: PASSWORD })
             .end(function (err, res) {
-                expect(res.statusCode).to.equal(403);
+                expect(res.statusCode).to.equal(409);
                 done();
             });
     });
@@ -622,7 +624,6 @@ describe('Users API', function () {
                         expect(res.statusCode).to.equal(200);
                         expect(res.body.username).to.equal(USERNAME_2.toLowerCase());
                         expect(res.body.email).to.equal(EMAIL_2_NEW.toLowerCase());
-                        expect(res.body.admin).to.equal(false);
                         expect(res.body.displayName).to.equal('');
 
                         done();
@@ -643,7 +644,6 @@ describe('Users API', function () {
                         expect(res.statusCode).to.equal(200);
                         expect(res.body.username).to.equal(USERNAME_2.toLowerCase());
                         expect(res.body.email).to.equal(EMAIL_2.toLowerCase());
-                        expect(res.body.admin).to.equal(false);
                         expect(res.body.displayName).to.equal('');
 
                         done();
@@ -664,7 +664,6 @@ describe('Users API', function () {
                         expect(res.statusCode).to.equal(200);
                         expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
                         expect(res.body.email).to.equal(EMAIL_0.toLowerCase());
-                        expect(res.body.admin).to.be.ok();
                         expect(res.body.displayName).to.equal(DISPLAY_NAME_0_NEW);
 
                         done();

src/routes/user.js

@@ -1,20 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    getCloudronConfig: getCloudronConfig
-};
-
-var cloudron = require('../cloudron.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    _ = require('underscore');
-
-function getCloudronConfig(req, res, next) {
-    cloudron.getConfig(function (error, cloudronConfig) {
-        if (error) return next(new HttpError(500, error));
-
-        var result = _.pick(cloudronConfig, 'apiServerOrigin', 'webServerOrigin', 'fqdn', 'adminFqdn', 'version', 'progress', 'isDemo', 'cloudronName', 'provider');
-
-        next(new HttpSuccess(200, result));
-    });
-}

src/routes/users.js

@@ -8,12 +8,11 @@ exports = module.exports = {
     remove: remove,
     verifyPassword: verifyPassword,
     sendInvite: sendInvite,
-    setGroups: setGroups
+    setGroups: setGroups,
+    transferOwnership: transferOwnership
 };
 
 var assert = require('assert'),
-    constants = require('../constants.js'),
-    generatePassword = require('../password.js').generate,
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     users = require('../users.js'),
@@ -33,13 +32,13 @@ function create(req, res, next) {
     if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
     if ('password' in req.body && typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be string'));
 
-    var password = req.body.password || generatePassword();
+    var password = req.body.password || null;
     var email = req.body.email;
     var sendInvite = req.body.invite;
     var username = 'username' in req.body ? req.body.username : null;
     var displayName = req.body.displayName || '';
 
-    users.create(username, password, email, displayName, auditSource(req), { invitor: req.user, sendInvite: sendInvite }, function (error, user) {
+    users.create(username, password, email, displayName, { invitor: req.user, sendInvite: sendInvite }, auditSource(req), function (error, user) {
         if (error && error.reason === UsersError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === UsersError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
@@ -50,12 +49,11 @@ function create(req, res, next) {
             displayName: user.displayName,
             email: user.email,
             fallbackEmail: user.fallbackEmail,
-            admin: user.admin,
             groupIds: [ ],
             resetToken: user.resetToken
         };
 
-        next(new HttpSuccess(201, userInfo ));
+        next(new HttpSuccess(201, userInfo));
     });
 }
 
@@ -69,7 +67,11 @@ function update(req, res, next) {
     if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
     if ('username' in req.body && typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be a string'));
 
-    if (req.user.id !== req.params.userId && !req.user.admin) return next(new HttpError(403, 'Not allowed'));
+    if ('admin' in req.body) {
+        if (typeof req.body.admin !== 'boolean') return next(new HttpError(400, 'admin must be a boolean'));
+        // this route is only allowed for admins, so req.user has to be an admin
+        if (req.user.id === req.params.userId && !req.body.admin) return next(new HttpError(409, 'Cannot remove admin flag on self'));
+    }
 
     users.update(req.params.userId, req.body, auditSource(req), function (error) {
         if (error && error.reason === UsersError.BAD_FIELD) return next(new HttpError(400, error.message));
@@ -85,7 +87,7 @@ function list(req, res, next) {
     users.list(function (error, results) {
         if (error) return next(new HttpError(500, error));
 
-        results = results.map(users.removePrivateFields);
+        results = results.map(users.removeRestrictedFields);
 
         next(new HttpSuccess(200, { users: results }));
     });
@@ -95,8 +97,6 @@ function get(req, res, next) {
     assert.strictEqual(typeof req.params.userId, 'string');
     assert.strictEqual(typeof req.user, 'object');
 
-    if (req.user.id !== req.params.userId && !req.user.admin) return next(new HttpError(403, 'Not allowed'));
-
     users.get(req.params.userId, function (error, result) {
         if (error && error.reason === UsersError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
@@ -108,12 +108,7 @@ function get(req, res, next) {
 function remove(req, res, next) {
     assert.strictEqual(typeof req.params.userId, 'string');
 
-    // rules:
-    // - admin can remove any user
-    // - admin cannot remove admin
-    // - user cannot remove himself <- TODO should this actually work?
-
-    if (req.user.id === req.params.userId) return next(new HttpError(403, 'Not allowed to remove yourself.'));
+    if (req.user.id === req.params.userId) return next(new HttpError(409, 'Not allowed to remove yourself.'));
 
     users.remove(req.params.userId, auditSource(req), function (error) {
         if (error && error.reason === UsersError.BAD_FIELD) return next(new HttpError(400, error.message));
@@ -127,14 +122,13 @@ function remove(req, res, next) {
 function verifyPassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    // using an 'sdk' token we skip password checks
-    if (req.authInfo.clientId === 'cid-sdk' || req.authInfo.clientId === 'cid-cli') return next();
+    if (req.authInfo.skipPasswordVerification) return next(); // using an 'sdk' token we skip password checks
 
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'API call requires user password'));
 
     users.verifyWithUsername(req.user.username, req.body.password, function (error) {
-        if (error && error.reason === UsersError.WRONG_PASSWORD) return next(new HttpError(403, 'Password incorrect'));
-        if (error && error.reason === UsersError.NOT_FOUND) return next(new HttpError(403, 'Password incorrect'));
+        if (error && error.reason === UsersError.WRONG_PASSWORD) return next(new HttpError(403, 'Password incorrect')); // not 401 intentionally
+        if (error && error.reason === UsersError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
 
         req.body.password = '<redacted>'; // this will prevent logs from displaying plain text password
@@ -160,13 +154,24 @@ function setGroups(req, res, next) {
 
     if (!Array.isArray(req.body.groupIds)) return next(new HttpError(400, 'API call requires a groups array.'));
 
-    // this route is only allowed for admins, so req.user has to be an admin
-    if (req.user.id === req.params.userId && req.body.groupIds.indexOf(constants.ADMIN_GROUP_ID) === -1) return next(new HttpError(403, 'Admin removing itself from admins is not allowed'));
-
-    users.setGroups(req.params.userId, req.body.groupIds, function (error) {
+    users.setMembership(req.params.userId, req.body.groupIds, function (error) {
         if (error && error.reason === UsersError.NOT_FOUND) return next(new HttpError(404, 'One or more groups not found'));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(204));
     });
 }
+
+function transferOwnership(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.params.userId, 'string');
+
+    if (typeof req.body.ownerId !== 'string') return next(new HttpError(400, 'ownerId must be a string'));
+
+    users.transferOwnership(req.params.userId, req.body.ownerId, auditSource(req), function (error) {
+        if (error && error.reason === UsersError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, {}));
+    });
+}

src/scripts/update.sh

@@ -9,7 +9,6 @@ fi
 
 readonly UPDATER_SERVICE="cloudron-updater"
 readonly DATA_FILE="/root/cloudron-update-data.json"
-readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 300"
 
 if [[ $# == 1 && "$1" == "--check" ]]; then
     echo "OK"
@@ -17,30 +16,16 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [[ $# != 2 ]]; then
-    echo "sourceTarballUrl and data arguments required"
+    echo "sourceDir and data arguments required"
     exit 1
 fi
 
-readonly sourceTarballUrl="${1}"
+readonly source_dir="${1}"
 readonly data="${2}"
 
-echo "Updating Cloudron with ${sourceTarballUrl}"
+echo "Updating Cloudron with ${source_dir}"
 
-# TODO: pre-download tarball
-box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)
-readonly installer_path="${box_src_tmp_dir}/scripts/installer.sh"
-echo "Downloading box code from ${sourceTarballUrl} to ${box_src_tmp_dir}"
-
-for try in `seq 1 10`; do
-    if $curl -L "${sourceTarballUrl}" | tar -zxf - -C "${box_src_tmp_dir}"; then break; fi
-    echo "Failed to download source tarball, trying again"
-    sleep 5
-done
-
-if [[ ${try} -eq 10 ]]; then
-    echo "Release tarball download failed"
-    exit 3
-fi
+readonly installer_path="${source_dir}/scripts/installer.sh"
 
 echo "=> reset service ${UPDATER_SERVICE} status in case it failed"
 if systemctl reset-failed "${UPDATER_SERVICE}"; then

src/server.js

@@ -13,7 +13,7 @@ var accesscontrol = require('./accesscontrol.js'),
     database = require('./database.js'),
     eventlog = require('./eventlog.js'),
     express = require('express'),
-    hat = require('hat'),
+    hat = require('./hat.js'),
     http = require('http'),
     middleware = require('./middleware'),
     passport = require('passport'),
@@ -91,12 +91,16 @@ function initializeExpressSync() {
     // scope middleware implicitly also adds bearer token verification
     var cloudronScope = routes.accesscontrol.scope(accesscontrol.SCOPE_CLOUDRON);
     var profileScope = routes.accesscontrol.scope(accesscontrol.SCOPE_PROFILE);
-    var usersScope = routes.accesscontrol.scope(accesscontrol.SCOPE_USERS);
-    var appsScope = routes.accesscontrol.scope(accesscontrol.SCOPE_APPS);
+    var usersReadScope = routes.accesscontrol.scope(accesscontrol.SCOPE_USERS_READ);
+    var usersManageScope = routes.accesscontrol.scope(accesscontrol.SCOPE_USERS_MANAGE);
+    var appsReadScope = routes.accesscontrol.scope(accesscontrol.SCOPE_APPS_READ);
+    var appsManageScope = [ routes.accesscontrol.scope(accesscontrol.SCOPE_APPS_MANAGE), routes.accesscontrol.verifyAppOwnership ];
     var settingsScope = routes.accesscontrol.scope(accesscontrol.SCOPE_SETTINGS);
     var mailScope = routes.accesscontrol.scope(accesscontrol.SCOPE_MAIL);
     var clientsScope = routes.accesscontrol.scope(accesscontrol.SCOPE_CLIENTS);
-    var domainsScope = routes.accesscontrol.scope(accesscontrol.SCOPE_DOMAINS);
+    var domainsReadScope = routes.accesscontrol.scope(accesscontrol.SCOPE_DOMAINS_READ);
+    var domainsManageScope = routes.accesscontrol.scope(accesscontrol.SCOPE_DOMAINS_MANAGE);
+    var appstoreScope = routes.accesscontrol.scope(accesscontrol.SCOPE_APPSTORE);
 
     // csrf protection
     var csrf = routes.oauth2.csrf();
@@ -114,23 +118,24 @@ function initializeExpressSync() {
     router.post('/api/v1/developer/login', routes.developer.login);
 
     // cloudron routes
-    router.get ('/api/v1/cloudron/config', cloudronScope, routes.cloudron.getConfig);
+    router.get ('/api/v1/cloudron/update', cloudronScope, routes.cloudron.getUpdateInfo);
     router.post('/api/v1/cloudron/update', cloudronScope, routes.cloudron.update);
     router.post('/api/v1/cloudron/check_for_updates', cloudronScope, routes.cloudron.checkForUpdates);
     router.post('/api/v1/cloudron/reboot', cloudronScope, routes.cloudron.reboot);
     router.get ('/api/v1/cloudron/graphs', cloudronScope, routes.graphs.getGraphs);
     router.get ('/api/v1/cloudron/disks', cloudronScope, routes.cloudron.getDisks);
-    router.get ('/api/v1/cloudron/logs', cloudronScope, routes.cloudron.getLogs);
-    router.get ('/api/v1/cloudron/logstream', cloudronScope, routes.cloudron.getLogStream);
+    router.get ('/api/v1/cloudron/logs/:unit', cloudronScope, routes.cloudron.getLogs);
+    router.get ('/api/v1/cloudron/logstream/:unit', cloudronScope, routes.cloudron.getLogStream);
     router.get ('/api/v1/cloudron/ssh/authorized_keys', cloudronScope, routes.ssh.getAuthorizedKeys);
     router.put ('/api/v1/cloudron/ssh/authorized_keys', cloudronScope, routes.ssh.addAuthorizedKey);
     router.get ('/api/v1/cloudron/ssh/authorized_keys/:identifier', cloudronScope, routes.ssh.getAuthorizedKey);
     router.del ('/api/v1/cloudron/ssh/authorized_keys/:identifier', cloudronScope, routes.ssh.delAuthorizedKey);
     router.get ('/api/v1/cloudron/eventlog', cloudronScope, routes.eventlog.get);
 
+    // config route (for dashboard)
+    router.get ('/api/v1/config', profileScope, routes.cloudron.getConfig);
+
     // working off the user behind the provided token
-    router.get ('/api/v1/user/apps', profileScope, routes.apps.getAllByUser);
-    router.get ('/api/v1/user/cloudron_config', profileScope, routes.user.getCloudronConfig);
     router.get ('/api/v1/profile', profileScope, routes.profile.get);
     router.post('/api/v1/profile', profileScope, routes.profile.update);
     router.post('/api/v1/profile/password', profileScope, routes.users.verifyPassword, routes.profile.changePassword);
@@ -139,20 +144,22 @@ function initializeExpressSync() {
     router.post('/api/v1/profile/twofactorauthentication/disable', profileScope, routes.users.verifyPassword, routes.profile.disableTwoFactorAuthentication);
 
     // user routes
-    router.get ('/api/v1/users', usersScope, routes.users.list);
-    router.post('/api/v1/users', usersScope, routes.users.create);
-    router.get ('/api/v1/users/:userId', usersScope, routes.users.get);
-    router.del ('/api/v1/users/:userId', usersScope, routes.users.verifyPassword, routes.users.remove);
-    router.post('/api/v1/users/:userId', usersScope, routes.users.update);
-    router.put ('/api/v1/users/:userId/groups', usersScope, routes.users.setGroups);
-    router.post('/api/v1/users/:userId/invite', usersScope, routes.users.sendInvite);
+    router.get ('/api/v1/users', usersReadScope, routes.users.list);
+    router.post('/api/v1/users', usersManageScope, routes.users.create);
+    router.get ('/api/v1/users/:userId', usersManageScope, routes.users.get); // this is manage scope because it returns non-restricted fields
+    router.del ('/api/v1/users/:userId', usersManageScope, routes.users.verifyPassword, routes.users.remove);
+    router.post('/api/v1/users/:userId', usersManageScope, routes.users.update);
+    router.put ('/api/v1/users/:userId/groups', usersManageScope, routes.users.setGroups);
+    router.post('/api/v1/users/:userId/invite', usersManageScope, routes.users.sendInvite);
+    router.post('/api/v1/users/:userId/transfer', usersManageScope, routes.users.transferOwnership);
 
     // Group management
-    router.get ('/api/v1/groups', usersScope, routes.groups.list);
-    router.post('/api/v1/groups', usersScope, routes.groups.create);
-    router.get ('/api/v1/groups/:groupId', usersScope, routes.groups.get);
-    router.put ('/api/v1/groups/:groupId/members', usersScope, routes.groups.updateMembers);
-    router.del ('/api/v1/groups/:groupId', usersScope, routes.users.verifyPassword, routes.groups.remove);
+    router.get ('/api/v1/groups', usersReadScope, routes.groups.list);
+    router.post('/api/v1/groups', usersManageScope, routes.groups.create);
+    router.get ('/api/v1/groups/:groupId', usersManageScope, routes.groups.get);
+    router.put ('/api/v1/groups/:groupId/members', usersManageScope, routes.groups.updateMembers);
+    router.post('/api/v1/groups/:groupId', usersManageScope, routes.groups.update);
+    router.del ('/api/v1/groups/:groupId', usersManageScope, routes.users.verifyPassword, routes.groups.remove);
 
     // form based login routes used by oauth2 frame
     router.get ('/api/v1/session/login', csrf, routes.oauth2.loginForm);
@@ -183,27 +190,28 @@ function initializeExpressSync() {
     router.del ('/api/v1/clients/:clientId/tokens/:tokenId', clientsScope, routes.clients.delToken);
 
     // app routes
-    router.get ('/api/v1/apps',          appsScope, routes.apps.getApps);
-    router.get ('/api/v1/apps/:id',      appsScope, routes.apps.getApp);
+    router.get ('/api/v1/apps',          appsReadScope, routes.apps.getApps);
+    router.get ('/api/v1/apps/:id',      appsManageScope, routes.apps.getApp);
     router.get ('/api/v1/apps/:id/icon', routes.apps.getAppIcon);
 
-    router.post('/api/v1/apps/install',       appsScope, routes.apps.installApp);
-    router.post('/api/v1/apps/:id/uninstall', appsScope, routes.users.verifyPassword, routes.apps.uninstallApp);
-    router.post('/api/v1/apps/:id/configure', appsScope, routes.apps.configureApp);
-    router.post('/api/v1/apps/:id/update',    appsScope, routes.apps.updateApp);
-    router.post('/api/v1/apps/:id/restore',   appsScope, routes.users.verifyPassword, routes.apps.restoreApp);
-    router.post('/api/v1/apps/:id/backup',    appsScope, routes.apps.backupApp);
-    router.get ('/api/v1/apps/:id/backups',   appsScope, routes.apps.listBackups);
-    router.post('/api/v1/apps/:id/stop',      appsScope, routes.apps.stopApp);
-    router.post('/api/v1/apps/:id/start',     appsScope, routes.apps.startApp);
-    router.get ('/api/v1/apps/:id/logstream', appsScope, routes.apps.getLogStream);
-    router.get ('/api/v1/apps/:id/logs',      appsScope, routes.apps.getLogs);
-    router.get ('/api/v1/apps/:id/exec',      appsScope, routes.apps.exec);
+    router.post('/api/v1/apps/install',       appsManageScope, routes.apps.installApp);
+    router.post('/api/v1/apps/:id/uninstall', appsManageScope, routes.users.verifyPassword, routes.apps.uninstallApp);
+    router.post('/api/v1/apps/:id/configure', appsManageScope, routes.apps.configureApp);
+    router.post('/api/v1/apps/:id/update',    appsManageScope, routes.apps.updateApp);
+    router.post('/api/v1/apps/:id/restore',   appsManageScope, routes.users.verifyPassword, routes.apps.restoreApp);
+    router.post('/api/v1/apps/:id/backup',    appsManageScope, routes.apps.backupApp);
+    router.get ('/api/v1/apps/:id/backups',   appsManageScope, routes.apps.listBackups);
+    router.post('/api/v1/apps/:id/stop',      appsManageScope, routes.apps.stopApp);
+    router.post('/api/v1/apps/:id/start',     appsManageScope, routes.apps.startApp);
+    router.get ('/api/v1/apps/:id/logstream', appsManageScope, routes.apps.getLogStream);
+    router.get ('/api/v1/apps/:id/logs',      appsManageScope, routes.apps.getLogs);
+    router.get ('/api/v1/apps/:id/exec',      appsManageScope, routes.apps.exec);
     // websocket cannot do bearer authentication
-    router.get ('/api/v1/apps/:id/execws',    routes.accesscontrol.websocketAuth.bind(null, [ accesscontrol.SCOPE_APPS ]), routes.apps.execWebSocket);
-    router.post('/api/v1/apps/:id/clone',     appsScope, routes.apps.cloneApp);
-    router.get ('/api/v1/apps/:id/download',  appsScope, routes.apps.downloadFile);
-    router.post('/api/v1/apps/:id/upload',    appsScope, multipart, routes.apps.uploadFile);
+    router.get ('/api/v1/apps/:id/execws',    routes.accesscontrol.websocketAuth.bind(null, [ accesscontrol.SCOPE_APPS_MANAGE ]), routes.accesscontrol.verifyAppOwnership, routes.apps.execWebSocket);
+    router.post('/api/v1/apps/:id/clone',     appsManageScope, routes.apps.cloneApp);
+    router.get ('/api/v1/apps/:id/download',  appsManageScope, routes.apps.downloadFile);
+    router.post('/api/v1/apps/:id/upload',    appsManageScope, multipart, routes.apps.uploadFile);
+    router.post('/api/v1/apps/:id/owner',     appsManageScope, routes.apps.setOwner);
 
     // settings routes (these are for the settings tab - avatar & name have public routes for normal users. see above)
     router.get ('/api/v1/settings/app_autoupdate_pattern', settingsScope, routes.settings.getAppAutoupdatePattern);
@@ -216,15 +224,18 @@ function initializeExpressSync() {
     router.post('/api/v1/settings/cloudron_avatar',    settingsScope, multipart, routes.settings.setCloudronAvatar);
     router.get ('/api/v1/settings/backup_config',      settingsScope, routes.settings.getBackupConfig);
     router.post('/api/v1/settings/backup_config',      settingsScope, routes.settings.setBackupConfig);
+    router.get ('/api/v1/settings/platform_config',      settingsScope, routes.settings.getPlatformConfig);
+    router.post('/api/v1/settings/platform_config',      settingsScope, routes.settings.setPlatformConfig);
+    router.get ('/api/v1/settings/spaces_config',      settingsScope, routes.settings.getSpacesConfig);
+    router.post('/api/v1/settings/spaces_config',      settingsScope, routes.settings.setSpacesConfig);
 
     router.get ('/api/v1/settings/time_zone',          settingsScope, routes.settings.getTimeZone);
     router.post('/api/v1/settings/time_zone',          settingsScope, routes.settings.setTimeZone);
-    router.get ('/api/v1/settings/appstore_config',    settingsScope, routes.settings.getAppstoreConfig);
-    router.post('/api/v1/settings/appstore_config',    settingsScope, routes.settings.setAppstoreConfig);
+    router.get ('/api/v1/settings/appstore_config',    appstoreScope, routes.settings.getAppstoreConfig);
+    router.post('/api/v1/settings/appstore_config',    appstoreScope, routes.settings.setAppstoreConfig);
 
     // email routes
     router.get ('/api/v1/mail/:domain',       mailScope, routes.mail.getDomain);
-    router.post('/api/v1/mail/:domain',       mailScope, routes.mail.updateDomain);
     router.post('/api/v1/mail',               mailScope, routes.mail.addDomain);
     router.get ('/api/v1/mail/:domain/stats', mailScope, routes.users.verifyPassword, routes.mail.getDomainStats);
     router.del ('/api/v1/mail/:domain',       mailScope, routes.users.verifyPassword, routes.mail.removeDomain);
@@ -233,6 +244,7 @@ function initializeExpressSync() {
     router.post('/api/v1/mail/:domain/catch_all',  mailScope, routes.mail.setCatchAllAddress);
     router.post('/api/v1/mail/:domain/relay',         mailScope, routes.mail.setMailRelay);
     router.post('/api/v1/mail/:domain/enable',        mailScope, routes.mail.setMailEnabled);
+    router.post('/api/v1/mail/:domain/dns',        mailScope, routes.mail.setDnsRecords);
     router.post('/api/v1/mail/:domain/send_test_mail',  mailScope, routes.mail.sendTestMail);
     router.get ('/api/v1/mail/:domain/mailboxes',  mailScope, routes.mail.getMailboxes);
     router.get ('/api/v1/mail/:domain/mailboxes/:name',  mailScope, routes.mail.getMailbox);
@@ -249,20 +261,21 @@ function initializeExpressSync() {
     router.del ('/api/v1/mail/:domain/lists/:name', mailScope, routes.mail.removeList);
 
     // feedback
-    router.post('/api/v1/feedback', usersScope, routes.cloudron.feedback);
+    router.post('/api/v1/feedback', cloudronScope, routes.cloudron.feedback);
 
     // backup routes
     router.get ('/api/v1/backups', settingsScope, routes.backups.get);
     router.post('/api/v1/backups', settingsScope, routes.backups.create);
 
     // domain routes
-    router.post('/api/v1/domains', domainsScope, routes.domains.add);
-    router.get ('/api/v1/domains', domainsScope, routes.domains.getAll);
-    router.get ('/api/v1/domains/:domain', domainsScope, routes.domains.get);
-    router.put ('/api/v1/domains/:domain', domainsScope, routes.domains.update);
-    router.del ('/api/v1/domains/:domain', domainsScope, routes.users.verifyPassword, routes.domains.del);
+    router.post('/api/v1/domains', domainsManageScope, routes.domains.add);
+    router.get ('/api/v1/domains', domainsReadScope, routes.domains.getAll);
+    router.get ('/api/v1/domains/:domain', domainsManageScope, routes.domains.get);  // this is manage scope because it returns non-restricted fields
+    router.put ('/api/v1/domains/:domain', domainsManageScope, routes.domains.update);
+    router.del ('/api/v1/domains/:domain', domainsManageScope, routes.users.verifyPassword, routes.domains.del);
 
     // caas routes
+    router.get('/api/v1/caas/config', cloudronScope, routes.caas.getConfig);
     router.post('/api/v1/caas/change_plan', cloudronScope, routes.users.verifyPassword, routes.caas.changePlan);
 
     // disable server socket "idle" timeout. we use the timeout middleware to handle timeouts on a route level
@@ -337,7 +350,7 @@ function start(callback) {
     gSysadminHttpServer = initializeSysadminExpressSync();
 
     async.series([
-        accesscontrol.initialize,  // hooks up authentication strategies into passport
+        routes.accesscontrol.initialize,  // hooks up authentication strategies into passport
         database.initialize,
         cloudron.initialize,
         setup.configureWebadmin,
@@ -355,7 +368,7 @@ function stop(callback) {
     async.series([
         cloudron.uninitialize,
         database.uninitialize,
-        accesscontrol.uninitialize,
+        routes.accesscontrol.uninitialize,
         gHttpServer.close.bind(gHttpServer),
         gSysadminHttpServer.close.bind(gSysadminHttpServer)
     ], function (error) {

src/settings.js

@@ -38,6 +38,9 @@ exports = module.exports = {
     getPlatformConfig: getPlatformConfig,
     setPlatformConfig: setPlatformConfig,
 
+    getSpacesConfig: getSpacesConfig,
+    setSpacesConfig: setSpacesConfig,
+
     getAll: getAll,
 
     // booleans. if you add an entry here, be sure to fix getAll
@@ -50,6 +53,7 @@ exports = module.exports = {
     APPSTORE_CONFIG_KEY: 'appstore_config',
     CAAS_CONFIG_KEY: 'caas_config',
     PLATFORM_CONFIG_KEY: 'platform_config',
+    SPACES_CONFIG_KEY: 'spaces_config',
 
     // strings
     APP_AUTOUPDATE_PATTERN_KEY: 'app_autoupdate_pattern',
@@ -93,6 +97,7 @@ var gDefaults = (function () {
     result[exports.CAAS_CONFIG_KEY] = {};
     result[exports.EMAIL_DIGEST] = true;
     result[exports.PLATFORM_CONFIG_KEY] = {};
+    result[exports.SPACES_CONFIG_KEY] = { enabled: false };
 
     return result;
 })();
@@ -311,8 +316,6 @@ function setBackupConfig(backupConfig, callback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (backupConfig.key && backupConfig.format !== 'tgz') return callback(new SettingsError(SettingsError.BAD_FIELD, 'format does not support encryption'));
-
     backups.testConfig(backupConfig, function (error) {
         if (error && error.reason === BackupsError.BAD_FIELD) return callback(new SettingsError(SettingsError.BAD_FIELD, error.message));
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new SettingsError(SettingsError.EXTERNAL_ERROR, error.message));
@@ -354,6 +357,32 @@ function setEmailDigest(enabled, callback) {
     });
 }
 
+function getSpacesConfig(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settingsdb.get(exports.SPACES_CONFIG_KEY, function (error, value) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, gDefaults[exports.SPACES_CONFIG_KEY]);
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        callback(null, JSON.parse(value));
+    });
+}
+
+function setSpacesConfig(value, callback) {
+    assert.strictEqual(typeof value, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if ('enabled' in value && typeof value.enabled !== 'boolean') return callback(new SettingsError(SettingsError.BAD_FIELD, 'enabled must be a boolean'));
+
+    settingsdb.set(exports.SPACES_CONFIG_KEY, JSON.stringify(value), function (error) {
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        exports.events.emit(exports.SPACES_CONFIG_KEY, value);
+
+        callback(null);
+    });
+}
+
 function getCaasConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -390,6 +419,11 @@ function getPlatformConfig(callback) {
 function setPlatformConfig(platformConfig, callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    for (let addon of [ 'mysql', 'postgresql', 'mail', 'mongodb' ]) {
+        if (!platformConfig[addon]) continue;
+        if (platformConfig[addon].memorySwap < platformConfig[addon].memory) return callback(new SettingsError(SettingsError.BAD_FIELD, 'memorySwap must be larger than memory'));
+    }
+
     settingsdb.set(exports.PLATFORM_CONFIG_KEY, JSON.stringify(platformConfig), function (error) {
         if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
 
@@ -471,7 +505,7 @@ function getAll(callback) {
         result[exports.DYNAMIC_DNS_KEY] = !!result[exports.DYNAMIC_DNS_KEY];
 
         // convert JSON objects
-        [exports.BACKUP_CONFIG_KEY, exports.UPDATE_CONFIG_KEY, exports.APPSTORE_CONFIG_KEY, exports.PLATFORM_CONFIG_KEY ].forEach(function (key) {
+        [exports.BACKUP_CONFIG_KEY, exports.UPDATE_CONFIG_KEY, exports.APPSTORE_CONFIG_KEY, exports.PLATFORM_CONFIG_KEY, exports.SPACES_CONFIG_KEY ].forEach(function (key) {
             result[key] = typeof result[key] === 'object' ? result[key] : safe.JSON.parse(result[key]);
         });
 

src/setup.js

@@ -11,8 +11,7 @@ exports = module.exports = {
     SetupError: SetupError
 };
 
-var accesscontrol = require('./accesscontrol.js'),
-    assert = require('assert'),
+var assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
     BackupsError = require('./backups.js').BackupsError,
@@ -33,11 +32,9 @@ var accesscontrol = require('./accesscontrol.js'),
     semver = require('semver'),
     settingsdb = require('./settingsdb.js'),
     settings = require('./settings.js'),
-    SettingsError = settings.SettingsError,
     shell = require('./shell.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
-    tokendb = require('./tokendb.js'),
     users = require('./users.js'),
     UsersError = users.UsersError,
     tld = require('tldjs'),
@@ -51,7 +48,10 @@ var gWebadminStatus = {
     dns: false,
     tls: false,
     configuring: false,
-    restoring: false
+    restore: {
+        active: false,
+        error: null
+    }
 };
 
 function SetupError(reason, errorOrMessage) {
@@ -166,7 +166,7 @@ function dnsSetup(adminFqdn, domain, zoneName, provider, dnsConfig, tlsConfig, c
 
     if (config.adminDomain()) return callback(new SetupError(SetupError.ALREADY_SETUP));
 
-    if (gWebadminStatus.configuring || gWebadminStatus.restoring) return callback(new SetupError(SetupError.BAD_STATE, 'Already restoring or configuring'));
+    if (gWebadminStatus.configuring || gWebadminStatus.restore.active) return callback(new SetupError(SetupError.BAD_STATE, 'Already restoring or configuring'));
 
     if (!tld.isValid(adminFqdn) || !adminFqdn.endsWith(domain)) return callback(new SetupError(SetupError.BAD_FIELD, 'adminFqdn must be a subdomain of domain'));
 
@@ -176,6 +176,7 @@ function dnsSetup(adminFqdn, domain, zoneName, provider, dnsConfig, tlsConfig, c
 
     function done(error) {
         if (error && error.reason === DomainsError.BAD_FIELD) return callback(new SetupError(SetupError.BAD_FIELD, error.message));
+        if (error && error.reason === DomainsError.ALREADY_EXISTS) return callback(new SetupError(SetupError.BAD_FIELD, error.message));
         if (error) return callback(new SetupError(SetupError.INTERNAL_ERROR, error));
 
         config.setAdminDomain(domain); // set fqdn only after dns config is valid, otherwise cannot re-setup if we failed
@@ -192,9 +193,9 @@ function dnsSetup(adminFqdn, domain, zoneName, provider, dnsConfig, tlsConfig, c
     }
 
     domains.get(domain, function (error, result) {
-        if (error && error.reason !== DomainsError.NOT_FOUND) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+        if (error && error.reason !== DomainsError.NOT_FOUND) return callback(new SetupError(SetupError.INTERNAL_ERROR, error));
 
-        if (result) return callback(new SettingsError(SettingsError.ALREADY_EXISTS, 'domain already exists'));
+        if (result) return callback(new SetupError(SetupError.BAD_STATE, 'Domain already exists'));
 
         async.series([
             domains.add.bind(null, domain, zoneName, provider, dnsConfig, null /* cert */, tlsConfig),
@@ -246,22 +247,18 @@ function activate(username, password, email, displayName, ip, auditSource, callb
         if (error && error.reason === UsersError.BAD_FIELD) return callback(new SetupError(SetupError.BAD_FIELD, error.message));
         if (error) return callback(new SetupError(SetupError.INTERNAL_ERROR, error));
 
-        clients.get('cid-webadmin', function (error, result) {
+        clients.addTokenByUserId('cid-webadmin', userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, function (error, result) {
             if (error) return callback(new SetupError(SetupError.INTERNAL_ERROR, error));
 
-            // Also generate a token so the admin creation can also act as a login
-            var token = tokendb.generateToken();
-            var expires = Date.now() + constants.DEFAULT_TOKEN_EXPIRATION;
+            eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
 
-            tokendb.add(token, userObject.id, result.id, expires, result.scope, function (error) {
-                if (error) return callback(new SetupError(SetupError.INTERNAL_ERROR, error));
-
-                eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
-
-                callback(null, { token: token, expires: expires });
-
-                setTimeout(cloudron.onActivated, 3000); // hack for now to not block the above http response
+            callback(null, {
+                userId: userObject.id,
+                token: result.accessToken,
+                expires: result.expires
             });
+
+            setTimeout(cloudron.onActivated, 3000); // hack for now to not block the above http response
         });
     });
 }
@@ -275,7 +272,7 @@ function restore(backupConfig, backupId, version, callback) {
     if (!semver.valid(version)) return callback(new SetupError(SetupError.BAD_STATE, 'version is not a valid semver'));
     if (semver.major(config.version()) !== semver.major(version) || semver.minor(config.version()) !== semver.minor(version)) return callback(new SetupError(SetupError.BAD_STATE, `Run cloudron-setup with --version ${version} to restore from this backup`));
 
-    if (gWebadminStatus.configuring || gWebadminStatus.restoring) return callback(new SetupError(SetupError.BAD_STATE, 'Already restoring or configuring'));
+    if (gWebadminStatus.configuring || gWebadminStatus.restore.active) return callback(new SetupError(SetupError.BAD_STATE, 'Already restoring or configuring'));
 
     users.count(function (error, count) {
         if (error) return callback(new SetupError(SetupError.INTERNAL_ERROR, error));
@@ -288,7 +285,8 @@ function restore(backupConfig, backupId, version, callback) {
 
             debug(`restore: restoring from ${backupId} from provider ${backupConfig.provider} with format ${backupConfig.format}`);
 
-            gWebadminStatus.restoring = true;
+            gWebadminStatus.restore.active = true;
+            gWebadminStatus.restore.error = null;
 
             callback(null); // do no block
 
@@ -297,11 +295,12 @@ function restore(backupConfig, backupId, version, callback) {
                 autoprovision,
                 // currently, our suggested restore flow is after a dnsSetup. This re-creates DKIM keys and updates the DNS
                 // for this reason, we have to re-setup DNS after a restore. Once we have a 100% IP based restore, we can skip this
-                mail.addDnsRecords.bind(null, config.adminDomain()),
+                mail.setDnsRecords.bind(null, config.adminDomain()),
                 shell.sudo.bind(null, 'restart', [ RESTART_CMD ])
             ], function (error) {
                 debug('restore:', error);
-                gWebadminStatus.restoring = false;
+                if (error) gWebadminStatus.restore.error = error.message;
+                gWebadminStatus.restore.active = false;
             });
         });
     });

src/storage/filesystem.js

@@ -3,15 +3,14 @@
 exports = module.exports = {
     upload: upload,
     download: download,
-    downloadDir: downloadDir,
 
     copy: copy,
 
+    listDir: listDir,
+
     remove: remove,
     removeDir: removeDir,
 
-    backupDone: backupDone,
-
     testConfig: testConfig
 };
 
@@ -22,6 +21,7 @@ var assert = require('assert'),
     fs = require('fs'),
     mkdirp = require('mkdirp'),
     path = require('path'),
+    readdirp = require('readdirp'),
     safe = require('safetydance'),
     shell = require('../shell.js');
 
@@ -68,30 +68,42 @@ function download(apiConfig, sourceFilePath, callback) {
     assert.strictEqual(typeof sourceFilePath, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug('download: %s', sourceFilePath);
+    debug(`download: ${sourceFilePath}`);
 
-    if (!safe.fs.existsSync(sourceFilePath)) return callback(new BackupsError(BackupsError.NOT_FOUND, 'File not found'));
+    if (!safe.fs.existsSync(sourceFilePath)) return callback(new BackupsError(BackupsError.NOT_FOUND, `File not found: ${sourceFilePath}`));
 
     var fileStream = fs.createReadStream(sourceFilePath);
     callback(null, fileStream);
 }
 
-function downloadDir(apiConfig, backupFilePath, destDir) {
+function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupFilePath, 'string');
-    assert.strictEqual(typeof destDir, 'string');
+    assert.strictEqual(typeof dir, 'string');
+    assert.strictEqual(typeof batchSize, 'number');
+    assert.strictEqual(typeof iteratorCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
 
-    var events = new EventEmitter();
+    var entries = [];
+    var entryStream = readdirp({ root: dir, entryType: 'files', lstat: true });
+    entryStream.on('data', function (data) {
+        if (data.stat.isSymbolicLink()) return;
 
-    events.emit('progress', `downloadDir: ${backupFilePath} to ${destDir}`);
+        entries.push({ fullPath: data.fullPath });
+        if (entries.length < batchSize) return;
+        entryStream.pause();
+        iteratorCallback(entries, function (error) {
+            if (error) return callback(error);
 
-    shell.exec('downloadDir', '/bin/cp', [ '-r', backupFilePath + '/.', destDir ], { }, function (error) {
-        if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
-
-        events.emit('done', null);
+            entries = [];
+            entryStream.resume();
+        });
+    });
+    entryStream.on('warn', function (error) {
+        debug('listDir: warning ', error);
+    });
+    entryStream.on('end', function () {
+        iteratorCallback(entries, callback);
     });
-
-    return events;
 }
 
 function copy(apiConfig, oldFilePath, newFilePath) {
@@ -141,7 +153,7 @@ function removeDir(apiConfig, pathPrefix) {
 
     var events = new EventEmitter();
 
-    events.emit('progress', `downloadDir: ${pathPrefix}`);
+    events.emit('progress', `removeDir: ${pathPrefix}`);
 
     shell.exec('removeDir', '/bin/rm', [ '-rf', pathPrefix ], { }, function (error) {
         if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
@@ -162,6 +174,8 @@ function testConfig(apiConfig, callback) {
 
     if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') return callback(new BackupsError(BackupsError.BAD_FIELD, 'noHardlinks must be boolean'));
 
+    if ('externalDisk' in apiConfig && typeof apiConfig.externalDisk !== 'boolean') return callback(new BackupsError(BackupsError.BAD_FIELD, 'externalDisk must be boolean'));
+
     fs.stat(apiConfig.backupFolder, function (error, result) {
         if (error) return callback(new BackupsError(BackupsError.BAD_FIELD, 'Directory does not exist or cannot be accessed: ' + error.message));
         if (!result.isDirectory()) return callback(new BackupsError(BackupsError.BAD_FIELD, 'Backup location is not a directory'));
@@ -175,11 +189,3 @@ function testConfig(apiConfig, callback) {
     });
 }
 
-function backupDone(apiConfig, backupId, appBackupIds, callback) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupId, 'string');
-    assert(Array.isArray(appBackupIds));
-    assert.strictEqual(typeof callback, 'function');
-
-    callback();
-}

src/storage/gcs.js

@@ -3,14 +3,13 @@
 exports = module.exports = {
     upload: upload,
     download: download,
-    downloadDir: downloadDir,
     copy: copy,
 
+    listDir: listDir,
+
     remove: remove,
     removeDir: removeDir,
 
-    backupDone: backupDone,
-
     testConfig: testConfig,
 
     // Used to mock GCS
@@ -23,9 +22,7 @@ var assert = require('assert'),
     BackupsError = require('../backups.js').BackupsError,
     debug = require('debug')('box:storage/gcs'),
     EventEmitter = require('events'),
-    fs = require('fs'),
     GCS = require('@google-cloud/storage'),
-    mkdirp = require('mkdirp'),
     PassThrough = require('stream').PassThrough,
     path = require('path');
 
@@ -117,24 +114,17 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac
 
     async.forever(function listAndDownload(foreverCallback) {
         bucket.getFiles(query, function (error, files, nextQuery) {
-            if (error) {
-                debug('remove: Failed to list %s. Not fatal.', error);
-                return foreverCallback(error);
-            }
+            if (error) return foreverCallback(error);
 
             if (files.length === 0) return foreverCallback(new Error('Done'));
 
-            debug('emitting ' + files.length + ' files found: ' + files.map(function(f) { return f.name; }).join(','));
-            iteratorCallback(files, function (error) {
-                if (error) {
-                    debug(`listDir page handled unsuccessfully ${error}`);
-                    return foreverCallback(error);
-                }
-
+            const entries = files.map(function (f) { return { fullPath: f.name }; });
+            iteratorCallback(entries, function (error) {
+                if (error) return foreverCallback(error);
                 if (!nextQuery) return foreverCallback(new Error('Done'));
 
                 query = nextQuery;
-                debug(`listDir next page token ${query.pageToken}`);
+
                 foreverCallback();
             });
         });
@@ -145,53 +135,6 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac
     });
 }
 
-function downloadDir(apiConfig, backupFilePath, destDir) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupFilePath, 'string');
-    assert.strictEqual(typeof destDir, 'string');
-
-    var events = new EventEmitter();
-    var total = 0;
-
-    function downloadFile(file, iteratorCallback) {
-        var relativePath = path.relative(backupFilePath, file.name);
-
-        events.emit('progress', `Downloading ${relativePath}`);
-
-        mkdirp(path.dirname(path.join(destDir, relativePath)), function (error) {
-            if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
-
-            download(apiConfig, file.name, function (error, sourceStream) {
-                if (error) return iteratorCallback(error);
-
-                var destStream = fs.createWriteStream(path.join(destDir, relativePath));
-
-                destStream.on('open', function () {
-                    sourceStream.pipe(destStream);
-                });
-
-                destStream.on('error', function (error) {
-                    iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
-                });
-
-                destStream.on('finish', iteratorCallback);
-            });
-        });
-    }
-
-    const concurrency = 10, batchSize = -1;
-
-    listDir(apiConfig, backupFilePath, batchSize, function (files, done) {
-        total += files.length;
-        async.eachLimit(files, concurrency, downloadFile, done);
-    }, function (error) {
-        events.emit('progress', `Downloaded ${total} files`);
-        events.emit('done', error);
-    });
-
-    return events;
-}
-
 function copy(apiConfig, oldFilePath, newFilePath) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof oldFilePath, 'string');
@@ -199,16 +142,15 @@ function copy(apiConfig, oldFilePath, newFilePath) {
 
     var events = new EventEmitter(), retryCount = 0;
 
-    function copyFile(file, iteratorCallback) {
+    function copyFile(entry, iteratorCallback) {
+        var relativePath = path.relative(oldFilePath, entry.fullPath);
 
-        var relativePath = path.relative(oldFilePath, file.name);
+        getBucket(apiConfig).file(entry.fullPath).copy(path.join(newFilePath, relativePath), function(error) {
+            if (error) debug('copyBackup: gcs copy error', error);
 
-        file.copy(path.join(newFilePath, relativePath), function(error) {
             if (error && error.code === 404) return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, 'Old backup not found'));
-            if (error) {
-                debug('copyBackup: gcs copy error', error);
-                return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
-            }
+            if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+
             iteratorCallback(null);
         });
 
@@ -218,17 +160,16 @@ function copy(apiConfig, oldFilePath, newFilePath) {
     const batchSize = -1;
     var total = 0, concurrency = 4;
 
-    listDir(apiConfig, oldFilePath, batchSize, function (files, done) {
-        total += files.length;
+    listDir(apiConfig, oldFilePath, batchSize, function (entries, done) {
+        total += entries.length;
 
         if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
         events.emit('progress', `${retryCount} errors. concurrency set to ${concurrency}`);
         retryCount = 0;
 
-        async.eachLimit(files, concurrency, copyFile, done);
+        async.eachLimit(entries, concurrency, copyFile, done);
     }, function (error) {
         events.emit('progress', `Copied ${total} files`);
-
         events.emit('done', error);
     });
 
@@ -242,9 +183,9 @@ function remove(apiConfig, filename, callback) {
 
     getBucket(apiConfig)
         .file(filename)
-        .delete(function(e) {
-            if (e) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, e.message);
-            else debug('removeBackups: Deleted: %s', filename);
+        .delete(function (error) {
+            if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, error.message);
+
             callback(null);
         });
 }
@@ -258,14 +199,16 @@ function removeDir(apiConfig, pathPrefix) {
     const batchSize = 1;
     var total = 0, concurrency = 4;
 
-    listDir(apiConfig, pathPrefix, batchSize, function (files, done) {
-        total += files.length;
+    listDir(apiConfig, pathPrefix, batchSize, function (entries, done) {
+        total += entries.length;
 
         if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
         events.emit('progress', `${retryCount} errors. concurrency set to ${concurrency}`);
         retryCount = 0;
 
-        async.eachLimit(files, concurrency, remove.bind(null, apiConfig), done);
+        async.eachLimit(entries, concurrency, function (entry, iteratorCallback) {
+            remove(apiConfig, entry.fullPath, iteratorCallback);
+        }, done);
     }, function (error) {
         events.emit('progress', `Deleted ${total} files`);
 
@@ -315,11 +258,3 @@ function testConfig(apiConfig, callback) {
     });
 }
 
-function backupDone(apiConfig, backupId, appBackupIds, callback) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupId, 'string');
-    assert(Array.isArray(appBackupIds));
-    assert.strictEqual(typeof callback, 'function');
-
-    callback();
-}

src/storage/interface.js

@@ -17,11 +17,11 @@ exports = module.exports = {
     downloadDir: downloadDir,
     copy: copy,
 
+    listDir: listDir,
+
     remove: remove,
     removeDir: removeDir,
 
-    backupDone: backupDone,
-
     testConfig: testConfig
 };
 
@@ -69,6 +69,16 @@ function copy(apiConfig, oldFilePath, newFilePath) {
     return events;
 }
 
+function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof dir, 'string');
+    assert.strictEqual(typeof batchSize, 'number');
+    assert.strictEqual(typeof iteratorCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback(new Error('not implemented'));
+}
+
 function remove(apiConfig, filename, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof filename, 'string');
@@ -98,11 +108,3 @@ function testConfig(apiConfig, callback) {
     callback(new Error('not implemented'));
 }
 
-function backupDone(apiConfig, backupId, appBackupIds, callback) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupId, 'string');
-    assert(Array.isArray(appBackupIds));
-    assert.strictEqual(typeof callback, 'function');
-
-    callback(new Error('not implemented'));
-}

src/storage/noop.js

@@ -6,11 +6,11 @@ exports = module.exports = {
     downloadDir: downloadDir,
     copy: copy,
 
+    listDir: listDir,
+
     remove: remove,
     removeDir: removeDir,
 
-    backupDone: backupDone,
-
     testConfig: testConfig
 };
 
@@ -39,6 +39,16 @@ function download(apiConfig, backupFilePath, callback) {
     callback(new Error('Cannot download from noop backend'));
 }
 
+function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof dir, 'string');
+    assert.strictEqual(typeof batchSize, 'number');
+    assert.strictEqual(typeof iteratorCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback();
+}
+
 function downloadDir(apiConfig, backupFilePath, destDir) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');
@@ -93,11 +103,3 @@ function testConfig(apiConfig, callback) {
     callback(null);
 }
 
-function backupDone(apiConfig, backupId, appBackupIds, callback) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupId, 'string');
-    assert(Array.isArray(appBackupIds));
-    assert.strictEqual(typeof callback, 'function');
-
-    callback(null);
-}

src/storage/s3.js

@@ -3,14 +3,13 @@
 exports = module.exports = {
     upload: upload,
     download: download,
-    downloadDir: downloadDir,
     copy: copy,
 
+    listDir: listDir,
+
     remove: remove,
     removeDir: removeDir,
 
-    backupDone: backupDone,
-
     testConfig: testConfig,
 
     // Used to mock AWS
@@ -26,9 +25,7 @@ var assert = require('assert'),
     config = require('../config.js'),
     debug = require('debug')('box:storage/s3'),
     EventEmitter = require('events'),
-    fs = require('fs'),
     https = require('https'),
-    mkdirp = require('mkdirp'),
     PassThrough = require('stream').PassThrough,
     path = require('path'),
     S3BlockReadStream = require('s3-block-read-stream'),
@@ -47,6 +44,10 @@ function mockRestore() {
 
 var gCachedCaasCredentials = { issueDate: null, credentials: null };
 
+function S3_NOT_FOUND(error) {
+    return error.code === 'NoSuchKey' || error.code === 'NotFound' || error.code === 'ENOENT';
+}
+
 function getCaasConfig(apiConfig, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -173,8 +174,8 @@ function download(apiConfig, backupFilePath, callback) {
         var multipartDownload = new S3BlockReadStream(s3, params, { blockSize: 64 * 1024 * 1024 /*, logCallback: debug */ });
 
         multipartDownload.on('error', function (error) {
-            if (error.code === 'NoSuchKey' || error.code === 'ENOENT') {
-                ps.emit('error', new BackupsError(BackupsError.NOT_FOUND));
+            if (S3_NOT_FOUND(error)) {
+                ps.emit('error', new BackupsError(BackupsError.NOT_FOUND, `Backup not found: ${backupFilePath}`));
             } else {
                 debug(`download: ${apiConfig.bucket}:${backupFilePath} s3 stream error.`, error);
                 ps.emit('error', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message || error.code)); // DO sets 'code'
@@ -187,26 +188,32 @@ function download(apiConfig, backupFilePath, callback) {
     });
 }
 
-function listDir(apiConfig, dir, iteratorCallback, callback) {
+function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof dir, 'string');
+    assert.strictEqual(typeof batchSize, 'number');
+    assert.strictEqual(typeof iteratorCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
     getS3Config(apiConfig, function (error, credentials) {
         if (error) return callback(error);
 
         var s3 = new AWS.S3(credentials);
         var listParams = {
             Bucket: apiConfig.bucket,
-            Prefix: dir
+            Prefix: dir,
+            MaxKeys: batchSize
         };
 
         async.forever(function listAndDownload(foreverCallback) {
             s3.listObjects(listParams, function (error, listData) {
-                if (error) {
-                    debug('remove: Failed to list %s. Not fatal.', error);
-                    return foreverCallback(error);
-                }
+                if (error) return foreverCallback(error);
 
                 if (listData.Contents.length === 0) return foreverCallback(new Error('Done'));
 
-                iteratorCallback(s3, listData.Contents, function (error) {
+                const entries = listData.Contents.map(function (c) { return { fullPath: c.Key, size: c.Size }; });
+
+                iteratorCallback(entries, function (error) {
                     if (error) return foreverCallback(error);
 
                     if (!listData.IsTruncated) return foreverCallback(new Error('Done'));
@@ -224,54 +231,6 @@ function listDir(apiConfig, dir, iteratorCallback, callback) {
     });
 }
 
-function downloadDir(apiConfig, backupFilePath, destDir) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupFilePath, 'string');
-    assert.strictEqual(typeof destDir, 'string');
-
-    var events = new EventEmitter();
-    var total = 0;
-
-    function downloadFile(s3, content, iteratorCallback) {
-        var relativePath = path.relative(backupFilePath, content.Key);
-
-        events.emit('progress', `Downloading ${relativePath}`);
-
-        mkdirp(path.dirname(path.join(destDir, relativePath)), function (error) {
-            if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
-
-            download(apiConfig, content.Key, function (error, sourceStream) {
-                if (error) return iteratorCallback(error);
-
-                var destStream = fs.createWriteStream(path.join(destDir, relativePath));
-
-                destStream.on('open', function () {
-                    sourceStream.pipe(destStream);
-                });
-
-                destStream.on('error', function (error) {
-                    return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
-                });
-
-                destStream.on('finish', iteratorCallback);
-            });
-        });
-    }
-
-    const concurrency = 10;
-
-    listDir(apiConfig, backupFilePath, function (s3, objects, done) {
-        total += objects.length;
-        async.eachLimit(objects, concurrency, downloadFile.bind(null, s3), done);
-    }, function (error) {
-        events.emit('progress', `Downloaded ${total} files`);
-
-        events.emit('done', error);
-    });
-
-    return events;
-}
-
 // https://github.com/aws/aws-sdk-js/blob/2b6bcbdec1f274fe931640c1b61ece999aae7a19/lib/util.js#L41
 // https://github.com/GeorgePhillips/node-s3-url-encode/blob/master/index.js
 // See aws-sdk-js/issues/1302
@@ -294,107 +253,112 @@ function copy(apiConfig, oldFilePath, newFilePath) {
 
     var events = new EventEmitter(), retryCount = 0;
 
-    function copyFile(s3, content, iteratorCallback) {
-        var relativePath = path.relative(oldFilePath, content.Key);
+    function copyFile(entry, iteratorCallback) {
+        getS3Config(apiConfig, function (error, credentials) {
+            if (error) return iteratorCallback(error);
 
-        function done(error) {
-            if (error) debug(`copy: s3 copy error when copying ${content.Key}: ${error}`);
+            var s3 = new AWS.S3(credentials);
+            var relativePath = path.relative(oldFilePath, entry.fullPath);
 
-            if (error && error.code === 'NoSuchKey') return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, `Old backup not found: ${content.Key}`));
-            if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error copying ${content.Key} : ${error.code} ${error}`));
+            function done(error) {
+                if (error) debug(`copy: s3 copy error when copying ${entry.fullPath}: ${error}`);
 
-            iteratorCallback(null);
-        }
+                if (error && S3_NOT_FOUND(error)) return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, `Old backup not found: ${entry.fullPath}`));
+                if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error copying ${entry.fullPath} : ${error.code} ${error}`));
 
-        var copyParams = {
-            Bucket: apiConfig.bucket,
-            Key: path.join(newFilePath, relativePath)
-        };
+                iteratorCallback(null);
+            }
 
-        // S3 copyObject has a file size limit of 5GB so if we have larger files, we do a multipart copy
-        // Exoscale takes too long to copy 5GB
-        const largeFileLimit = apiConfig.provider === 'exoscale-sos' ? 1024 * 1024 * 1024 : 5 * 1024 * 1024 * 1024;
+            var copyParams = {
+                Bucket: apiConfig.bucket,
+                Key: path.join(newFilePath, relativePath)
+            };
 
-        if (content.Size < largeFileLimit) {
-            events.emit('progress', `Copying ${relativePath || oldFilePath}`);
+            // S3 copyObject has a file size limit of 5GB so if we have larger files, we do a multipart copy
+            // Exoscale takes too long to copy 5GB
+            const largeFileLimit = apiConfig.provider === 'exoscale-sos' ? 1024 * 1024 * 1024 : 5 * 1024 * 1024 * 1024;
 
-            copyParams.CopySource = encodeCopySource(apiConfig.bucket, content.Key);
-            s3.copyObject(copyParams, done).on('retry', function (response) {
-                ++retryCount;
-                events.emit('progress', `Retrying (${response.retryCount+1}) copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}`);
-                // on DO, we get a random 408. these are not retried by the SDK
-                if (response.error) response.error.retryable = true; // https://github.com/aws/aws-sdk-js/issues/412
-            });
+            if (entry.size < largeFileLimit) {
+                events.emit('progress', `Copying ${relativePath || oldFilePath}`);
 
-            return;
-        }
+                copyParams.CopySource = encodeCopySource(apiConfig.bucket, entry.fullPath);
+                s3.copyObject(copyParams, done).on('retry', function (response) {
+                    ++retryCount;
+                    events.emit('progress', `Retrying (${response.retryCount+1}) copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}`);
+                    // on DO, we get a random 408. these are not retried by the SDK
+                    if (response.error) response.error.retryable = true; // https://github.com/aws/aws-sdk-js/issues/412
+                });
 
-        events.emit('progress', `Copying (multipart) ${relativePath || oldFilePath}`);
+                return;
+            }
 
-        s3.createMultipartUpload(copyParams, function (error, result) {
-            if (error) return done(error);
+            events.emit('progress', `Copying (multipart) ${relativePath || oldFilePath}`);
 
-            // Exoscale (96M) was suggested by exoscale. 1GB - rather random size for others
-            const chunkSize = apiConfig.provider === 'exoscale-sos' ? 96 * 1024 * 1024 : 1024 * 1024 * 1024;
-            var uploadId = result.UploadId;
-            var uploadedParts = [];
-            var partNumber = 1;
-            var startBytes = 0;
-            var endBytes = 0;
-            var size = content.Size-1;
+            s3.createMultipartUpload(copyParams, function (error, result) {
+                if (error) return done(error);
 
-            function copyNextChunk() {
-                endBytes = startBytes + chunkSize;
-                if (endBytes > size) endBytes = size;
+                // Exoscale (96M) was suggested by exoscale. 1GB - rather random size for others
+                const chunkSize = apiConfig.provider === 'exoscale-sos' ? 96 * 1024 * 1024 : 1024 * 1024 * 1024;
+                var uploadId = result.UploadId;
+                var uploadedParts = [];
+                var partNumber = 1;
+                var startBytes = 0;
+                var endBytes = 0;
+                var size = entry.size-1;
 
-                var params = {
-                    Bucket: apiConfig.bucket,
-                    Key: path.join(newFilePath, relativePath),
-                    CopySource: encodeCopySource(apiConfig.bucket, content.Key), // See aws-sdk-js/issues/1302
-                    CopySourceRange: 'bytes=' + startBytes + '-' + endBytes,
-                    PartNumber: partNumber,
-                    UploadId: uploadId
-                };
-
-                s3.uploadPartCopy(params, function (error, result) {
-                    if (error) return done(error);
-
-                    uploadedParts.push({ ETag: result.CopyPartResult.ETag, PartNumber: partNumber });
-
-                    if (endBytes < size) {
-                        startBytes = endBytes + 1;
-                        partNumber++;
-                        return copyNextChunk();
-                    }
+                function copyNextChunk() {
+                    endBytes = startBytes + chunkSize;
+                    if (endBytes > size) endBytes = size;
 
                     var params = {
                         Bucket: apiConfig.bucket,
                         Key: path.join(newFilePath, relativePath),
-                        MultipartUpload: { Parts: uploadedParts },
+                        CopySource: encodeCopySource(apiConfig.bucket, entry.fullPath), // See aws-sdk-js/issues/1302
+                        CopySourceRange: 'bytes=' + startBytes + '-' + endBytes,
+                        PartNumber: partNumber,
                         UploadId: uploadId
                     };
 
-                    s3.completeMultipartUpload(params, done);
-                }).on('retry', function (response) {
-                    ++retryCount;
-                    events.emit('progress', `Retrying (${response.retryCount+1}) multipart copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}`);
-                });
-            }
+                    s3.uploadPartCopy(params, function (error, result) {
+                        if (error) return done(error);
 
-            copyNextChunk();
+                        uploadedParts.push({ ETag: result.CopyPartResult.ETag, PartNumber: partNumber });
+
+                        if (endBytes < size) {
+                            startBytes = endBytes + 1;
+                            partNumber++;
+                            return copyNextChunk();
+                        }
+
+                        var params = {
+                            Bucket: apiConfig.bucket,
+                            Key: path.join(newFilePath, relativePath),
+                            MultipartUpload: { Parts: uploadedParts },
+                            UploadId: uploadId
+                        };
+
+                        s3.completeMultipartUpload(params, done);
+                    }).on('retry', function (response) {
+                        ++retryCount;
+                        events.emit('progress', `Retrying (${response.retryCount+1}) multipart copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}`);
+                    });
+                }
+
+                copyNextChunk();
+            });
         });
     }
 
     var total = 0, concurrency = 4;
 
-    listDir(apiConfig, oldFilePath, function listDirIterator(s3, objects, done) {
-        total += objects.length;
+    listDir(apiConfig, oldFilePath, 1000, function listDirIterator(entries, done) {
+        total += entries.length;
 
         if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
-        events.emit('progress', `Copying ${total-objects.length}-${total}. ${retryCount} errors so far. concurrency set to ${concurrency}`);
+        events.emit('progress', `Copying ${total-entries.length}-${total}. ${retryCount} errors so far. concurrency set to ${concurrency}`);
         retryCount = 0;
 
-        async.eachLimit(objects, concurrency, copyFile.bind(null, s3), done);
+        async.eachLimit(entries, concurrency, copyFile, done);
     }, function (error) {
         events.emit('progress', `Copied ${total} files with error: ${error}`);
 
@@ -437,33 +401,39 @@ function removeDir(apiConfig, pathPrefix) {
     var events = new EventEmitter();
     var total = 0;
 
-    listDir(apiConfig, pathPrefix, function listDirIterator(s3, objects, done) {
-        total += objects.length;
+    getS3Config(apiConfig, function (error, credentials) {
+        if (error) events.emit('done', error);
 
-        const chunkSize = apiConfig.provider !== 'digitalocean-spaces' ? 1000 : 100; // throttle objects in each request
-        var chunks = chunk(objects, chunkSize);
+        var s3 = new AWS.S3(credentials);
 
-        async.eachSeries(chunks, function deleteFiles(contents, iteratorCallback) {
-            var deleteParams = {
-                Bucket: apiConfig.bucket,
-                Delete: {
-                    Objects: contents.map(function (c) { return { Key: c.Key }; })
-                }
-            };
+        listDir(apiConfig, pathPrefix, 1000, function listDirIterator(entries, done) {
+            total += entries.length;
 
-            events.emit('progress', `Removing ${contents.length} files from ${contents[0].Key} to ${contents[contents.length-1].Key}`);
+            const chunkSize = apiConfig.provider !== 'digitalocean-spaces' ? 1000 : 100; // throttle objects in each request
+            var chunks = chunk(entries, chunkSize);
 
-            // deleteObjects does not return error if key is not found
-            s3.deleteObjects(deleteParams, function (error /*, deleteData */) {
-                if (error) events.emit('progress', `Unable to remove ${deleteParams.Key} ${error.message}`);
+            async.eachSeries(chunks, function deleteFiles(objects, iteratorCallback) {
+                var deleteParams = {
+                    Bucket: apiConfig.bucket,
+                    Delete: {
+                        Objects: objects.map(function (o) { return { Key: o.fullPath }; })
+                    }
+                };
 
-                iteratorCallback(error);
-            });
-        }, done);
-    }, function (error) {
-        events.emit('progress', `Removed ${total} files`);
+                events.emit('progress', `Removing ${objects.length} files from ${objects[0].fullPath} to ${objects[objects.length-1].fullPath}`);
 
-        events.emit('done', error);
+                // deleteObjects does not return error if key is not found
+                s3.deleteObjects(deleteParams, function (error /*, deleteData */) {
+                    if (error) events.emit('progress', `Unable to remove ${deleteParams.Key} ${error.message}`);
+
+                    iteratorCallback(error);
+                });
+            }, done);
+        }, function (error) {
+            events.emit('progress', `Removed ${total} files`);
+
+            events.emit('done', error);
+        });
     });
 
     return events;
@@ -512,30 +482,3 @@ function testConfig(apiConfig, callback) {
         });
     });
 }
-
-function backupDone(apiConfig, backupId, appBackupIds, callback) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupId, 'string');
-    assert(Array.isArray(appBackupIds));
-    assert.strictEqual(typeof callback, 'function');
-
-    if (apiConfig.provider !== 'caas') return callback();
-
-    debug('[%s] backupDone: %s apps %j', backupId, backupId, appBackupIds);
-
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + apiConfig.fqdn + '/backupDone';
-    var data = {
-        boxVersion: config.version(),
-        backupId: backupId,
-        appId: null,        // now unused
-        appVersion: null,   // now unused
-        appBackupIds: appBackupIds
-    };
-
-    superagent.post(url).send(data).query({ token: apiConfig.token }).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 200) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-
-        return callback(null);
-    });
-}

src/taskmanager.js

@@ -13,11 +13,16 @@ exports = module.exports = {
 };
 
 var appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     child_process = require('child_process'),
     debug = require('debug')('box:taskmanager'),
+    fs = require('fs'),
     locker = require('./locker.js'),
+    mkdirp = require('mkdirp'),
+    path = require('path'),
+    paths = require('./paths.js'),
     sendFailureLogs = require('./logcollector.js').sendFailureLogs,
     util = require('util'),
     _ = require('underscore');
@@ -39,12 +44,11 @@ function resumeTasks(callback) {
 
     gPaused = false;
 
-    appdb.getAll(function (error, apps) {
+    apps.getAll(function (error, result) {
         if (error) return callback(error);
 
-        apps.forEach(function (app) {
+        result.forEach(function (app) {
             if (app.installationState === appdb.ISTATE_INSTALLED && app.runState === appdb.RSTATE_RUNNING) return;
-
             if (app.installationState === appdb.ISTATE_ERROR) return;
 
             debug('Creating process for %s (%s) with state %s', app.fqdn, app.id, app.installationState);
@@ -122,26 +126,39 @@ function startAppTask(appId, callback) {
         return callback();
     }
 
-    // when parent process dies, apptask processes are killed because KillMode=control-group in systemd unit file
-    gActiveTasks[appId] = child_process.fork(__dirname + '/apptask.js', [ appId ]);
+    // ensure log folder
+    mkdirp.sync(path.join(paths.LOG_DIR, appId));
+    var logFilePath = path.join(paths.LOG_DIR, appId, 'apptask.log');
 
-    var pid = gActiveTasks[appId].pid;
-    debug('Started task of %s pid: %s', appId, pid);
-
-    gActiveTasks[appId].once('exit', function (code, signal) {
-        debug('Task for %s pid %s completed with status %s', appId, pid, code);
-        if (code === null /* signal */ || (code !== 0 && code !== 50)) { // apptask crashed
-            debug('Apptask crashed with code %s and signal %s', code, signal);
-            sendFailureLogs('apptask', { unit: 'box' });
-            appdb.update(appId, { installationState: appdb.ISTATE_ERROR, installationProgress: 'Apptask crashed with code ' + code + ' and signal ' + signal }, NOOP_CALLBACK);
-        } else if (code === 50) {
-            sendFailureLogs('apptask', { unit: 'box' });
+    // will autoclose
+    fs.open(logFilePath, 'a', function (error, fd) {
+        if (error) {
+            debug('Unable to open log file, queueing task for %s', appId, error);
+            gPendingTasks.push(appId);
+            return callback();
         }
-        delete gActiveTasks[appId];
-        locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
-    });
 
-    callback();
+        // when parent process dies, apptask processes are killed because KillMode=control-group in systemd unit file
+        gActiveTasks[appId] = child_process.fork(__dirname + '/apptask.js', [ appId ], { stdio: [ 'pipe', fd, fd, 'ipc' ]});
+
+        var pid = gActiveTasks[appId].pid;
+        debug('Started task of %s pid: %s. See logs at %s', appId, pid, logFilePath);
+
+        gActiveTasks[appId].once('exit', function (code, signal) {
+            debug('Task for %s pid %s completed with status %s', appId, pid, code);
+            if (code === null /* signal */ || (code !== 0 && code !== 50)) { // apptask crashed
+                debug('Apptask crashed with code %s and signal %s', code, signal);
+                sendFailureLogs('apptask', { unit: 'box' });
+                appdb.update(appId, { installationState: appdb.ISTATE_ERROR, installationProgress: 'Apptask crashed with code ' + code + ' and signal ' + signal }, NOOP_CALLBACK);
+            } else if (code === 50) {
+                sendFailureLogs('apptask', { unit: 'box' });
+            }
+            delete gActiveTasks[appId];
+            locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
+        });
+
+        callback();
+    });
 }
 
 function stopAppTask(appId, callback) {

src/test/accesscontrol-test.js

@@ -0,0 +1,80 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var accesscontrol = require('../accesscontrol.js'),
+    expect = require('expect.js');
+
+describe('access control', function () {
+    describe('canonicalScopeString', function () {
+        it('only * scope', function () {
+            expect(accesscontrol.canonicalScopeString('*')).to.be(accesscontrol.VALID_SCOPES.join(','));
+        });
+
+        it('identity for non-*', function () {
+            expect(accesscontrol.canonicalScopeString('foo,bar')).to.be('bar,foo'); // becomes sorted
+        });
+    });
+
+    describe('intersectScopes', function () { // args: allowed, wanted
+        it('both are same', function () {
+            expect(accesscontrol.intersectScopes([ 'apps', 'clients' ], [ 'apps', 'clients' ])).to.eql([ 'apps', 'clients' ]);
+        });
+
+        it('some are different', function () {
+            expect(accesscontrol.intersectScopes([ 'apps' ], [ 'apps', 'clients' ])).to.eql(['apps']);
+            expect(accesscontrol.intersectScopes([ 'clients', 'domains', 'mail' ], [ 'mail' ])).to.eql(['mail']);
+        });
+
+        it('everything is different', function () {
+            expect(accesscontrol.intersectScopes(['cloudron', 'domains' ], ['apps','clients'])).to.eql([]);
+        });
+
+        it('subscopes', function () {
+            expect(accesscontrol.intersectScopes(['apps:read' ], ['apps'])).to.eql(['apps:read']);
+            expect(accesscontrol.intersectScopes(['apps:read','domains','profile'], ['apps','domains:manage','profile'])).to.eql(['apps:read','domains:manage','profile']);
+            expect(accesscontrol.intersectScopes(['apps:read','domains','profile'], ['apps','apps:read'])).to.eql(['apps:read']);
+        });
+    });
+
+    describe('validateScopeString', function () {
+        it('allows valid scopes', function () {
+            expect(accesscontrol.validateScopeString('apps')).to.be(null);
+            expect(accesscontrol.validateScopeString('apps,mail')).to.be(null);
+            expect(accesscontrol.validateScopeString('apps:read,mail')).to.be(null);
+            expect(accesscontrol.validateScopeString('apps,mail:write')).to.be(null);
+        });
+
+        it('disallows invalid scopes', function () {
+            expect(accesscontrol.validateScopeString('apps, mail')).to.be.an(Error);
+            expect(accesscontrol.validateScopeString('random')).to.be.an(Error);
+            expect(accesscontrol.validateScopeString('')).to.be.an(Error);
+        });
+    });
+
+    describe('hasScopes', function () {
+        it('succeeds if it contains the scope', function () {
+            expect(accesscontrol.hasScopes([ 'apps' ], [ 'apps' ])).to.be(null);
+            expect(accesscontrol.hasScopes([ 'apps', 'mail' ], [ 'mail' ])).to.be(null);
+            expect(accesscontrol.hasScopes([ 'clients', '*', 'apps', 'mail' ], [ 'mail' ])).to.be(null);
+
+            // subscope
+            expect(accesscontrol.hasScopes([ 'apps' ], [ 'apps:read' ])).to.be(null);
+            expect(accesscontrol.hasScopes([ 'apps:read' ], [ 'apps:read' ])).to.be(null);
+            expect(accesscontrol.hasScopes([ 'apps' , 'mail' ], [ 'apps:*' ])).to.be(null);
+            expect(accesscontrol.hasScopes([ '*' ], [ 'apps:read' ])).to.be(null);
+        });
+
+        it('fails if it does not contain the scope', function () {
+            expect(accesscontrol.hasScopes([ 'apps' ], [ 'mail' ])).to.be.an(Error);
+            expect(accesscontrol.hasScopes([ 'apps', 'mail' ], [ 'clients' ])).to.be.an(Error);
+
+            // subscope
+            expect(accesscontrol.hasScopes([ 'apps:write' ], [ 'apps:read' ])).to.be.an(Error);
+        });
+    });
+});

src/test/apps-test.js

@@ -10,13 +10,12 @@ var appdb = require('../appdb.js'),
     AppsError = apps.AppsError,
     async = require('async'),
     config = require('../config.js'),
-    constants = require('../constants.js'),
     database = require('../database.js'),
     domains = require('../domains.js'),
     expect = require('expect.js'),
     groupdb = require('../groupdb.js'),
     groups = require('../groups.js'),
-    hat = require('hat'),
+    hat = require('../hat.js'),
     settings = require('../settings.js'),
     settingsdb = require('../settingsdb.js'),
     userdb = require('../userdb.js');
@@ -32,7 +31,9 @@ describe('Apps', function () {
         createdAt: 'sometime back',
         modifiedAt: 'now',
         resetToken: hat(256),
-        displayName: ''
+        displayName: '',
+        groupIds: [],
+        admin: true
     };
 
     var USER_0 = {
@@ -45,7 +46,9 @@ describe('Apps', function () {
         createdAt: 'sometime back',
         modifiedAt: 'now',
         resetToken: hat(256),
-        displayName: ''
+        displayName: '',
+        groupIds: [],
+        admin: false
     };
 
     var USER_1 = {
@@ -58,7 +61,9 @@ describe('Apps', function () {
         createdAt: 'sometime back',
         modifiedAt: 'now',
         resetToken: hat(256),
-        displayName: ''
+        displayName: '',
+        groupIds: [ 'somegroup' ],
+        admin: false
     };
 
     var GROUP_0 = {
@@ -104,7 +109,8 @@ describe('Apps', function () {
         accessRestriction: null,
         memoryLimit: 0,
         robotsTxt: null,
-        sso: false
+        sso: false,
+        ownerId: USER_0.id
     };
 
     var APP_1 = {
@@ -118,7 +124,8 @@ describe('Apps', function () {
         },
         portBindings: {},
         accessRestriction: { users: [ 'someuser' ], groups: [ GROUP_0.id ] },
-        memoryLimit: 0
+        memoryLimit: 0,
+        ownerId: USER_0.id
     };
 
     var APP_2 = {
@@ -134,7 +141,8 @@ describe('Apps', function () {
         accessRestriction: { users: [ 'someuser', USER_0.id ], groups: [ GROUP_1.id ] },
         memoryLimit: 0,
         robotsTxt: null,
-        sso: false
+        sso: false,
+        ownerId: USER_0.id
     };
 
     before(function (done) {
@@ -153,11 +161,10 @@ describe('Apps', function () {
             userdb.add.bind(null, USER_1.id, USER_1),
             groupdb.add.bind(null, GROUP_0.id, GROUP_0.name),
             groupdb.add.bind(null, GROUP_1.id, GROUP_1.name),
-            groups.addMember.bind(null, constants.ADMIN_GROUP_ID, ADMIN_0.id),
             groups.addMember.bind(null, GROUP_0.id, USER_1.id),
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.domain, APP_0.portBindings, APP_0),
-            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.domain, APP_1.portBindings, APP_1),
-            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.domain, APP_2.portBindings, APP_2),
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.domain, APP_0.ownerId, APP_0.portBindings, APP_0),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.domain, APP_1.ownerId, APP_1.portBindings, APP_1),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.domain, APP_2.ownerId, APP_2.portBindings, APP_2),
             settingsdb.set.bind(null, settings.BACKUP_CONFIG_KEY, JSON.stringify({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' }))
         ], done);
     });
@@ -175,16 +182,18 @@ describe('Apps', function () {
         });
 
         it('cannot have >63 length subdomains', function () {
-            var s = '';
-            for (var i = 0; i < 64; i++) s += 's';
+            var s = Array(64).fill('s').join('');
             expect(apps._validateHostname(s, 'example.com', s + '.example.com')).to.be.an(Error);
+            expect(apps._validateHostname(`dev.${s}`, 'example.com', `dev.${s}.example.com`)).to.be.an(Error);
         });
 
         it('allows only alphanumerics and hypen', function () {
             expect(apps._validateHostname('#2r', 'example.com', '#2r.example.com')).to.be.an(Error);
             expect(apps._validateHostname('a%b', 'example.com', 'a%b.example.com')).to.be.an(Error);
             expect(apps._validateHostname('ab_', 'example.com', 'ab_.example.com')).to.be.an(Error);
-            expect(apps._validateHostname('a.b', 'example.com', 'a.b.example.com')).to.be.an(Error);
+            expect(apps._validateHostname('ab.', 'example.com', 'ab.example.com')).to.be.an(Error);
+            expect(apps._validateHostname('ab..c', 'example.com', 'ab..c.example.com')).to.be.an(Error);
+            expect(apps._validateHostname('.ab', 'example.com', '.ab.example.com')).to.be.an(Error);
             expect(apps._validateHostname('-ab', 'example.com', '-ab.example.com')).to.be.an(Error);
             expect(apps._validateHostname('ab-', 'example.com', 'ab-.example.com')).to.be.an(Error);
         });
@@ -199,6 +208,8 @@ describe('Apps', function () {
         it('allow valid domains', function () {
             expect(apps._validateHostname('a', 'example.com', 'a.example.com')).to.be(null);
             expect(apps._validateHostname('a0-x', 'example.com', 'a0-x.example.com')).to.be(null);
+            expect(apps._validateHostname('a0.x', 'example.com', 'a0-x.example.com')).to.be(null);
+            expect(apps._validateHostname('a0.x.y', 'example.com', 'a0.x.y.example.com')).to.be(null);
             expect(apps._validateHostname('01', 'example.com', '01.example.com')).to.be(null);
         });
     });
@@ -283,8 +294,11 @@ describe('Apps', function () {
     });
 
     describe('hasAccessTo', function () {
+        const someuser = { id: 'someuser', groupIds: [], admin: false };
+        const adminuser = { id: 'adminuser', groupIds: [ 'groupie' ], admin: true };
+
         it('returns true for unrestricted access', function (done) {
-            apps.hasAccessTo({ accessRestriction: null }, { id: 'someuser' }, function (error, access) {
+            apps.hasAccessTo({ accessRestriction: null }, someuser, function (error, access) {
                 expect(error).to.be(null);
                 expect(access).to.be(true);
                 done();
@@ -292,7 +306,7 @@ describe('Apps', function () {
         });
 
         it('returns true for allowed user', function (done) {
-            apps.hasAccessTo({ accessRestriction: { users: [ 'someuser' ] } }, { id: 'someuser' }, function (error, access) {
+            apps.hasAccessTo({ accessRestriction: { users: [ 'someuser' ] } }, someuser, function (error, access) {
                 expect(error).to.be(null);
                 expect(access).to.be(true);
                 done();
@@ -300,7 +314,7 @@ describe('Apps', function () {
         });
 
         it('returns true for allowed user with multiple allowed', function (done) {
-            apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'someuser', 'anotheruser' ] } }, { id: 'someuser' }, function (error, access) {
+            apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'someuser', 'anotheruser' ] } }, someuser, function (error, access) {
                 expect(error).to.be(null);
                 expect(access).to.be(true);
                 done();
@@ -308,7 +322,7 @@ describe('Apps', function () {
         });
 
         it('returns false for not allowed user', function (done) {
-            apps.hasAccessTo({ accessRestriction: { users: [ 'foo' ] } }, { id: 'someuser' }, function (error, access) {
+            apps.hasAccessTo({ accessRestriction: { users: [ 'foo' ] } }, someuser, function (error, access) {
                 expect(error).to.be(null);
                 expect(access).to.be(false);
                 done();
@@ -316,7 +330,7 @@ describe('Apps', function () {
         });
 
         it('returns false for not allowed user with multiple allowed', function (done) {
-            apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'anotheruser' ] } }, { id: 'someuser' }, function (error, access) {
+            apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'anotheruser' ] } }, someuser, function (error, access) {
                 expect(error).to.be(null);
                 expect(access).to.be(false);
                 done();
@@ -324,7 +338,7 @@ describe('Apps', function () {
         });
 
         it('returns false for no group or user', function (done) {
-            apps.hasAccessTo({ accessRestriction: { users: [ ], groups: [ ] } }, { id: 'someuser' }, function (error, access) {
+            apps.hasAccessTo({ accessRestriction: { users: [ ], groups: [ ] } }, someuser, function (error, access) {
                 expect(error).to.be(null);
                 expect(access).to.be(false);
                 done();
@@ -332,12 +346,20 @@ describe('Apps', function () {
         });
 
         it('returns false for invalid group or user', function (done) {
-            apps.hasAccessTo({ accessRestriction: { users: [ ], groups: [ 'nop' ] } }, { id: 'someuser' }, function (error, access) {
+            apps.hasAccessTo({ accessRestriction: { users: [ ], groups: [ 'nop' ] } }, someuser, function (error, access) {
                 expect(error).to.be(null);
                 expect(access).to.be(false);
                 done();
             });
         });
+
+        it('returns true for admin user', function (done) {
+            apps.hasAccessTo({ accessRestriction: { users: [ ], groups: [ 'nop' ] } }, adminuser, function (error, access) {
+                expect(error).to.be(null);
+                expect(access).to.be(true);
+                done();
+            });
+        });
     });
 
     describe('getAllByUser', function () {