data dir: allow sameness of old and new dir

this makes it easy to migrate to a new volume setup

(cherry picked from commit a32166bc9d)

.gitignore

@@ -1,5 +1,6 @@
 node_modules/
 coverage/
+.nyc_output/
 webadmin/dist/
 installer/src/certs/server.key
 

CHANGES

@@ -2429,3 +2429,76 @@
 * firewall: add retry for xtables lock
 * redis: fix issue where protected mode was enabled with no password
 
+[7.1.2]
+* Fix crash in cloudron-firewall when ports are whitelisted
+* eventlog: add event for certificate cleanup
+* eventlog: log event for mailbox alias update
+* backups: fix incorrect mountpoint check with managed mounts
+
+[7.1.3]
+* Fix security issue where an admin can impersonate an owner
+* block list: can upload up to 2MB
+* dns: fix issue where link local address was picked up for ipv6
+* setup: ufw may not be installed
+* mysql: fix default collation of databases
+
+[7.1.4]
+* wildcard dns: fix handling of ENODATA
+* cloudflare: fix error handling
+* openvpn: ipv6 support
+* dyndns: fix issue where eventlog was getting filled with empty entries
+* mandatory 2fa: Fix typo in 2FA check
+
+[7.2.0]
+* mail: hide log button for non-superadmins
+* firewall: do not add duplicate ldap redirect rules
+* ldap: respond to RootDSE
+* Check if CNAME record exists and remove it if overwrite is set
+* cifs: use credentials file for better password support
+* installer: rework script to fix DNS resolution issues
+* backup cleaner: do not clean if not mounted
+* restore: fix sftp private key perms
+* support: add a separate system user named cloudron-support
+* sshfs: fix bug where sshfs mounts were generated without unbound dependancy
+* cloudron-setup: add --setup-token
+* notifications: add installation event
+* backups: set label of backup and control it's retention
+* wasabi: add new regions (London, Frankfurt, Paris, Toronto)
+* docker: update to 20.10.14
+* Ensure LDAP usernames are always treated lowercase
+* Add a way to make LDAP users local
+* proxyAuth: set X-Remote-User (rfc3875)
+* GoDaddy: there is now a delete API
+* nginx: use ubuntu packages for ubuntu 20.04 and 22.04
+* Ubuntu 22.04 LTS support
+* Add Hetzner DNS
+* cron: add support for extensions (@reboot, @weekly etc)
+* Add profile backgroundImage api
+* exec: rework API to get exit code
+* Add update available filter
+
+[7.2.1]
+* Refactor backup code to use async/await
+* mongodb: fix bug where a small timeout prevented import of large backups
+* Add update available filter
+* exec: rework API to get exit code
+* Add profile backgroundImage api
+* cron: add support for extensions (@reboot, @weekly etc)
+
+[7.2.2]
+* Update cloudron-manifestformat for new scheduler patterns
+* collectd: FQDNLookup causes collectd install to fail
+
+[7.2.3]
+* appstore: allow re-registration on server side delete
+* transfer ownership route is not used anymore
+* graphite: fix issue where disk names with '.' do not render
+* dark mode fixes
+* sendmail: mail from display name
+* Use volumes for app data instead of raw path
+* initial xfs support
+
+[7.2.4]
+* volumes: Ensure long volume names do not overflow the table
+* Move all appstore filter to the left
+* app data: allow sameness of old and new dir

box.js

@@ -46,8 +46,7 @@ async function main() {
     const [error] = await safe(startServers());
     if (error) return exitSync({ error: new Error(`Error starting server: ${JSON.stringify(error)}`), code: 1 });
 
-    // require those here so that logging handler is already setup
-    require('supererror');
+    // require this here so that logging handler is already setup
     const debug = require('debug')('box:box');
 
     process.on('SIGINT', async function () {

migrations/20220331193223-settings-delete-licenseKey.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM settings WHERE name=?', [ 'license_key' ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220401044845-settings-rename-cloudron-token-to-appstore-api-token.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'appstore_api_token', 'cloudron_token' ], callback);
+};
+
+exports.down = function(db, callback) {
+  callback();
+};

migrations/20220401045439-settings-get-appstore-web-token.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const superagent = require('superagent');
+
+exports.up = function(db, callback) {
+    db.all('SELECT value FROM settings WHERE name="api_server_origin"', function (error, results) {
+        if (error || results.length === 0) return callback(error);
+        const apiServerOrigin = results[0].value;
+
+        db.all('SELECT value FROM settings WHERE name="appstore_api_token"', function (error, results) {
+            if (error || results.length === 0) return callback(error);
+            const apiToken = results[0].value;
+
+            console.log(`Getting appstore web token from ${apiServerOrigin}`);
+
+            superagent.post(`${apiServerOrigin}/api/v1/user_token`)
+                .send({})
+                .query({ accessToken: apiToken })
+                .timeout(30 * 1000).end(function (error, response) {
+                    if (error && !error.response) {
+                        console.log('Network error getting web token', error);
+                        return callback();
+                    }
+                    if (response.statusCode !== 201 || !response.body.accessToken) {
+                        console.log(`Bad status getting web token: ${response.status} ${response.text}`);
+                        return callback();
+                    }
+
+                    db.runSql('INSERT settings (name, value) VALUES(?, ?)', [ 'appstore_web_token', response.body.accessToken ], callback);
+                });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220402233643-backups-add-label.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN label VARCHAR(128) DEFAULT ""', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN label', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20220404211215-backups-rename-id-to-remotePath.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const async = require('async'),
+    hat = require('../src/hat.js');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * from backups', function (error, allBackups) {
+        if (error) return callback(error);
+
+        console.log(`Fixing up ${allBackups.length} backup entries`);
+        const idMap = {};
+        allBackups.forEach(b => {
+            b.remotePath = b.id;
+            b.id = `${b.type}_${b.identifier}_v${b.packageVersion}_${hat(256)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying
+            idMap[b.remotePath] = b.id;
+        });
+
+        db.runSql('ALTER TABLE backups ADD COLUMN remotePath VARCHAR(256)', function (error) {
+            if (error) return callback(error);
+
+            db.runSql('ALTER TABLE backups CHANGE COLUMN dependsOn dependsOnJson TEXT', function (error) {
+                if (error) return callback(error);
+
+                async.eachSeries(allBackups, function (backup, iteratorDone) {
+                    const dependsOnPaths = backup.dependsOn ? backup.dependsOn.split(',') : []; // previously, it was paths
+                    let dependsOnIds = [];
+                    dependsOnPaths.forEach(p => { if (idMap[p]) dependsOnIds.push(idMap[p]); });
+
+                    db.runSql('UPDATE backups SET id = ?, remotePath = ?, dependsOnJson = ? WHERE id = ?', [ backup.id, backup.remotePath, JSON.stringify(dependsOnIds), backup.remotePath ], iteratorDone);
+                }, function (error) {
+                    if (error) return callback(error);
+
+                    db.runSql('ALTER TABLE backups MODIFY COLUMN remotePath VARCHAR(256) NOT NULL UNIQUE', callback);
+                });
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN remotePath', function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE backups RENAME COLUMN dependsOnJson to dependsOn', function (error) {
+            if (error) return callback(error);
+
+            callback(error);
+        });
+    });
+};
+

migrations/20220426060528-make-apps-sso-consistent.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM apps', function (error, apps) {
+        if (error) return callback(error);
+
+        async.eachSeries(apps, function (app, iteratorDone) {
+            const manifest = JSON.parse(app.manifestJson);
+            const hasSso = !!manifest.addons['proxyAuth'] || !!manifest.addons['ldap'];
+            if (hasSso || !app.sso) return iteratorDone();
+
+            console.log(`Unsetting sso flag of ${app.id}`);
+            db.runSql('UPDATE apps SET sso=? WHERE id=?', [ 0, app.id ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220505162417-settings-add-console-origin.js

@@ -0,0 +1,20 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM settings WHERE name = ?', [ 'api_server_origin' ], function (error, result) {
+        if (error || result.length === 0) return callback(error);
+
+        let consoleOrigin;
+        switch (result[0].value) {
+        case 'https://api.dev.cloudron.io': consoleOrigin = 'https://console.dev.cloudron.io'; break;
+        case 'https://api.staging.cloudron.io': consoleOrigin = 'https://console.staging.cloudron.io'; break;
+        default: consoleOrigin = 'https://console.cloudron.io'; break;
+        }
+
+        db.runSql('REPLACE INTO settings (name, value) VALUES (?, ?)', [ 'console_server_origin', consoleOrigin ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220514170234-users-add-backgroundImage.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN backgroundImage MEDIUMBLOB', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN backgroundImage', callback);
+};

migrations/20220601004757-apps-add-mailboxDisplayName.js

@@ -0,0 +1,12 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN mailboxDisplayName VARCHAR(128) DEFAULT "" NOT NULL', [], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN mailboxDisplayName', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20220602050517-apps-add-storageVolumeId.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const path = require('path'),
+    safe = require('safetydance'),
+    uuid = require('uuid');
+
+function getMountPoint(dataDir) {
+    const output = safe.child_process.execSync(`df --output=target "${dataDir}" | tail -1`, { encoding: 'utf8' });
+    if (!output) return dataDir;
+    const mountPoint = output.trim();
+    if (mountPoint === '/') return dataDir;
+    return mountPoint;
+}
+
+exports.up = async function(db) {
+    await db.runSql('ALTER TABLE apps ADD storageVolumeId VARCHAR(128), ADD FOREIGN KEY(storageVolumeId) REFERENCES volumes(id)');
+    await db.runSql('ALTER TABLE apps ADD storageVolumePrefix VARCHAR(128)');
+    await db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_storageVolume UNIQUE (storageVolumeId, storageVolumePrefix)');
+
+    const apps = await db.runSql('SELECT * FROM apps WHERE dataDir IS NOT NULL');
+    const allVolumes = await db.runSql('SELECT * FROM volumes');
+
+    for (const app of apps) {
+        console.log(`data-dir (${app.id}): migrating data dir ${app.dataDir}`);
+
+        const mountPoint = getMountPoint(app.dataDir);
+        const prefix = path.relative(mountPoint, app.dataDir);
+
+        console.log(`data-dir (${app.id}): migrating to mountpoint ${mountPoint} and prefix ${prefix}`);
+
+        const volume = allVolumes.find(v => v.hostPath === mountPoint);
+        if (volume) {
+            console.log(`data-dir (${app.id}): using existing volume ${volume.id}`);
+            await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ volume.id, prefix, app.id ]);
+            continue;
+        }
+
+        const id = uuid.v4().replace(/-/g, ''); // to make systemd mount file names more readable
+        const name = `app-${app.id}`;
+        const type = app.dataDir === mountPoint ? 'filesystem' : 'mountpoint';
+
+        console.log(`data-dir (${app.id}): creating new volume ${id}`);
+        await db.runSql('INSERT INTO volumes (id, name, hostPath, mountType, mountOptionsJson) VALUES (?, ?, ?, ?, ?)', [ id, name, mountPoint, type, JSON.stringify({}) ]);
+        await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ id, prefix, app.id ]);
+    }
+
+    await db.runSql('ALTER TABLE apps DROP COLUMN dataDir');
+};
+
+exports.down = async function(/*db*/) {
+};

migrations/schema.sql

@@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS users(
     resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     active BOOLEAN DEFAULT 1,
     avatar MEDIUMBLOB NOT NULL,
+    backgroundImage MEDIUMBLOB,
     loginLocationsJson MEDIUMTEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
 
     INDEX creationTime_index (creationTime),
@@ -85,13 +86,15 @@ CREATE TABLE IF NOT EXISTS apps(
     enableAutomaticUpdate BOOLEAN DEFAULT 1,
     enableMailbox BOOLEAN DEFAULT 1, // whether sendmail addon is enabled
     mailboxName VARCHAR(128), // mailbox of this app
-    mailboxDomain VARCHAR(128), // mailbox domain of this apps
+    mailboxDomain VARCHAR(128), // mailbox domain of this app
+    mailboxDisplayName VARCHAR(128), // mailbox display name
     enableInbox BOOLEAN DEFAULT 0, // whether recvmail addon is enabled
     inboxName VARCHAR(128), // mailbox of this app
-    inboxDomain VARCHAR(128), // mailbox domain of this apps
+    inboxDomain VARCHAR(128), // mailbox domain of this app
     label VARCHAR(128),     // display name
     tagsJson VARCHAR(2048), // array of tags
-    dataDir VARCHAR(256) UNIQUE,
+    storageVolumeId VARCHAR(128),
+    storageVolumePrefix VARCHAR(128),
     taskId INTEGER, // current task
     errorJson TEXT,
     servicesConfigJson TEXT, // app services configuration
@@ -102,6 +105,8 @@ CREATE TABLE IF NOT EXISTS apps(
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
+    FOREIGN KEY(storageVolumeId) REFERENCES volumes(id),
+    UNIQUE (storageVolumeId, storageVolumePrefix),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
@@ -133,12 +138,14 @@ CREATE TABLE IF NOT EXISTS appEnvVars(
 
 CREATE TABLE IF NOT EXISTS backups(
     id VARCHAR(128) NOT NULL,
+    remotePath VARCHAR(256) NOT NULL UNIQUE,
+    label VARCHAR(128) DEFAULT "",
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     packageVersion VARCHAR(128) NOT NULL, /* app version or box version */
     encryptionVersion INTEGER, /* when null, unencrypted backup */
     type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
     identifier VARCHAR(128) NOT NULL, /* 'box' or the app id */
-    dependsOn TEXT, /* comma separate list of objects this backup depends on */
+    dependsOnJson TEXT, /* comma separate list of objects this backup depends on */
     state VARCHAR(16) NOT NULL,
     manifestJson TEXT, /* to validate if the app can be installed in this version of box */
     format VARCHAR(16) DEFAULT "tgz",

package.json

@@ -12,13 +12,13 @@
   },
   "dependencies": {
     "@google-cloud/dns": "^2.2.4",
-    "@google-cloud/storage": "^5.16.1",
+    "@google-cloud/storage": "^5.19.2",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^3.2.2",
-    "aws-sdk": "^2.1053.0",
+    "async": "^3.2.3",
+    "aws-sdk": "^2.1115.0",
     "basic-auth": "^2.0.1",
-    "body-parser": "^1.19.1",
-    "cloudron-manifestformat": "^5.15.0",
+    "body-parser": "^1.20.0",
+    "cloudron-manifestformat": "^5.16.0",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.1.1",
     "connect-timeout": "^1.9.0",
@@ -27,39 +27,32 @@
     "cron": "^1.8.2",
     "db-migrate": "^0.11.13",
     "db-migrate-mysql": "^2.2.0",
-    "debug": "^4.3.3",
-    "delay": "^5.0.0",
+    "debug": "^4.3.4",
     "dockerode": "^3.3.1",
     "ejs": "^3.1.6",
     "ejs-cli": "^2.2.3",
-    "express": "^4.17.2",
+    "express": "^4.17.3",
     "ipaddr.js": "^2.0.1",
     "js-yaml": "^4.1.0",
     "json": "^11.0.0",
     "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.3.1",
+    "ldapjs": "^2.3.2",
     "lodash": "^4.17.21",
-    "lodash.chunk": "^4.2.0",
-    "moment": "^2.29.1",
+    "moment": "^2.29.2",
     "moment-timezone": "^0.5.34",
     "morgan": "^1.10.0",
-    "multiparty": "^4.2.2",
+    "multiparty": "^4.2.3",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.7.2",
+    "nodemailer": "^6.7.3",
     "nodemailer-smtp-transport": "^2.7.4",
-    "once": "^1.4.0",
-    "pretty-bytes": "^5.6.0",
     "progress-stream": "^2.0.0",
-    "proxy-middleware": "^0.15.0",
     "qrcode": "^1.5.0",
     "readdirp": "^3.6.0",
-    "s3-block-read-stream": "^0.5.0",
     "safetydance": "^2.2.0",
-    "semver": "^7.3.5",
+    "semver": "^7.3.7",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^7.0.1",
-    "supererror": "^0.7.2",
+    "superagent": "^7.1.1",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
@@ -67,18 +60,18 @@
     "underscore": "^1.13.2",
     "uuid": "^8.3.2",
     "validator": "^13.7.0",
-    "ws": "^8.4.0",
+    "ws": "^8.5.0",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.4.1",
     "js2xmlparser": "^4.0.2",
-    "mocha": "^9.1.3",
+    "mocha": "^9.2.2",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^13.2.1",
+    "nock": "^13.2.4",
     "node-sass": "^7.0.1",
-    "recursive-readdir": "^2.2.2"
+    "nyc": "^15.1.0"
   },
   "scripts": {
     "test": "./runTests",

runTests

@@ -79,10 +79,15 @@ echo "=> Run database migrations"
 cd "${source_dir}"
 BOX_ENV=test DATABASE_URL=mysql://root:password@${MYSQL_IP}/box node_modules/.bin/db-migrate up
 
-echo "=> Run tests with mocha"
 TESTS=${DEFAULT_TESTS}
 if [[ $# -gt 0 ]]; then
     TESTS="$*"
 fi
 
-BOX_ENV=test ./node_modules/mocha/bin/_mocha --bail --no-timeouts --exit -R spec ${TESTS}
+if [[ -z ${COVERAGE+x} ]]; then
+    echo "=> Run tests with mocha"
+    BOX_ENV=test ./node_modules/.bin/mocha --bail --no-timeouts --exit -R spec ${TESTS}
+else
+    echo "=> Run tests with mocha and coverage"
+    BOX_ENV=test ./node_modules/.bin/nyc --reporter=html  ./node_modules/.bin/mocha --no-timeouts --exit -R spec ${TESTS}
+fi

scripts/cloudron-setup

@@ -26,8 +26,8 @@ readonly GREEN='\033[32m'
 readonly DONE='\033[m'
 
 # verify the system has minimum requirements met
-if [[ "${rootfs_type}" != "ext4" ]]; then
-    echo "Error: Cloudron requires '/' to be ext4" # see #364
+if [[ "${rootfs_type}" != "ext4" && "${rootfs_type}" != "xfs" ]]; then
+    echo "Error: Cloudron requires '/' to be ext4 or xfs" # see #364
     exit 1
 fi
 
@@ -46,23 +46,30 @@ if [[ "$(uname -m)" != "x86_64" ]]; then
     exit 1
 fi
 
+if cvirt=$(systemd-detect-virt --container); then
+    echo "Error: Cloudron does not support ${cvirt}, only runs on bare metal or with full hardware virtualization"
+    exit 1
+fi
+
 # do not use is-active in case box service is down and user attempts to re-install
 if systemctl cat box.service >/dev/null 2>&1; then
     echo "Error: Cloudron is already installed. To reinstall, start afresh"
     exit 1
 fi
 
-initBaseImage="true"
 provider="generic"
 requestedVersion=""
 installServerOrigin="https://api.cloudron.io"
 apiServerOrigin="https://api.cloudron.io"
 webServerOrigin="https://cloudron.io"
+consoleServerOrigin="https://console.cloudron.io"
 sourceTarballUrl=""
 rebootServer="true"
-setupToken=""
+setupToken="" # this is a OTP for securing an installation (https://forum.cloudron.io/topic/6389/add-password-for-initial-configuration)
+appstoreSetupToken=""
+redo="false"
 
-args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot,generate-setup-token" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,provider:,version:,env:,skip-reboot,generate-setup-token,setup-token:,redo" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
@@ -74,17 +81,20 @@ while true; do
         if [[ "$2" == "dev" ]]; then
             apiServerOrigin="https://api.dev.cloudron.io"
             webServerOrigin="https://dev.cloudron.io"
+            consoleServerOrigin="https://console.dev.cloudron.io"
             installServerOrigin="https://api.dev.cloudron.io"
         elif [[ "$2" == "staging" ]]; then
             apiServerOrigin="https://api.staging.cloudron.io"
             webServerOrigin="https://staging.cloudron.io"
+            consoleServerOrigin="https://console.staging.cloudron.io"
             installServerOrigin="https://api.staging.cloudron.io"
         elif [[ "$2" == "unstable" ]]; then
             installServerOrigin="https://api.dev.cloudron.io"
         fi
         shift 2;;
-    --skip-baseimage-init) initBaseImage="false"; shift;;
     --skip-reboot) rebootServer="false"; shift;;
+    --redo) redo="true"; shift;;
+    --setup-token) appstoreSetupToken="$2"; shift 2;;
     --generate-setup-token) setupToken="$(openssl rand -hex 10)"; shift;;
     --) break;;
     *) echo "Unknown option $1"; exit 1;;
@@ -99,14 +109,16 @@ fi
 
 # Only --help works with mismatched ubuntu
 ubuntu_version=$(lsb_release -rs)
-if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubuntu_version}" != "20.04" ]]; then
-    echo "Cloudron requires Ubuntu 16.04, 18.04 or 20.04" > /dev/stderr
+if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubuntu_version}" != "20.04" && "${ubuntu_version}" != "22.04" ]]; then
+    echo "Cloudron requires Ubuntu 18.04, 20.04, 22.04" > /dev/stderr
     exit 1
 fi
 
 if which nginx >/dev/null || which docker >/dev/null || which node > /dev/null; then
-    echo "Error: Some packages like nginx/docker/nodejs are already installed. Cloudron requires specific versions of these packages and will install them as part of it's installation. Please start with a fresh Ubuntu install and run this script again." > /dev/stderr
-    exit 1
+    if [[ "${redo}" == "false" ]]; then
+        echo "Error: Some packages like nginx/docker/nodejs are already installed. Cloudron requires specific versions of these packages and will install them as part of it's installation. Please start with a fresh Ubuntu install and run this script again." > /dev/stderr
+        exit 1
+    fi
 fi
 
 # Install MOTD file for stack script style installations. this is removed by the trap exit handler. Heredoc quotes prevents parameter expansion
@@ -143,17 +155,15 @@ echo ""
 echo " Join us at https://forum.cloudron.io for any questions."
 echo ""
 
-if [[ "${initBaseImage}" == "true" ]]; then
-    echo "=> Updating apt and installing script dependencies"
-    if ! apt-get update &>> "${LOG_FILE}"; then
-        echo "Could not update package repositories. See ${LOG_FILE}"
-        exit 1
-    fi
+echo "=> Updating apt and installing script dependencies"
+if ! apt-get update &>> "${LOG_FILE}"; then
+    echo "Could not update package repositories. See ${LOG_FILE}"
+    exit 1
+fi
 
-    if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install --no-install-recommends curl python3 ubuntu-standard software-properties-common -y &>> "${LOG_FILE}"; then
-        echo "Could not install setup dependencies (curl). See ${LOG_FILE}"
-        exit 1
-    fi
+if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install --no-install-recommends curl python3 ubuntu-standard software-properties-common -y &>> "${LOG_FILE}"; then
+    echo "Could not install setup dependencies (curl). See ${LOG_FILE}"
+    exit 1
 fi
 
 echo "=> Checking version"
@@ -181,15 +191,13 @@ if ! $curl -sL "${sourceTarballUrl}" | tar -zxf - -C "${box_src_tmp_dir}"; then
     exit 1
 fi
 
-if [[ "${initBaseImage}" == "true" ]]; then
-    echo -n "=> Installing base dependencies and downloading docker images (this takes some time) ..."
-    # initializeBaseUbuntuImage.sh args (provider, infraversion path) are only to support installation of pre 5.3 Cloudrons
-    if ! /bin/bash "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "generic" "../src" &>> "${LOG_FILE}"; then
-        echo "Init script failed. See ${LOG_FILE} for details"
-        exit 1
-    fi
-    echo ""
+echo -n "=> Installing base dependencies and downloading docker images (this takes some time) ..."
+init_ubuntu_script=$(test -f "${box_src_tmp_dir}/scripts/init-ubuntu.sh" && echo "${box_src_tmp_dir}/scripts/init-ubuntu.sh" || echo "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh")
+if ! /bin/bash "${init_ubuntu_script}" &>> "${LOG_FILE}"; then
+    echo "Init script failed. See ${LOG_FILE} for details"
+    exit 1
 fi
+echo ""
 
 # The provider flag is still used for marketplace images
 echo "=> Installing Cloudron version ${version} (this takes some time) ..."
@@ -204,6 +212,20 @@ fi
 
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('api_server_origin', '${apiServerOrigin}');" 2>/dev/null
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('web_server_origin', '${webServerOrigin}');" 2>/dev/null
+mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('console_server_origin', '${consoleServerOrigin}');" 2>/dev/null
+
+if [[ -n "${appstoreSetupToken}" ]]; then
+    if ! setupResponse=$(curl -sX POST -H "Content-type: application/json" --data "{\"setupToken\": \"${appstoreSetupToken}\"}" "${apiServerOrigin}/api/v1/cloudron_setup_done"); then
+        echo "Could not complete setup. See ${LOG_FILE} for details"
+        exit 1
+    fi
+
+    cloudronId=$(echo "${setupResponse}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["cloudronId"])')
+    mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('cloudron_id', '${cloudronId}');" 2>/dev/null
+
+    appstoreApiToken=$(echo "${setupResponse}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["cloudronToken"])')
+    mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('appstore_api_token', '${appstoreApiToken}');" 2>/dev/null
+fi
 
 echo -n "=> Waiting for cloudron to be ready (this takes some time) ..."
 while true; do

scripts/cloudron-support

@@ -8,7 +8,7 @@ set -eu -o pipefail
 PASTEBIN="https://paste.cloudron.io"
 OUT="/tmp/cloudron-support.log"
 LINE="\n========================================================\n"
-CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQVilclYAIu+ioDp/sgzzFz6YU0hPcRYY7ze/LiF/lC7uQqK062O54BFXTvQ3ehtFZCx3bNckjlT2e6gB8Qq07OM66De4/S/g+HJW4TReY2ppSPMVNag0TNGxDzVH8pPHOysAm33LqT2b6L/wEXwC6zWFXhOhHjcMqXvi8Ejaj20H1HVVcf/j8qs5Thkp9nAaFTgQTPu8pgwD8wDeYX1hc9d0PYGesTADvo6HF4hLEoEnefLw7PaStEbzk2fD3j7/g5r5HcgQQXBe74xYZ/1gWOX2pFNuRYOBSEIrNfJEjFJsqk3NR1+ZoMGK7j+AZBR4k0xbrmncQLcQzl6MMDzkp support@cloudron.io"
+CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGWS+930b8QdzbchGljt3KSljH9wRhYvht8srrtQHdzg support@cloudron.io"
 HELP_MESSAGE="
 This script collects diagnostic information to help debug server related issues.
 
@@ -77,6 +77,31 @@ if [[ "`df --output="avail" /tmp | sed -n 2p`" -lt "5120" ]]; then
     exit 1
 fi
 
+if [[ "${enableSSH}" == "true" ]]; then
+    ssh_port=$(cat /etc/ssh/sshd_config | grep "Port " | sed -e "s/.*Port //")
+
+    ssh_user="cloudron-support"
+    keys_file="/home/cloudron-support/.ssh/authorized_keys"
+
+    echo -e $LINE"SSH"$LINE >> $OUT
+    echo "Username: ${ssh_user}" >> $OUT
+    echo "Port:     ${ssh_port}" >> $OUT
+    echo "Key file: ${keys_file}" >> $OUT
+
+    echo -n "Enabling ssh access for the Cloudron support team..."
+    mkdir -p $(dirname "${keys_file}")       # .ssh does not exist sometimes
+    touch "${keys_file}"                # required for concat to work
+    if ! grep -q "${CLOUDRON_SUPPORT_PUBLIC_KEY}" "${keys_file}"; then
+        echo -e "\n${CLOUDRON_SUPPORT_PUBLIC_KEY}" >> "${keys_file}"
+        chmod 600 "${keys_file}"
+        chown "${ssh_user}" "${keys_file}"
+    fi
+
+    echo "Done"
+
+    exit 0
+fi
+
 echo -n "Generating Cloudron Support stats..."
 
 # clear file
@@ -119,37 +144,6 @@ iptables -L &>> $OUT
 
 echo "Done"
 
-if [[ "${enableSSH}" == "true" ]]; then
-    ssh_port=$(cat /etc/ssh/sshd_config | grep "Port " | sed -e "s/.*Port //")
-    permit_root_login=$(grep -q ^PermitRootLogin.*yes /etc/ssh/sshd_config && echo "yes" || echo "no")
-
-    # support.js uses similar logic
-    if [[ -d /home/ubuntu ]]; then
-        ssh_user="ubuntu"
-        keys_file="/home/ubuntu/.ssh/authorized_keys"
-    else
-        ssh_user="root"
-        keys_file="/root/.ssh/authorized_keys"
-    fi
-
-    echo -e $LINE"SSH"$LINE >> $OUT
-    echo "Username: ${ssh_user}" >> $OUT
-    echo "Port:     ${ssh_port}" >> $OUT
-    echo "PermitRootLogin: ${permit_root_login}" >> $OUT
-    echo "Key file: ${keys_file}" >> $OUT
-
-    echo -n "Enabling ssh access for the Cloudron support team..."
-    mkdir -p $(dirname "${keys_file}")       # .ssh does not exist sometimes
-    touch "${keys_file}"                # required for concat to work
-    if ! grep -q "${CLOUDRON_SUPPORT_PUBLIC_KEY}" "${keys_file}"; then
-        echo -e "\n${CLOUDRON_SUPPORT_PUBLIC_KEY}" >> "${keys_file}"
-        chmod 600 "${keys_file}"
-        chown "${ssh_user}" "${keys_file}"
-    fi
-
-    echo "Done"
-fi
-
 echo -n "Uploading information..."
 paste_key=$(curl -X POST ${PASTEBIN}/documents --silent --data-binary "@$OUT" | python3 -c "import sys, json; print(json.load(sys.stdin)['key'])")
 echo "Done"

scripts/init-ubuntu.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# This script is run on the base ubuntu. Put things here which are managed by ubuntu
+
 set -euv -o pipefail
 
 readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -29,10 +31,37 @@ debconf-set-selections <<< 'mysql-server mysql-server/root_password_again passwo
 
 # this enables automatic security upgrades (https://help.ubuntu.com/community/AutomaticSecurityUpdates)
 # resolvconf is needed for unbound to work property after disabling systemd-resolved in 18.04
+case "${ubuntu_version}" in
+16.04)
+    gpg_package="gnupg"
+    mysql_package="mysql-server-5.7"
+    ntpd_package=""
+    python_package="python2.7"
+    nginx_package="" # we use custom package for TLS v1.3 support
+    ;;
+18.04)
+    gpg_package="gpg"
+    mysql_package="mysql-server-5.7"
+    ntpd_package=""
+    python_package="python2.7"
+    nginx_package="" # we use custom package for TLS v1.3 support
+    ;;
+20.04)
+    gpg_package="gpg"
+    mysql_package="mysql-server-8.0"
+    ntpd_package="systemd-timesyncd"
+    python_package="python3.8"
+    nginx_package="nginx-full"
+    ;;
+22.04)
+    gpg_package="gpg"
+    mysql_package="mysql-server-8.0"
+    ntpd_package="systemd-timesyncd"
+    python_package="python3.10"
+    nginx_package="nginx-full"
+    ;;
+esac
 
-gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
-mysql_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "mysql-server-8.0" || echo "mysql-server-5.7")
-ntpd_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "systemd-timesyncd" || echo "")
 apt-get -y install --no-install-recommends \
     acl \
     apparmor \
@@ -45,11 +74,12 @@ apt-get -y install --no-install-recommends \
     $gpg_package \
     ipset \
     iptables \
-    libpython2.7 \
+    lib${python_package} \
     linux-generic \
     logrotate \
     $mysql_package \
     nfs-common \
+    $nginx_package \
     $ntpd_package \
     openssh-server \
     pwgen \
@@ -62,12 +92,6 @@ apt-get -y install --no-install-recommends \
     unzip \
     xfsprogs
 
-echo "==> installing nginx for xenial for TLSv3 support"
-curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
-# apt install with install deps (as opposed to dpkg -i)
-apt install -y /tmp/nginx.deb
-rm /tmp/nginx.deb
-
 # on some providers like scaleway the sudo file is changed and we want to keep the old one
 apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends sudo
 
@@ -75,36 +99,7 @@ apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends
 # debconf-set-selection of unattended-upgrades/enable_auto_updates + dpkg-reconfigure does not work
 cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
-echo "==> Installing node.js"
-readonly node_version=16.13.1
-mkdir -p /usr/local/node-${node_version}
-curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-${node_version}
-ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
-ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
-apt-get install -y --no-install-recommends python   # Install python which is required for npm rebuild
-[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
-
-# https://docs.docker.com/engine/installation/linux/ubuntulinux/
-echo "==> Installing Docker"
-
-# create systemd drop-in file. if you channge options here, be sure to fixup installer.sh as well
-mkdir -p /etc/systemd/system/docker.service.d
-echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
-
-# there are 3 packages for docker - containerd, CLI and the daemon
-readonly docker_version=20.10.12
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.9-1_amd64.deb" -o /tmp/containerd.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
-# apt install with install deps (as opposed to dpkg -i)
-apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
-rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
-
-storage_driver=$(docker info | grep "Storage Driver" | sed 's/.*: //')
-if [[ "${storage_driver}" != "overlay2" ]]; then
-    echo "Docker is using "${storage_driver}" instead of overlay2"
-    exit 1
-fi
+apt-get install -y --no-install-recommends $python_package   # Install python which is required for npm rebuild
 
 # do not upgrade grub because it might prompt user and break this script
 echo "==> Enable memory accounting"
@@ -112,30 +107,26 @@ apt-get -y --no-upgrade --no-install-recommends install grub2-common
 sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
 update-grub
 
-echo "==> Downloading docker images"
-if [ ! -f "${arg_infraversionpath}/infra_version.js" ]; then
-    echo "No infra_versions.js found"
-    exit 1
-fi
-
-images=$(node -e "var i = require('${arg_infraversionpath}/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
-
-echo -e "\tPulling docker images: ${images}"
-for image in ${images}; do
-    docker pull "${image}"
-    docker pull "${image%@sha256:*}" # this will tag the image for readability
-done
-
 echo "==> Install collectd"
 # without this, libnotify4 will install gnome-shell
-apt-get install -y libnotify4 --no-install-recommends
-if ! apt-get install -y --no-install-recommends libcurl3-gnutls collectd collectd-utils; then
-    # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
-    echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
-    sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
-fi
+apt-get install -y libnotify4 libcurl3-gnutls --no-install-recommends
 # https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
-[[ "${ubuntu_version}" == "20.04" ]] && echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
+if [[ "${ubuntu_version}" == "22.04" ]]; then
+    readonly launchpad="https://launchpad.net/ubuntu/+source/collectd/5.12.0-9/+build/23189375/+files"
+    cd /tmp && wget -q "${launchpad}/collectd_5.12.0-9_amd64.deb" "${launchpad}/collectd-utils_5.12.0-9_amd64.deb" "${launchpad}/collectd-core_5.12.0-9_amd64.deb" "${launchpad}/libcollectdclient1_5.12.0-9_amd64.deb"
+    cd /tmp && apt install -y --no-install-recommends ./libcollectdclient1_5.12.0-9_amd64.deb ./collectd-core_5.12.0-9_amd64.deb ./collectd_5.12.0-9_amd64.deb ./collectd-utils_5.12.0-9_amd64.deb && rm -f /tmp/collectd_*.deb
+    echo -e "\nLD_PRELOAD=/usr/lib/python3.10/config-3.10-x86_64-linux-gnu/libpython3.10.so" >> /etc/default/collectd
+else
+    if ! apt-get install -y --no-install-recommends collectd collectd-utils; then
+        # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
+        echo "Failed to install collectd, continuing anyway. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
+        sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
+    fi
+
+    if [[ "${ubuntu_version}" == "20.04" ]]; then
+        echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
+    fi
+fi
 
 # some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
 echo "==> Configuring host"
@@ -153,7 +144,7 @@ sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See h
 
 # https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
 echo "==> Disabling motd news"
-if [ -f "/etc/default/motd-news" ]; then
+if [[ -f "/etc/default/motd-news" ]]; then
     sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
 fi
 
@@ -185,8 +176,23 @@ systemctl stop systemd-resolved || true
 systemctl disable systemd-resolved || true
 
 # on vultr, ufw is enabled by default. we have our own firewall
-ufw disable
+ufw disable || true
 
 # we need unbound to work as this is required for installer.sh to do any DNS requests
 echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: no" > /etc/unbound/unbound.conf.d/cloudron-network.conf
 systemctl restart unbound
+
+# Ubuntu 22 has private home directories by default (https://discourse.ubuntu.com/t/private-home-directories-for-ubuntu-21-04-onwards/)
+sed -e 's/^HOME_MODE\([[:space:]]\+\).*$/HOME_MODE\10755/' -i /etc/login.defs
+
+# create the yellowtent user. system user has different numeric range, no age and won't show in login/gdm UI
+# the nologin will also disable su/login
+if ! id yellowtent 2>/dev/null; then
+    useradd --system --comment "Cloudron Box" --create-home --shell /usr/sbin/nologin yellowtent
+fi
+
+# add support user (no password, sudo)
+if ! id cloudron-support 2>/dev/null; then
+    useradd --system --comment "Cloudron Support (support@cloudron.io)" --create-home --no-user-group --shell /bin/bash cloudron-support
+fi
+

scripts/installer.sh

@@ -71,12 +71,17 @@ readonly is_update=$(systemctl is-active -q box && echo "yes" || echo "no")
 
 log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION)"
 
-log "updating docker"
+# https://docs.docker.com/engine/installation/linux/ubuntulinux/
+readonly docker_version=20.10.14
+if ! which docker 2>/dev/null || [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
+    log "installing/updating docker"
+
+    # create systemd drop-in file already to make sure images are with correct driver
+    mkdir -p /etc/systemd/system/docker.service.d
+    echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
 
-readonly docker_version=20.10.12
-if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
     # there are 3 packages for docker - containerd, CLI and the daemon
-    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.9-1_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.5.11-1_amd64.deb" -o /tmp/containerd.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
@@ -86,41 +91,40 @@ if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]];
     rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
 fi
 
+# we want atleast nginx 1.14 for TLS v1.3 support. Ubuntu 20/22 already has nginx 1.18
+# Ubuntu 18 OpenSSL does not have TLS v1.3 support, so we use the upstream nginx packages
 readonly nginx_version=$(nginx -v 2>&1)
-if [[ "${nginx_version}" != *"1.18."* ]]; then
-    log "installing nginx 1.18"
-    $curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
+if [[ "${ubuntu_version}" == "20.04" ]]; then
+    if [[ "${nginx_version}" == *"Ubuntu"* ]]; then
+        log "switching nginx to ubuntu package"
+        prepare_apt_once
+        apt remove -y nginx
+        apt install -y nginx-full
+    fi
+elif [[ "${ubuntu_version}" == "18.04" ]]; then
+    if [[ "${nginx_version}" != *"1.18."* ]]; then
+        log "installing/updating nginx 1.18"
+        $curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
 
-    prepare_apt_once
+        prepare_apt_once
 
-    # apt install with install deps (as opposed to dpkg -i)
-    apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
-    rm /tmp/nginx.deb
+        # apt install with install deps (as opposed to dpkg -i)
+        apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
+        rm /tmp/nginx.deb
+    fi
 fi
 
-if ! which mount.nfs; then
-    log "installing nfs-common"
-    prepare_apt_once
-    apt install -y nfs-common
-fi
-
-if ! which sshfs; then
-    log "installing sshfs"
-    prepare_apt_once
-    apt install -y sshfs
-fi
-
-log "updating node"
-readonly node_version=16.13.1
-if [[ "$(node --version)" != "v${node_version}" ]]; then
+readonly node_version=16.14.2
+if ! which node 2>/dev/null || [[ "$(node --version)" != "v${node_version}" ]]; then
+    log "installing/updating node ${node_version}"
     mkdir -p /usr/local/node-${node_version}
     $curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-${node_version}
     ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
     ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-14.17.6
+    rm -rf /usr/local/node-16.13.1
 fi
 
-# this is here (and not in updater.js) because rebuild requires the above node
+# note that rebuild requires the above node
 for try in `seq 1 10`; do
     # for reasons unknown, the dtrace package will fail. but rebuilding second time will work
 
@@ -138,7 +142,7 @@ if [[ ${try} -eq 10 ]]; then
 fi
 
 log "downloading new addon images"
-images=$(node -e "var i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
+images=$(node -e "let i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
 log "\tPulling docker images: ${images}"
 for image in ${images}; do
@@ -164,10 +168,15 @@ while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLO
     sleep 5
 done
 
-if ! id "${user}" 2>/dev/null; then
-    useradd "${user}" -m
+log "creating cloudron-support user"
+if ! id cloudron-support 2>/dev/null; then
+    useradd --system --comment "Cloudron Support (support@cloudron.io)" --create-home --no-user-group --shell /bin/bash cloudron-support
 fi
 
+log "locking the ${user} account"
+usermod --shell /usr/sbin/nologin "${user}"
+passwd --lock "${user}"
+
 if [[ "${is_update}" == "yes" ]]; then
     log "stop box service for update"
     ${box_src_dir}/setup/stop.sh

setup/start.sh

@@ -71,6 +71,8 @@ mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
 mkdir -p "${PLATFORM_DATA_DIR}/update"
 mkdir -p "${PLATFORM_DATA_DIR}/sftp/ssh" # sftp keys
 mkdir -p "${PLATFORM_DATA_DIR}/firewall"
+mkdir -p "${PLATFORM_DATA_DIR}/sshfs"
+mkdir -p "${PLATFORM_DATA_DIR}/cifs"
 
 # ensure backups folder exists and is writeable
 mkdir -p /var/backups
@@ -127,19 +129,13 @@ systemctl restart unbound
 systemctl restart cloudron-syslog
 
 log "Configuring sudoers"
-rm -f /etc/sudoers.d/${USER}
-cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER}
+rm -f /etc/sudoers.d/${USER} /etc/sudoers.d/cloudron
+cp "${script_dir}/start/sudoers" /etc/sudoers.d/cloudron
 
 log "Configuring collectd"
 rm -rf /etc/collectd /var/log/collectd.log
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
-if [[ "${ubuntu_version}" == "20.04" ]]; then
-    # https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
-    if ! grep -q LD_PRELOAD /etc/default/collectd; then
-        echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
-    fi
-fi
 systemctl restart collectd
 
 log "Configuring sysctl"
@@ -178,7 +174,7 @@ fi
 
 # worker_rlimit_nofile in nginx config can be max this number
 mkdir -p /etc/systemd/system/nginx.service.d
-if ! grep -q "^LimitNOFILE=" /etc/systemd/system/nginx.service.d/cloudron.conf; then
+if ! grep -q "^LimitNOFILE=" /etc/systemd/system/nginx.service.d/cloudron.conf 2>/dev/null; then
     echo -e "[Service]\nLimitNOFILE=16384\n" > /etc/systemd/system/nginx.service.d/cloudron.conf
 fi
 
@@ -213,7 +209,8 @@ done
 
 readonly mysql_root_password="password"
 mysqladmin -u root -ppassword password password # reset default root password
-if [[ "${ubuntu_version}" == "20.04" ]]; then
+readonly mysqlVersion=$(mysql -NB -u root -p${mysql_root_password} -e 'SELECT VERSION()' 2>/dev/null)
+if [[ "${mysqlVersion}" == "8.0."* ]]; then
     # mysql 8 added a new caching_sha2_password scheme which mysqljs does not support
     mysql -u root -p${mysql_root_password} -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '${mysql_root_password}';"
 fi
@@ -234,7 +231,7 @@ log "Changing ownership"
 # note, change ownership after db migrate. this allow db migrate to move files around as root and then we can fix it up here
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
 chown -R "${USER}" /etc/cloudron
-chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall"
+chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall" "${PLATFORM_DATA_DIR}/sshfs" "${PLATFORM_DATA_DIR}/cifs"
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 chown "${USER}:${USER}" "${APPS_DATA_DIR}"

setup/start/cloudron-firewall.sh

@@ -41,16 +41,14 @@ ipxtables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,44
 
 # whitelist any user ports. we used to use --dports but it has a 15 port limit (XT_MULTI_PORTS)
 ports_json="/home/yellowtent/platformdata/firewall/ports.json"
-if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(','))" 2>/dev/null); then
-    IFS=',' arr=(${allowed_tcp_ports})
-    for p in "${arr[@]}"; do
+if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(' '))" 2>/dev/null); then
+    for p in $allowed_tcp_ports; do
         ipxtables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
     done
 fi
 
-if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(','))" 2>/dev/null); then
-    IFS=',' arr=(${allowed_udp_ports})
-    for p in "${arr[@]}"; do
+if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(' '))" 2>/dev/null); then
+    for p in $allowed_udp_ports; do
         ipxtables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
     done
 fi
@@ -63,6 +61,9 @@ ipset create cloudron_ldap_allowlist6 hash:net family inet6 || true
 ipset flush cloudron_ldap_allowlist6
 
 ldap_allowlist_json="/home/yellowtent/platformdata/firewall/ldap_allowlist.txt"
+# delete any existing redirect rule
+$iptables -t nat -D PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004 2>/dev/null || true
+$ip6tables -t nat -D PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004 >/dev/null || true
 if [[ -f "${ldap_allowlist_json}" ]]; then
     # without the -n block, any last line without a new line won't be read it!
     while read -r line || [[ -n "$line" ]]; do
@@ -76,8 +77,10 @@ if [[ -f "${ldap_allowlist_json}" ]]; then
     done < "${ldap_allowlist_json}"
 
     # ldap server we expose 3004 and also redirect from standard ldaps port 636
-    ipxtables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
+    $iptables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
     $iptables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist src -p tcp --dport 3004 -j ACCEPT
+
+    $ip6tables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
     $ip6tables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist6 src -p tcp --dport 3004 -j ACCEPT
 fi
 
@@ -144,6 +147,7 @@ for port in 3306 5432 6379 27017; do
     $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
+# Add the rate limit chain to input chain
 $iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
 $ip6tables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $ip6tables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
 

setup/start/collectd/collectd.conf

@@ -13,7 +13,7 @@
 ##############################################################################
 
 Hostname "localhost"
-#FQDNLookup true
+FQDNLookup false
 #BaseDir "/var/lib/collectd"
 #PluginDir "/usr/lib/collectd"
 #TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db"
@@ -232,7 +232,7 @@ LoadPlugin swap
 
 <Plugin write_graphite>
    <Node "graphing">
-       Host "localhost"
+       Host "127.0.0.1"
        Port "2003"
        Protocol "tcp"
        LogSendErrors true

setup/start/collectd/df.py

@@ -14,7 +14,7 @@ def read():
     for d in disks:
         device = d[0]
         if 'devicemapper' in d[1] or not device.startswith('/dev/'): continue
-        instance = device[len('/dev/'):].replace('/', '_') # see #348
+        instance = device[len('/dev/'):].replace('/', '_').replace('.', '_') # see #348
 
         try:
             st = os.statvfs(d[1]) # handle disk removal

setup/start/sudoers

@@ -1,6 +1,9 @@
 # sudo logging breaks journalctl output with very long urls (systemd bug)
 Defaults !syslog
 
+Defaults!/home/yellowtent/box/src/scripts/checkvolume.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/checkvolume.sh
+
 Defaults!/home/yellowtent/box/src/scripts/clearvolume.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/clearvolume.sh
 
@@ -64,3 +67,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmount.sh
 
 Defaults!/home/yellowtent/box/src/scripts/remountmount.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/remountmount.sh
+
+cloudron-support ALL=(ALL) NOPASSWD: ALL
+

src/acme2.js

@@ -47,16 +47,16 @@ function urlBase64Encode(string) {
 }
 
 function b64(str) {
-    var buf = Buffer.isBuffer(str) ? str : Buffer.from(str);
+    const buf = Buffer.isBuffer(str) ? str : Buffer.from(str);
     return urlBase64Encode(buf.toString('base64'));
 }
 
 function getModulus(pem) {
     assert(Buffer.isBuffer(pem));
 
-    var stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
+    const stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
     if (!stdout) return null;
-    var match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
+    const match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
     if (!match) return null;
     return Buffer.from(match[1], 'hex');
 }
@@ -324,7 +324,7 @@ Acme2.prototype.createKeyAndCsr = async function (hostname, keyFilePath, csrFile
     if (!csrDer) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
     if (!safe.fs.writeFileSync(csrFilePath, csrDer)) throw new BoxError(BoxError.FS_ERROR, safe.error); // bookkeeping. inspect with openssl req -text -noout -in hostname.csr -inform der
 
-    await safe(fs.promises.rmdir(tmpdir, { recursive: true }));
+    await safe(fs.promises.rm(tmpdir, { recursive: true, force: true }));
 
     debug('createKeyAndCsr: csr file (DER) saved at %s', csrFilePath);
 

src/apphealthmonitor.js

@@ -38,14 +38,14 @@ async function setHealth(app, health) {
             debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app: app });
         }
     } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
         if (lastHealth === apps.HEALTH_HEALTHY) {
             debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app: app });
         }
     } else {
         debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);

src/apps.js

@@ -42,7 +42,7 @@ exports = module.exports = {
     setMailbox,
     setInbox,
     setLocation,
-    setDataDir,
+    setStorage,
     repair,
 
     restore,
@@ -54,6 +54,7 @@ exports = module.exports = {
 
     backup,
     listBackups,
+    updateBackup,
 
     getTask,
     getLogPaths,
@@ -65,7 +66,9 @@ exports = module.exports = {
     stop,
     restart,
 
-    exec,
+    createExec,
+    startExec,
+    getExec,
 
     checkManifestConstraints,
     downloadManifest,
@@ -78,7 +81,7 @@ exports = module.exports = {
     schedulePendingTasks,
     restartAppsUsingAddons,
 
-    getDataDir,
+    getStorageDir,
     getIcon,
     getMemoryLimit,
     getLimits,
@@ -156,7 +159,7 @@ const appstore = require('./appstore.js'),
     mail = require('./mail.js'),
     manifestFormat = require('cloudron-manifestformat'),
     mounts = require('./mounts.js'),
-    once = require('once'),
+    once = require('./once.js'),
     os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
@@ -165,6 +168,7 @@ const appstore = require('./appstore.js'),
     semver = require('semver'),
     services = require('./services.js'),
     settings = require('./settings.js'),
+    shell = require('./shell.js'),
     spawn = require('child_process').spawn,
     split = require('split'),
     superagent = require('superagent'),
@@ -175,6 +179,7 @@ const appstore = require('./appstore.js'),
     util = require('util'),
     uuid = require('uuid'),
     validator = require('validator'),
+    volumes = require('./volumes.js'),
     _ = require('underscore');
 
 const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
@@ -182,11 +187,13 @@ const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationS
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson', 'apps.operatorsJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', 'apps.crontab',
     'apps.creationTime', 'apps.updateTime', 'apps.enableAutomaticUpdate',
-    'apps.enableMailbox', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain',
-    'apps.dataDir', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
+    'apps.enableMailbox', 'apps.mailboxDisplayName', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain',
+    'apps.storageVolumeId', 'apps.storageVolumePrefix', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
 
 // const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
 
+const CHECKVOLUME_CMD = path.join(__dirname, 'scripts/checkvolume.sh');
+
 function validatePortBindings(portBindings, manifest) {
     assert.strictEqual(typeof portBindings, 'object');
     assert.strictEqual(typeof manifest, 'object');
@@ -299,6 +306,18 @@ function translateSecondaryDomains(secondaryDomains) {
 function parseCrontab(crontab) {
     assert(crontab === null || typeof crontab === 'string');
 
+    // https://www.man7.org/linux/man-pages/man5/crontab.5.html#EXTENSIONS
+    const KNOWN_EXTENSIONS = {
+        '@service'  : '@service', // runs once
+        '@reboot'   : '@service',
+        '@yearly'   : '0 0 1 1 *',
+        '@annually' : '0 0 1 1 *',
+        '@monthly'  : '0 0 1 * *',
+        '@weekly'   : '0 0 * * 0',
+        '@daily'    : '0 0 * * *',
+        '@hourly'   : '0 * * * *',
+    };
+
     const result = [];
     if (!crontab) return result;
 
@@ -306,20 +325,28 @@ function parseCrontab(crontab) {
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i].trim();
         if (!line || line.startsWith('#')) continue;
-        const parts = /^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/.exec(line);
-        if (!parts) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron configuration at line ${i+1}`);
-        const schedule = parts.slice(1, 6).join(' ');
-        const command = parts[6];
+        if (line.startsWith('@')) {
+            const parts = /^(@\S+)\s+(.+)$/.exec(line);
+            if (!parts) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron configuration at line ${i+1}`);
+            const [, extension, command] = parts;
+            if (!KNOWN_EXTENSIONS[extension]) throw new BoxError(BoxError.BAD_FIELD, `Unknown extension pattern at line ${i+1}`);
+            result.push({ schedule: KNOWN_EXTENSIONS[extension], command });
+        } else {
+            const parts = /^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/.exec(line);
+            if (!parts) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron configuration at line ${i+1}`);
+            const schedule = parts.slice(1, 6).join(' ');
+            const command = parts[6];
 
-        try {
-            new CronJob('00 ' + schedule, function() {}); // second is disallowed
-        } catch (ex) {
-            throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern at line ${i+1}`);
+            try {
+                new CronJob('00 ' + schedule, function() {}); // second is disallowed
+            } catch (ex) {
+                throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern at line ${i+1}`);
+            }
+
+            if (command.length === 0) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern. Command must not be empty at line ${i+1}`); // not possible with the regexp we have
+
+            result.push({ schedule, command });
         }
-
-        if (command.length === 0) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern. Command must not be empty at line ${i+1}`); // not possible with the regexp we have
-
-        result.push({ schedule, command });
     }
 
     return result;
@@ -454,28 +481,29 @@ function validateEnv(env) {
     return null;
 }
 
-function validateDataDir(dataDir) {
-    if (dataDir === null) return null;
+async function checkStorage(app, volumeId, prefix) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof volumeId, 'string');
+    assert.strictEqual(typeof prefix, 'string');
 
-    if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`);
-    if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`);
-    if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`);
+    const volume = await volumes.get(volumeId);
+    if (volume === null) throw new BoxError(BoxError.BAD_FIELD, 'Storage volume not found');
 
-    // nfs shares will have the directory mounted already
-    let stat = safe.fs.lstatSync(dataDir);
-    if (stat) {
-        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`);
-        let entries = safe.fs.readdirSync(dataDir);
-        if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`);
-        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`);
-    }
+    const status = await volumes.getStatus(volume);
+    if (status.state !== 'active') throw new BoxError(BoxError.BAD_FIELD, 'Volume is not active');
 
-    // backup logic relies on paths not overlapping (because it recurses)
-    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`);
+    if (path.isAbsolute(prefix)) throw new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" must be a relative path`);
+    if (prefix.endsWith('/')) throw new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" contains trailing slash`);
+    if (prefix !== '' && path.normalize(prefix) !== prefix) throw new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" is not a normalized path`);
 
-    // if we made it this far, it cannot start with any of these realistically
-    const fhs = [ '/bin', '/boot', '/etc', '/lib', '/lib32', '/lib64', '/proc', '/run', '/sbin', '/tmp', '/usr' ];
-    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`);
+    const sourceDir = await getStorageDir(app);
+    const targetDir = path.join(volume.hostPath, prefix);
+    const rel = path.relative(sourceDir, targetDir);
+    if (!rel.startsWith('../') && rel.split('/').length > 1) throw new BoxError(BoxError.BAD_FIELD, 'Only one level subdirectory moves are supported');
+
+    const [error] = await safe(shell.promises.sudo('checkStorage', [ CHECKVOLUME_CMD, targetDir, sourceDir ], {}));
+    if (error && error.code === 2) throw new BoxError(BoxError.BAD_FIELD, `Target directory ${targetDir} is not empty`);
+    if (error && error.code === 3) throw new BoxError(BoxError.BAD_FIELD, `Target directory ${targetDir} does not support chown`);
 
     return null;
 }
@@ -507,17 +535,20 @@ function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, port
         if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is in use`);
     }
 
-    if (match[2] === 'dataDir') {
-        return new BoxError(BoxError.BAD_FIELD, `Data directory ${match[1]} is in use`);
+    if (match[2] === 'apps_storageVolume') {
+        return new BoxError(BoxError.BAD_FIELD, `Storage directory ${match[1]} is in use`);
     }
 
     return new BoxError(BoxError.ALREADY_EXISTS, `${match[2]} '${match[1]}' is in use`);
 }
 
-function getDataDir(app, dataDir) {
-    assert(dataDir === null || typeof dataDir === 'string');
+async function getStorageDir(app) {
+    assert.strictEqual(typeof app, 'object');
 
-    return dataDir || path.join(paths.APPS_DATA_DIR, app.id, 'data');
+    if (!app.storageVolumeId) return path.join(paths.APPS_DATA_DIR, app.id, 'data');
+    const volume = await volumes.get(app.storageVolumeId);
+    if (!volume) throw new BoxError(BoxError.NOT_FOUND, 'Volume not found'); // not possible
+    return path.join(volume.hostPath, app.storageVolumePrefix);
 }
 
 function removeInternalFields(app) {
@@ -526,8 +557,8 @@ function removeInternalFields(app) {
         'subdomain', 'domain', 'fqdn', 'crontab',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'operators',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts',
-        'enableMailbox', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
+        'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'storageVolumeId', 'storageVolumePrefix', 'mounts',
+        'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
 }
 
 // non-admins can only see these
@@ -756,6 +787,7 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
         tagsJson = data.tags ? JSON.stringify(data.tags) : null,
         mailboxName = data.mailboxName || null,
         mailboxDomain = data.mailboxDomain || null,
+        mailboxDisplayName = data.mailboxDisplayName || '',
         reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null,
         servicesConfigJson = data.servicesConfig ? JSON.stringify(data.servicesConfig) : null,
         enableMailbox = data.enableMailbox || false,
@@ -766,10 +798,11 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
     queries.push({
         query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
             + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, '
-            + 'enableMailbox) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+            + 'enableMailbox, mailboxDisplayName) '
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
         args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
-            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox ]
+            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon,
+            enableMailbox, mailboxDisplayName ]
     });
 
     queries.push({
@@ -1112,10 +1145,12 @@ async function onTaskFinished(error, appId, installationState, taskId, auditSour
         await eventlog.add(eventlog.ACTION_APP_UPDATE_FINISH, auditSource, { app, toManifest, fromManifest, success, errorMessage });
         break;
     }
-    case exports.ISTATE_PENDING_BACKUP:
-        await eventlog.add(eventlog.ACTION_APP_BACKUP_FINISH, auditSource, { app, success, errorMessage, backupId: task.result });
+    case exports.ISTATE_PENDING_BACKUP: {
+        const backup = await backups.get(task.result);
+        await eventlog.add(eventlog.ACTION_APP_BACKUP_FINISH, auditSource, { app, success, errorMessage, remotePath: backup?.remotePath, backupId: task.result });
         break;
     }
+    }
 }
 
 async function scheduleTask(appId, installationState, taskId, auditSource) {
@@ -1281,7 +1316,7 @@ async function install(data, auditSource) {
     let sso = 'sso' in data ? data.sso : null;
     if ('sso' in data && !('optionalSso' in manifest)) throw new BoxError(BoxError.BAD_FIELD, 'sso can only be specified for apps with optionalSso');
     // if sso was unspecified, enable it by default if possible
-    if (sso === null) sso = !!manifest.addons['ldap'] || !!manifest.addons['proxyAuth'];
+    if (sso === null) sso = !!manifest.addons?.ldap || !!manifest.addons?.proxyAuth;
 
     error = validateEnv(env);
     if (error) throw error;
@@ -1568,6 +1603,7 @@ async function setMailbox(app, data, auditSource) {
     const optional = 'optional' in app.manifest.addons.sendmail ? app.manifest.addons.sendmail.optional : false;
     if (!optional && !enableMailbox) throw new BoxError(BoxError.BAD_FIELD, 'App requires sendmail to be enabled');
 
+    const mailboxDisplayName = data.mailboxDisplayName || '';
     let mailboxName = data.mailboxName || null;
     const mailboxDomain = data.mailboxDomain || null;
 
@@ -1580,15 +1616,20 @@ async function setMailbox(app, data, auditSource) {
         } else {
             mailboxName = mailboxNameForSubdomain(app.subdomain, app.domain, app.manifest);
         }
+
+        if (mailboxDisplayName) {
+            error = mail.validateDisplayName(mailboxDisplayName);
+            if (error) throw new BoxError(BoxError.BAD_FIELD, error.message);
+        }
     }
 
     const task = {
         args: {},
-        values: { enableMailbox, mailboxName, mailboxDomain }
+        values: { enableMailbox, mailboxName, mailboxDomain, mailboxDisplayName }
     };
     const taskId = await addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task, auditSource);
 
-    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mailboxName, mailboxDomain, taskId });
+    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mailboxName, mailboxDomain, mailboxDisplayName, taskId });
 
     return { taskId };
 }
@@ -1761,25 +1802,29 @@ async function setLocation(app, data, auditSource) {
     return { taskId };
 }
 
-async function setDataDir(app, dataDir, auditSource) {
+async function setStorage(app, volumeId, volumePrefix, auditSource) {
     assert.strictEqual(typeof app, 'object');
-    assert(dataDir === null || typeof dataDir === 'string');
+    assert(volumeId === null || typeof volumeId === 'string');
+    assert(volumePrefix === null || typeof volumePrefix === 'string');
     assert.strictEqual(typeof auditSource, 'object');
 
     const appId = app.id;
     let error = checkAppState(app, exports.ISTATE_PENDING_DATA_DIR_MIGRATION);
     if (error) throw error;
 
-    error = validateDataDir(dataDir);
-    if (error) throw error;
+    if (volumeId) {
+        await checkStorage(app, volumeId, volumePrefix);
+    } else {
+        volumeId = volumePrefix = null;
+    }
 
     const task = {
-        args: { newDataDir: dataDir },
+        args: { newStorageVolumeId: volumeId, newStorageVolumePrefix: volumePrefix },
         values: {}
     };
     const taskId = await addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, auditSource);
 
-    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, dataDir, taskId });
+    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, volumeId, volumePrefix, taskId });
 
     return { taskId };
 }
@@ -1847,6 +1892,9 @@ async function updateApp(app, data, auditSource) {
         values.mailboxDomain = app.domain;
     }
 
+    const hasSso = !!updateConfig.manifest.addons?.proxyAuth || !!updateConfig.manifest.addons?.ldap;
+    if (!hasSso && app.sso) values.sso = false; // turn off sso flag, if the update removes sso options
+
     const task = {
         args: { updateConfig },
         values
@@ -2022,7 +2070,7 @@ async function restore(app, backupId, auditSource) {
         values.mailboxDomain = app.domain;
     }
 
-    const restoreConfig = { backupId, backupFormat: backupInfo.format };
+    const restoreConfig = { remotePath: backupInfo.remotePath, backupFormat: backupInfo.format };
 
     const task = {
         args: {
@@ -2036,7 +2084,7 @@ async function restore(app, backupId, auditSource) {
 
     const taskId = await addTask(appId, exports.ISTATE_PENDING_RESTORE, task, auditSource);
 
-    await eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app: app, backupId: backupInfo.id, fromManifest: app.manifest, toManifest: backupInfo.manifest, taskId });
+    await eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { app, backupId: backupInfo.id, remotePath: backupInfo.remotePath, fromManifest: app.manifest, toManifest: backupInfo.manifest, taskId });
 
     return { taskId };
 }
@@ -2049,10 +2097,10 @@ async function importApp(app, data, auditSource) {
     const appId = app.id;
 
     // all fields are optional
-    data.backupId = data.backupId || null;
+    data.remotePath = data.remotePath || null;
     data.backupFormat = data.backupFormat || null;
     data.backupConfig = data.backupConfig || null;
-    const { backupId, backupFormat, backupConfig } = data;
+    const { remotePath, backupFormat, backupConfig } = data;
 
     let error = backupFormat ? validateBackupFormat(backupFormat) : null;
     if (error) throw error;
@@ -2088,7 +2136,7 @@ async function importApp(app, data, auditSource) {
         }
     }
 
-    const restoreConfig = { backupId, backupFormat, backupConfig };
+    const restoreConfig = { remotePath, backupFormat, backupConfig };
 
     const task = {
         args: {
@@ -2101,7 +2149,7 @@ async function importApp(app, data, auditSource) {
     };
     const taskId = await addTask(appId, exports.ISTATE_PENDING_IMPORT, task, auditSource);
 
-    await eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app: app, backupId, fromManifest: app.manifest, toManifest: app.manifest, taskId });
+    await eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app: app, remotePath, fromManifest: app.manifest, toManifest: app.manifest, taskId });
 
     return { taskId };
 }
@@ -2157,6 +2205,7 @@ async function clone(app, data, user, auditSource) {
 
     const backupInfo = await backups.get(backupId);
 
+    if (!backupInfo) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
     if (!backupInfo.manifest) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Could not get restore config');
     if (backupInfo.encryptionVersion === 1) throw new BoxError(BoxError.BAD_FIELD, 'This encrypted backup was created with an older Cloudron version and cannot be cloned');
 
@@ -2206,7 +2255,8 @@ async function clone(app, data, user, auditSource) {
         tags: app.tags,
         enableAutomaticUpdate: app.enableAutomaticUpdate,
         icon: icons.icon,
-        enableMailbox: app.enableMailbox
+        enableMailbox: app.enableMailbox,
+        mailboxDisplayName: app.mailboxDisplayName
     };
 
     const [addError] = await safe(add(newAppId, appStoreId, manifest, subdomain, domain, translatePortBindings(portBindings, manifest), obj));
@@ -2215,7 +2265,7 @@ async function clone(app, data, user, auditSource) {
 
     await purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' });
 
-    const restoreConfig = { backupId, backupFormat: backupInfo.format };
+    const restoreConfig = { remotePath: backupInfo.remotePath, backupFormat: backupInfo.format };
     const task = {
         args: { restoreConfig, overwriteDns, skipDnsSetup, oldManifest: null },
         values: {},
@@ -2229,7 +2279,7 @@ async function clone(app, data, user, auditSource) {
     newApp.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
     newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
-    await eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId, oldApp: app, newApp, taskId });
+    await eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId, remotePath: backupInfo.remotePath, oldApp: app, newApp, taskId });
 
     return { id: newAppId, taskId };
 }
@@ -2328,7 +2378,7 @@ function checkManifestConstraints(manifest) {
     return null;
 }
 
-async function exec(app, options) {
+async function createExec(app, options) {
     assert.strictEqual(typeof app, 'object');
     assert(options && typeof options === 'object');
 
@@ -2339,7 +2389,7 @@ async function exec(app, options) {
         throw new BoxError(BoxError.BAD_STATE, 'App not installed or running');
     }
 
-    const execOptions = {
+    const createOptions = {
         AttachStdin: true,
         AttachStdout: true,
         AttachStderr: true,
@@ -2351,6 +2401,18 @@ async function exec(app, options) {
         Cmd: cmd
     };
 
+    return await docker.createExec(app.containerId, createOptions);
+}
+
+async function startExec(app, execId, options) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof execId, 'string');
+    assert(options && typeof options === 'object');
+
+    if (app.installationState !== exports.ISTATE_INSTALLED || app.runState !== exports.RSTATE_RUNNING) {
+        throw new BoxError(BoxError.BAD_STATE, 'App not installed or running');
+    }
+
     const startOptions = {
         Detach: false,
         Tty: options.tty,
@@ -2366,10 +2428,26 @@ async function exec(app, options) {
         stderr: true
     };
 
-    const stream = await docker.execContainer(app.containerId, { execOptions, startOptions, rows: options.rows, columns: options.columns });
+    const stream = await docker.startExec(execId, startOptions);
+
+    if (options.rows && options.columns) {
+        // there is a race where resizing too early results in a 404 "no such exec"
+        // https://git.cloudron.io/cloudron/box/issues/549
+        setTimeout(async function () {
+            await safe(docker.resizeExec(execId, { h: options.rows, w: options.columns }, { debug }));
+        }, 2000);
+    }
+
     return stream;
 }
 
+async function getExec(app, execId) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof execId, 'string');
+
+    return await docker.getExec(execId);
+}
+
 function canAutoupdateApp(app, updateInfo) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof updateInfo, 'object');
@@ -2452,6 +2530,18 @@ async function listBackups(app, page, perPage) {
     return await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, page, perPage);
 }
 
+async function updateBackup(app, backupId, data) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof data, 'object');
+
+    const backup = await backups.get(backupId);
+    if (!backup) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
+    if (backup.identifier !== app.id) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); // some other app's backup
+
+    await backups.update(backupId, data);
+}
+
 async function restoreInstalledApps(options, auditSource) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');
@@ -2461,11 +2551,11 @@ async function restoreInstalledApps(options, auditSource) {
     apps = apps.filter(app => app.installationState !== exports.ISTATE_PENDING_RESTORE); // safeguard against tasks being created non-stop if we crash on startup
 
     for (const app of apps) {
-        const [error, results] = await safe(backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1));
+        const [error, result] = await safe(backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1));
         let installationState, restoreConfig, oldManifest;
-        if (!error && results.length) {
+        if (!error && result.length) {
             installationState = exports.ISTATE_PENDING_RESTORE;
-            restoreConfig = { backupId: results[0].id, backupFormat: results[0].format };
+            restoreConfig = { remotePath: result[0].remotePath, backupFormat: result[0].format };
             oldManifest = app.manifest;
         } else {
             installationState = exports.ISTATE_PENDING_INSTALL;
@@ -2582,7 +2672,8 @@ async function downloadFile(app, filePath) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof filePath, 'string');
 
-    const statStream = await exec(app, { cmd: [ 'stat', '--printf=%F-%s', filePath ], tty: true });
+    const statExecId = await createExec(app, { cmd: [ 'stat', '--printf=%F-%s', filePath ], tty: true });
+    const statStream = await startExec(app, statExecId, { tty: true });
     const data = await drainStream(statStream);
 
     const parts = data.split('-');
@@ -2603,7 +2694,8 @@ async function downloadFile(app, filePath) {
         throw new BoxError(BoxError.NOT_FOUND, 'only files or dirs can be downloaded');
     }
 
-    const inputStream = await exec(app, { cmd, tty: false });
+    const execId = await createExec(app, { cmd, tty: false });
+    const inputStream = await startExec(app, execId, { tty: false });
 
     // transforms the docker stream into a normal stream
     const stdoutStream = new TransformStream({
@@ -2644,7 +2736,8 @@ async function uploadFile(app, sourceFilePath, destFilePath) {
     const escapedDestFilePath = safe.child_process.execSync(`printf %q '${destFilePath.replace(/'/g, '\'\\\'\'')}'`, { shell: '/bin/bash', encoding: 'utf8' });
     debug(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`);
 
-    const destStream = await exec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false });
+    const execId = await createExec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false });
+    const destStream = await startExec(app, execId, { tty: false });
 
     return new Promise((resolve, reject) => {
         const done = once(error => reject(new BoxError(BoxError.FS_ERROR, error.message)));

src/appstore.js

@@ -13,14 +13,17 @@ exports = module.exports = {
     purchaseApp,
     unpurchaseApp,
 
-    createUserToken,
+    getWebToken,
     getSubscription,
     isFreePlan,
 
     getAppUpdate,
     getBoxUpdate,
 
-    createTicket
+    createTicket,
+
+    // exported for tests
+    _unregister: unregister
 };
 
 const apps = require('./apps.js'),
@@ -34,6 +37,7 @@ const apps = require('./apps.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     settings = require('./settings.js'),
+    sysinfo = require('./sysinfo.js'),
     superagent = require('superagent'),
     support = require('./support.js'),
     util = require('util');
@@ -78,17 +82,15 @@ async function login(email, password, totpToken) {
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof totpToken, 'string');
 
-    const data = { email, password, totpToken };
-
-    const url = settings.apiServerOrigin() + '/api/v1/login';
-    const [error, response] = await safe(superagent.post(url)
-        .send(data)
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/login`)
+        .send({ email, password, totpToken })
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${response.status}`);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. status code: ${response.status}`);
+    if (!response.body.accessToken) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. invalid response: ${response.text}`);
 
     return response.body; // { userId, accessToken }
 }
@@ -97,54 +99,36 @@ async function registerUser(email, password) {
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof password, 'string');
 
-    const data = { email, password };
-
-    const url = settings.apiServerOrigin() + '/api/v1/register_user';
-    const [error, response] = await safe(superagent.post(url)
-        .send(data)
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/register_user`)
+        .send({ email, password, utmSource: 'cloudron-dashboard' })
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-
-    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, 'account already exists');
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${response.status}`);
+    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, 'Registration error: account already exists');
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Registration error. invalid response: ${response.status}`);
 }
 
-async function createUserToken() {
+async function getWebToken() {
     if (settings.isDemo()) throw new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode');
 
-    const token = await settings.getCloudronToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    const token = await settings.getAppstoreWebToken();
+    if (!token) throw new BoxError(BoxError.NOT_FOUND); // user will have to re-login with password somehow
 
-    const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
-
-    const [error, response] = await safe(superagent.post(url)
-        .send({})
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
-
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${response.status}`);
-
-    return response.body.accessToken;
+    return token;
 }
 
 async function getSubscription() {
-    const token = await settings.getCloudronToken();
+    const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    const url = settings.apiServerOrigin() + '/api/v1/subscription';
-
-    const [error, response] = await safe(superagent.get(url)
+    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/subscription`)
         .query({ accessToken: token })
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR);
     if (response.status === 502) throw new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`);
 
@@ -165,12 +149,10 @@ async function purchaseApp(data) {
     assert(data.appstoreId || data.manifestId);
     assert.strictEqual(typeof data.appId, 'string');
 
-    const token = await settings.getCloudronToken();
+    const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps`;
-
-    const [error, response] = await safe(superagent.post(url)
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/cloudronapps`)
         .send(data)
         .query({ accessToken: token })
         .timeout(30 * 1000)
@@ -180,7 +162,6 @@ async function purchaseApp(data) {
     if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND); // appstoreId does not exist
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
     if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     // 200 if already purchased, 201 is newly purchased
     if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', response.status, response.body));
 }
@@ -190,7 +171,7 @@ async function unpurchaseApp(appId, data) {
     assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId }
     assert(data.appstoreId || data.manifestId);
 
-    const token = await settings.getCloudronToken();
+    const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
     const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
@@ -203,8 +184,7 @@ async function unpurchaseApp(appId, data) {
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 404) return;   // was never purchased
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed to get app. status:${response.status}`);
 
     [error, response] = await safe(superagent.del(url)
         .send(data)
@@ -214,31 +194,28 @@ async function unpurchaseApp(appId, data) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
+    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed. status:${response.status}`);
 }
 
 async function getBoxUpdate(options) {
     assert.strictEqual(typeof options, 'object');
 
-    const token = await settings.getCloudronToken();
+    const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
-
     const query = {
         accessToken: token,
         boxVersion: constants.VERSION,
         automatic: options.automatic
     };
 
-    const [error, response] = await safe(superagent.get(url)
+    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/boxupdate`)
         .query(query)
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status === 204) return; // no update
     if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
@@ -263,10 +240,9 @@ async function getAppUpdate(app, options) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
 
-    const token = await settings.getCloudronToken();
+    const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
     const query = {
         accessToken: token,
         boxVersion: constants.VERSION,
@@ -275,14 +251,13 @@ async function getAppUpdate(app, options) {
         automatic: options.automatic
     };
 
-    const [error, response] = await safe(superagent.get(url)
+    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/appupdate`)
         .query(query)
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status === 204) return; // no update
     if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
@@ -306,24 +281,23 @@ async function getAppUpdate(app, options) {
 async function registerCloudron(data) {
     assert.strictEqual(typeof data, 'object');
 
-    const url = `${settings.apiServerOrigin()}/api/v1/register_cloudron`;
+    const { domain, accessToken, version, existingApps } = data;
 
-    const [error, response] = await safe(superagent.post(url)
-        .send(data)
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/register_cloudron`)
+        .send({ domain, accessToken, version, existingApps })
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${response.statusCode} ${error.message}`);
 
-    // cloudronId, token, licenseKey
+    // cloudronId, token
     if (!response.body.cloudronId) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id');
     if (!response.body.cloudronToken) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token');
-    if (!response.body.licenseKey) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no license');
 
     await settings.setCloudronId(response.body.cloudronId);
-    await settings.setCloudronToken(response.body.cloudronToken);
-    await settings.setLicenseKey(response.body.licenseKey);
+    await settings.setAppstoreApiToken(response.body.cloudronToken);
+    await settings.setAppstoreWebToken(accessToken);
 
     debug(`registerCloudron: Cloudron registered with id ${response.body.cloudronId}`);
 }
@@ -331,23 +305,23 @@ async function registerCloudron(data) {
 async function updateCloudron(data) {
     assert.strictEqual(typeof data, 'object');
 
-    const token = await settings.getCloudronToken();
+    const { domain } = data;
+
+    const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    const url = `${settings.apiServerOrigin()}/api/v1/update_cloudron`;
     const query = {
         accessToken: token
     };
 
-    const [error, response] = await safe(superagent.post(url)
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/update_cloudron`)
         .query(query)
-        .send(data)
+        .send({ domain })
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
     debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
@@ -356,13 +330,20 @@ async function updateCloudron(data) {
 async function registerWithLoginCredentials(options) {
     assert.strictEqual(typeof options, 'object');
 
-    const token = await settings.getCloudronToken();
-    if (token) throw new BoxError(BoxError.CONFLICT, 'Cloudron is already registered');
-
     if (options.signup) await registerUser(options.email, options.password);
 
     const result = await login(options.email, options.password, options.totpToken || '');
     await registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION });
+
+    for (const app of await apps.list()) {
+        await purchaseApp({ appId: app.id, appstoreId: app.appStoreId, manifestId: app.manifest.id || 'customapp' });
+    }
+}
+
+async function unregister() {
+    await settings.setCloudronId('');
+    await settings.setAppstoreApiToken('');
+    await settings.setAppstoreWebToken('');
 }
 
 async function createTicket(info, auditSource) {
@@ -374,11 +355,12 @@ async function createTicket(info, auditSource) {
     assert.strictEqual(typeof info.description, 'string');
     assert.strictEqual(typeof auditSource, 'object');
 
-    const token = await settings.getCloudronToken();
+    const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
     if (info.enableSshSupport) {
         await safe(support.enableRemoteSupport(true, auditSource));
+        info.ipv4 = await sysinfo.getServerIPv4();
     }
 
     info.app = info.appId ? await apps.get(info.appId) : null;
@@ -405,30 +387,26 @@ async function createTicket(info, auditSource) {
     const [error, response] = await safe(request);
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+    await eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
 
     return { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` };
 }
 
 async function getApps() {
-    const token = await settings.getCloudronToken();
+    const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
     const unstable = await settings.getUnstableAppsConfig();
 
-    const url = `${settings.apiServerOrigin()}/api/v1/apps`;
-
-    const [error, response] = await safe(superagent.get(url)
+    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/apps`)
         .query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable })
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 403 || response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', response.status, response.body));
     if (!response.body.apps) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
@@ -445,7 +423,7 @@ async function getAppVersion(appId, version) {
 
     if (!isAppAllowed(appId, listingConfig)) throw new BoxError(BoxError.FEATURE_DISABLED);
 
-    const token = await settings.getCloudronToken();
+    const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
     let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
@@ -459,7 +437,6 @@ async function getAppVersion(appId, version) {
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 403 || response.statusCode === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
     if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND);
-    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', response.status, response.body));
 
     return response.body;

src/apptask.js

@@ -41,11 +41,14 @@ const apps = require('./apps.js'),
     sysinfo = require('./sysinfo.js'),
     _ = require('underscore');
 
-const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/app.ejs', { encoding: 'utf8' }),
-    MV_VOLUME_CMD = path.join(__dirname, 'scripts/mvvolume.sh'),
+const MV_VOLUME_CMD = path.join(__dirname, 'scripts/mvvolume.sh'),
     LOGROTATE_CONFIG_EJS = fs.readFileSync(__dirname + '/logrotate.ejs', { encoding: 'utf8' }),
     CONFIGURE_LOGROTATE_CMD = path.join(__dirname, 'scripts/configurelogrotate.sh');
 
+// https://rootlesscontaine.rs/getting-started/common/cgroup2/#checking-whether-cgroup-v2-is-already-enabled
+const CGROUP_VERSION = fs.existsSync('/sys/fs/cgroup/cgroup.controllers') ? '2' : '1';
+const COLLECTD_CONFIG_EJS = fs.readFileSync(`${__dirname}/collectd/app_cgroup_v${CGROUP_VERSION}.ejs`, { encoding: 'utf8' });
+
 function makeTaskError(error, app) {
     assert.strictEqual(typeof error, 'object');
     assert.strictEqual(typeof app, 'object');
@@ -75,7 +78,7 @@ async function allocateContainerIp(app) {
         const iprange = iputils.intFromIp('172.18.20.255') - iputils.intFromIp('172.18.16.1');
         let rnd = Math.floor(Math.random() * iprange);
         const containerIp = iputils.ipFromInt(iputils.intFromIp('172.18.16.1') + rnd);
-        updateApp(app, { containerIp });
+        await updateApp(app, { containerIp });
     });
 }
 
@@ -157,7 +160,8 @@ async function deleteAppDir(app, options) {
 async function addCollectdProfile(app) {
     assert.strictEqual(typeof app, 'object');
 
-    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir: apps.getDataDir(app, app.dataDir) });
+    const appDataDir = await apps.getStorageDir(app);
+    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir });
     await collectd.addProfile(app.id, collectdConf);
 }
 
@@ -265,19 +269,22 @@ async function waitForDnsPropagation(app) {
     }
 }
 
-async function moveDataDir(app, targetDir) {
+async function moveDataDir(app, targetVolumeId, targetVolumePrefix) {
     assert.strictEqual(typeof app, 'object');
-    assert(targetDir === null || typeof targetDir === 'string');
+    assert(targetVolumeId === null || typeof targetVolumeId === 'string');
+    assert(targetVolumePrefix === null || typeof targetVolumePrefix === 'string');
 
-    const resolvedSourceDir = apps.getDataDir(app, app.dataDir);
-    const resolvedTargetDir = apps.getDataDir(app, targetDir);
+    const resolvedSourceDir = await apps.getStorageDir(app);
+    const resolvedTargetDir = await apps.getStorageDir(_.extend({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix }));
 
     debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
 
-    if (resolvedSourceDir === resolvedTargetDir) return;
+    if (resolvedSourceDir !== resolvedTargetDir) {
+        const [error] = await safe(shell.promises.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
+        if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`);
+    }
 
-    const [error] = await safe(shell.promises.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
-    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`);
+    await updateApp(app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix });
 }
 
 async function downloadImage(manifest) {
@@ -328,7 +335,7 @@ async function install(app, args, progressCallback) {
     }
     await services.teardownAddons(app, addonsToRemove);
 
-    if (!restoreConfig || restoreConfig.backupId) { // in-place import should not delete data dir
+    if (!restoreConfig || restoreConfig.remotePath) { // in-place import should not delete data dir
         await deleteAppDir(app, { removeDirectory: false }); // do not remove any symlinked appdata dir
     }
 
@@ -357,7 +364,7 @@ async function install(app, args, progressCallback) {
     if (!restoreConfig) { // install
         await progressCallback({ percent: 60, message: 'Setting up addons' });
         await services.setupAddons(app, app.manifest.addons);
-    } else if (app.installationState === apps.ISTATE_PENDING_IMPORT && !restoreConfig.backupId) { // in-place import
+    } else if (app.installationState === apps.ISTATE_PENDING_IMPORT && !restoreConfig.remotePath) { // in-place import
         await progressCallback({ percent: 60, message: 'Importing addons in-place' });
         await services.setupAddons(app, app.manifest.addons);
         await services.clearAddons(app, _.omit(app.manifest.addons, 'localstorage'));
@@ -517,8 +524,9 @@ async function migrateDataDir(app, args, progressCallback) {
     assert.strictEqual(typeof args, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    const newDataDir = args.newDataDir;
-    assert(newDataDir === null || typeof newDataDir === 'string');
+    const { newStorageVolumeId, newStorageVolumePrefix } = args;
+    assert(newStorageVolumeId === null || typeof newStorageVolumeId === 'string');
+    assert(newStorageVolumePrefix === null || typeof newStorageVolumePrefix === 'string');
 
     await progressCallback({ percent: 10, message: 'Cleaning up old install' });
     await deleteContainers(app, { managedOnly: true });
@@ -526,12 +534,12 @@ async function migrateDataDir(app, args, progressCallback) {
     await progressCallback({ percent: 45, message: 'Ensuring app data directory' });
     await createAppDir(app);
 
-    // re-setup addons since this creates the localStorage volume
+    // re-setup addons since this creates the localStorage destination
     await progressCallback({ percent: 50, message: 'Setting up addons' });
-    await services.setupAddons(_.extend({}, app, { dataDir: newDataDir }), app.manifest.addons);
+    await services.setupAddons(_.extend({}, app, { storageVolumeId: newStorageVolumeId, storageVolumePrefix: newStorageVolumePrefix }), app.manifest.addons);
 
     await progressCallback({ percent: 60, message: 'Moving data dir' });
-    await moveDataDir(app, newDataDir);
+    await moveDataDir(app, newStorageVolumeId, newStorageVolumePrefix);
 
     await progressCallback({ percent: 90, message: 'Creating container' });
     await createContainer(app);
@@ -539,7 +547,7 @@ async function migrateDataDir(app, args, progressCallback) {
     await startApp(app);
 
     await progressCallback({ percent: 100, message: 'Done' });
-    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir });
+    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
 }
 
 // configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local"

src/backupcleaner.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const BoxError = require('./boxerror.js');
+
 exports = module.exports = {
     run,
 
@@ -8,16 +10,17 @@ exports = module.exports = {
 
 const apps = require('./apps.js'),
     assert = require('assert'),
+    backupFormat = require('./backupformat.js'),
     backups = require('./backups.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:backupcleaner'),
     moment = require('moment'),
+    mounts = require('./mounts.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     storage = require('./storage.js'),
-    util = require('util'),
     _ = require('underscore');
 
 function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
@@ -35,7 +38,7 @@ function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
             else backup.discardReason = 'creating-too-long';
         } else if (referencedBackupIds.includes(backup.id)) {
             backup.keepReason = 'reference';
-        } else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) {
+        } else if ((backup.preserveSecs === -1) || ((now - backup.creationTime) < (backup.preserveSecs * 1000))) {
             backup.keepReason = 'preserveSecs';
         } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) {
             backup.keepReason = 'keepWithinSecs';
@@ -78,49 +81,42 @@ function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
     }
 }
 
-async function cleanupBackup(backupConfig, backup, progressCallback) {
+async function removeBackup(backupConfig, backup, progressCallback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof backup, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    const backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
+    const backupFilePath = backupFormat.api(backup.format).getBackupFilePath(backupConfig, backup.remotePath);
 
-    return new Promise((resolve) => {
-        function done(error) {
-            if (error) {
-                debug('cleanupBackup: error removing backup %j : %s', backup, error.message);
-                return resolve();
-            }
+    let removeError;
+    if (backup.format ==='tgz') {
+        progressCallback({ message: `${backup.remotePath}: Removing ${backupFilePath}`});
+        [removeError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFilePath));
+    } else {
+        progressCallback({ message: `${backup.remotePath}: Removing directory ${backupFilePath}`});
+        [removeError] = await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath, progressCallback));
+    }
 
-            // prune empty directory if possible
-            storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), async function (error) {
-                if (error) debug('cleanupBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message);
+    if (removeError) {
+        debug('removeBackup: error removing backup %j : %s', backup, removeError.message);
+        return;
+    }
 
-                const [delError] = await safe(backups.del(backup.id));
-                if (delError) debug('cleanupBackup: error removing from database', delError);
-                else debug('cleanupBackup: removed %s', backup.id);
+    // prune empty directory if possible
+    const [pruneError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath)));
+    if (pruneError) debug('removeBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), pruneError.message);
 
-                resolve();
-            });
-        }
-
-        if (backup.format ==='tgz') {
-            progressCallback({ message: `${backup.id}: Removing ${backupFilePath}`});
-            storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, done);
-        } else {
-            const events = storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath);
-            events.on('progress', (message) => progressCallback({ message: `${backup.id}: ${message}` }));
-            events.on('done', done);
-        }
-    });
+    const [delError] = await safe(backups.del(backup.id));
+    if (delError) debug(`removeBackup: error removing ${backup.id} from database`, delError);
+    else debug(`removeBackup: removed ${backup.remotePath}`);
 }
 
-async function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback) {
+async function cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback) {
     assert.strictEqual(typeof backupConfig, 'object');
-    assert(Array.isArray(referencedAppBackupIds));
+    assert(Array.isArray(referencedBackupIds));
     assert.strictEqual(typeof progressCallback, 'function');
 
-    let removedAppBackupIds = [];
+    const removedAppBackupPaths = [];
 
     const allApps = await apps.list();
     const allAppIds = allApps.map(a => a.id);
@@ -137,49 +133,49 @@ async function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressC
     // apply backup policy per app. keep latest backup only for existing apps
     let appBackupsToRemove = [];
     for (const appId of Object.keys(appBackupsById)) {
-        applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds);
+        applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedBackupIds);
         appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
     }
 
     for (const appBackup of appBackupsToRemove) {
         await progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
-        removedAppBackupIds.push(appBackup.id);
-        await cleanupBackup(backupConfig, appBackup, progressCallback); // never errors
+        removedAppBackupPaths.push(appBackup.remotePath);
+        await removeBackup(backupConfig, appBackup, progressCallback); // never errors
     }
 
     debug('cleanupAppBackups: done');
 
-    return removedAppBackupIds;
+    return removedAppBackupPaths;
 }
 
-async function cleanupMailBackups(backupConfig, referencedAppBackupIds, progressCallback) {
+async function cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback) {
     assert.strictEqual(typeof backupConfig, 'object');
-    assert(Array.isArray(referencedAppBackupIds));
+    assert(Array.isArray(referencedBackupIds));
     assert.strictEqual(typeof progressCallback, 'function');
 
-    let removedMailBackupIds = [];
+    const removedMailBackupPaths = [];
 
     const mailBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_MAIL, 1, 1000);
 
-    applyBackupRetentionPolicy(mailBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), referencedAppBackupIds);
+    applyBackupRetentionPolicy(mailBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), referencedBackupIds);
 
     for (const mailBackup of mailBackups) {
         if (mailBackup.keepReason) continue;
-        await progressCallback({ message: `Removing mail backup ${mailBackup.id}`});
-        removedMailBackupIds.push(mailBackup.id);
-        await cleanupBackup(backupConfig, mailBackup, progressCallback); // never errors
+        await progressCallback({ message: `Removing mail backup ${mailBackup.remotePath}`});
+        removedMailBackupPaths.push(mailBackup.remotePath);
+        await removeBackup(backupConfig, mailBackup, progressCallback); // never errors
     }
 
     debug('cleanupMailBackups: done');
 
-    return removedMailBackupIds;
+    return removedMailBackupPaths;
 }
 
 async function cleanupBoxBackups(backupConfig, progressCallback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    let referencedAppBackupIds = [], removedBoxBackupIds = [];
+    let referencedBackupIds = [], removedBoxBackupPaths = [];
 
     const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
 
@@ -187,48 +183,48 @@ async function cleanupBoxBackups(backupConfig, progressCallback) {
 
     for (const boxBackup of boxBackups) {
         if (boxBackup.keepReason) {
-            referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn);
+            referencedBackupIds = referencedBackupIds.concat(boxBackup.dependsOn);
             continue;
         }
 
-        await progressCallback({ message: `Removing box backup ${boxBackup.id}`});
+        await progressCallback({ message: `Removing box backup ${boxBackup.remotePath}`});
 
-        removedBoxBackupIds.push(boxBackup.id);
-        await cleanupBackup(backupConfig, boxBackup, progressCallback);
+        removedBoxBackupPaths.push(boxBackup.remotePath);
+        await removeBackup(backupConfig, boxBackup, progressCallback);
     }
 
     debug('cleanupBoxBackups: done');
 
-    return { removedBoxBackupIds, referencedAppBackupIds };
+    return { removedBoxBackupPaths, referencedBackupIds };
 }
 
+// cleans up the database by checking if backup existsing in the remote
 async function cleanupMissingBackups(backupConfig, progressCallback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
     const perPage = 1000;
-    let missingBackupIds = [];
-    const backupExists = util.promisify(storage.api(backupConfig.provider).exists);
+    const missingBackupPaths = [];
 
-    if (constants.TEST) return missingBackupIds;
+    if (constants.TEST) return missingBackupPaths;
 
     let page = 1, result = [];
     do {
         result = await backups.list(page, perPage);
 
         for (const backup of result) {
-            let backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
+            let backupFilePath = backupFormat.api(backup.format).getBackupFilePath(backupConfig, backup.remotePath);
             if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
 
-            const [existsError, exists] = await safe(backupExists(backupConfig, backupFilePath));
+            const [existsError, exists] = await safe(storage.api(backupConfig.provider).exists(backupConfig, backupFilePath));
             if (existsError || exists) continue;
 
-            await progressCallback({ message: `Removing missing backup ${backup.id}`});
+            await progressCallback({ message: `Removing missing backup ${backup.remotePath}`});
 
             const [delError] = await safe(backups.del(backup.id));
-            if (delError) debug(`cleanupBackup: error removing ${backup.id} from database`, delError);
+            if (delError) debug(`cleanupMissingBackups: error removing ${backup.id} from database`, delError);
 
-            missingBackupIds.push(backup.id);
+            missingBackupPaths.push(backup.remotePath);
         }
 
         ++ page;
@@ -236,7 +232,7 @@ async function cleanupMissingBackups(backupConfig, progressCallback) {
 
     debug('cleanupMissingBackups: done');
 
-    return missingBackupIds;
+    return missingBackupPaths;
 }
 
 // removes the snapshots of apps that have been uninstalled
@@ -249,29 +245,23 @@ async function cleanupSnapshots(backupConfig) {
 
     delete info.box;
 
+    const progressCallback = (progress) => { debug(`cleanupSnapshots: ${progress.message}`); };
+
     for (const appId of Object.keys(info)) {
         const app = await apps.get(appId);
         if (app) continue; // app is still installed
 
-        await new Promise((resolve) => {
-            async function done(/* ignoredError */) {
-                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
-                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`));
+        if (info[appId].format ==='tgz') {
+            await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`)), { debug });
+        } else {
+            await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`), progressCallback), { debug });
+        }
 
-                await safe(backups.setSnapshotInfo(appId, null /* info */), { debug });
-                debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`);
+        safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
+        safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`));
 
-                resolve();
-            }
-
-            if (info[appId].format ==='tgz') {
-                storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), done);
-            } else {
-                const events = storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format));
-                events.on('progress', function (detail) { debug(`cleanupSnapshots: ${detail}`); });
-                events.on('done', done);
-            }
-        });
+        await safe(backups.setSnapshotInfo(appId, null /* info */), { debug });
+        debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`);
     }
 
     debug('cleanupSnapshots: done');
@@ -282,25 +272,32 @@ async function run(progressCallback) {
 
     const backupConfig = await settings.getBackupConfig();
 
+    if (mounts.isManagedProvider(backupConfig.provider) || backupConfig.provider === 'mountpoint') {
+        const hostPath = mounts.isManagedProvider(backupConfig.provider) ? paths.MANAGED_BACKUP_MOUNT_DIR : backupConfig.mountPoint;
+        const status = await mounts.getStatus(backupConfig.provider, hostPath); // { state, message }
+        debug(`clean: mount point status is ${JSON.stringify(status)}`);
+        if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
+    }
+
     if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
         debug('cleanup: keeping all backups');
         return {};
     }
 
     await progressCallback({ percent: 10, message: 'Cleaning box backups' });
-    const { removedBoxBackupIds, referencedAppBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback);
+    const { removedBoxBackupPaths, referencedBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback); // references is app or mail backup ids
 
     await progressCallback({ percent: 20, message: 'Cleaning mail backups' });
-    const removedMailBackupIds = await cleanupMailBackups(backupConfig, referencedAppBackupIds, progressCallback);
+    const removedMailBackupPaths = await cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback);
 
     await progressCallback({ percent: 40, message: 'Cleaning app backups' });
-    const removedAppBackupIds = await cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback);
+    const removedAppBackupPaths = await cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback);
 
     await progressCallback({ percent: 70, message: 'Cleaning missing backups' });
-    const missingBackupIds = await cleanupMissingBackups(backupConfig, progressCallback);
+    const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback);
 
     await progressCallback({ percent: 90, message: 'Cleaning snapshots' });
     await cleanupSnapshots(backupConfig);
 
-    return { removedBoxBackupIds, removedMailBackupIds, removedAppBackupIds, missingBackupIds };
+    return { removedBoxBackupPaths, removedMailBackupPaths, removedAppBackupPaths, missingBackupPaths };
 }

src/backupformat.js

@@ -0,0 +1,12 @@
+'use strict';
+
+exports = module.exports = {
+    api
+};
+
+function api(format) {
+    switch (format) {
+    case 'tgz': return require('./backupformat/tgz.js');
+    case 'rsync': return require('./backupformat/rsync.js');
+    }
+}

src/backupformat/rsync.js

@@ -0,0 +1,245 @@
+'use strict';
+
+exports = module.exports = {
+    getBackupFilePath,
+    download,
+    upload,
+
+    _saveFsMetadata: saveFsMetadata,
+    _restoreFsMetadata: restoreFsMetadata
+};
+
+const assert = require('assert'),
+    async = require('async'),
+    BoxError = require('../boxerror.js'),
+    DataLayout = require('../datalayout.js'),
+    debug = require('debug')('box:backupformat/rsync'),
+    fs = require('fs'),
+    hush = require('../hush.js'),
+    once = require('../once.js'),
+    path = require('path'),
+    safe = require('safetydance'),
+    storage = require('../storage.js'),
+    syncer = require('../syncer.js'),
+    util = require('util');
+
+function getBackupFilePath(backupConfig, remotePath) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+
+    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
+    return path.join(rootPath, remotePath);
+}
+
+function sync(backupConfig, remotePath, dataLayout, progressCallback, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB
+    const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10);
+    const removeDir = util.callbackify(storage.api(backupConfig.provider).removeDir);
+    const remove = util.callbackify(storage.api(backupConfig.provider).remove);
+
+    syncer.sync(dataLayout, function processTask(task, iteratorCallback) {
+        debug('sync: processing task: %j', task);
+        // the empty task.path is special to signify the directory
+        const destPath = task.path && backupConfig.encryption ? hush.encryptFilePath(task.path, backupConfig.encryption) : task.path;
+        const backupFilePath = path.join(getBackupFilePath(backupConfig, remotePath), destPath);
+
+        if (task.operation === 'removedir') {
+            debug(`Removing directory ${backupFilePath}`);
+            return removeDir(backupConfig, backupFilePath, progressCallback, iteratorCallback);
+        } else if (task.operation === 'remove') {
+            debug(`Removing ${backupFilePath}`);
+            return remove(backupConfig, backupFilePath, iteratorCallback);
+        }
+
+        let retryCount = 0;
+        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+            retryCallback = once(retryCallback); // protect again upload() erroring much later after read stream error
+
+            ++retryCount;
+            if (task.operation === 'add') {
+                progressCallback({ message: `Adding ${task.path}` + (retryCount > 1 ?  ` (Try ${retryCount})` : '') });
+                debug(`Adding ${task.path} position ${task.position} try ${retryCount}`);
+                const stream = hush.createReadStream(dataLayout.toLocalPath('./' + task.path), backupConfig.encryption);
+                stream.on('error', (error) => retryCallback(error.message.includes('ENOENT') ? null : error)); // ignore error if file disappears
+                stream.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0MBps looks wrong
+                    progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}MBps` }); // 0M@0MBps looks wrong
+                });
+                // only create the destination path when we have confirmation that the source is available. otherwise, we end up with
+                // files owned as 'root' and the cp later will fail
+                stream.on('open', function () {
+                    storage.api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, function (error) {
+                        debug(error ? `Error uploading ${task.path} try ${retryCount}: ${error.message}` : `Uploaded ${task.path}`);
+                        retryCallback(error);
+                    });
+                });
+            }
+        }, iteratorCallback);
+    }, concurrency, function (error) {
+        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+
+        callback();
+    });
+}
+
+// this is not part of 'snapshotting' because we need root access to traverse
+async function saveFsMetadata(dataLayout, metadataFile) {
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof metadataFile, 'string');
+
+    // contains paths prefixed with './'
+    const metadata = {
+        emptyDirs: [],
+        execFiles: [],
+        symlinks: []
+    };
+
+    // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
+    for (let lp of dataLayout.localPaths()) {
+        const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (emptyDirs === null) throw new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`);
+        if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
+
+        const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (execFiles === null) throw new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`);
+        if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
+
+        const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (symlinks === null) throw new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`);
+        if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => {
+            const target = safe.fs.readlinkSync(sl);
+            return { path: dataLayout.toRemotePath(sl), target };
+        }));
+    }
+
+    if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) throw new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`);
+}
+
+async function restoreFsMetadata(dataLayout, metadataFile) {
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof metadataFile, 'string');
+
+    debug(`Recreating empty directories in ${dataLayout.toString()}`);
+
+    const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
+    if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
+    const metadata = safe.JSON.parse(metadataJson);
+    if (metadata === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message);
+
+    for (const emptyDir of metadata.emptyDirs) {
+        const [mkdirError] = await safe(fs.promises.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }));
+        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to create path: ${mkdirError.message}`);
+    }
+
+    for (const execFile of metadata.execFiles) {
+        const [chmodError] = await safe(fs.promises.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8)));
+        if (chmodError) throw new BoxError(BoxError.FS_ERROR, `unable to chmod: ${chmodError.message}`);
+    }
+
+    for (const symlink of (metadata.symlinks || [])) {
+        if (!symlink.target) continue;
+        // the path may not exist if we had a directory full of symlinks
+        const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }));
+        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink (mkdir): ${mkdirError.message}`);
+        const [symlinkError] = await safe(fs.promises.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file'));
+        if (symlinkError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink: ${symlinkError.message}`);
+    }
+}
+
+function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
+
+    function downloadFile(entry, done) {
+        let relativePath = path.relative(backupFilePath, entry.fullPath);
+        if (backupConfig.encryption) {
+            const { error, result } = hush.decryptFilePath(relativePath, backupConfig.encryption);
+            if (error) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file'));
+            relativePath = result;
+        }
+        const destFilePath = dataLayout.toLocalPath('./' + relativePath);
+
+        fs.mkdir(path.dirname(destFilePath), { recursive: true }, function (error) {
+            if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));
+
+            async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+                storage.api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) {
+                    if (error) {
+                        progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                        return retryCallback(error);
+                    }
+
+                    let destStream = hush.createWriteStream(destFilePath, backupConfig.encryption);
+
+                    // protect against multiple errors. must destroy the write stream so that a previous retry does not write
+                    let closeAndRetry = once((error) => {
+                        if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                        else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` });
+                        sourceStream.destroy();
+                        destStream.destroy();
+                        retryCallback(error);
+                    });
+
+                    destStream.on('progress', function (progress) {
+                        const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                        if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0MBps looks wrong
+                        progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}MBps` });
+                    });
+                    destStream.on('error', closeAndRetry);
+
+                    sourceStream.on('error', closeAndRetry);
+
+                    progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
+
+                    sourceStream.pipe(destStream, { end: true }).on('done', closeAndRetry);
+                });
+            }, done);
+        });
+    }
+
+    storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
+        // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
+        const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
+
+        async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
+    }, callback);
+}
+
+async function download(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
+
+    const backupFilePath = getBackupFilePath(backupConfig, remotePath);
+    const downloadDirAsync = util.promisify(downloadDir);
+
+    await downloadDirAsync(backupConfig, backupFilePath, dataLayout, progressCallback);
+    await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
+}
+
+async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof dataLayout, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const syncAsync = util.promisify(sync);
+
+    await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
+    await syncAsync(backupConfig, remotePath, dataLayout, progressCallback);
+}

src/backupformat/tgz.js

@@ -0,0 +1,195 @@
+'use strict';
+
+exports = module.exports = {
+    getBackupFilePath,
+    download,
+    upload
+};
+
+const assert = require('assert'),
+    async = require('async'),
+    BoxError = require('../boxerror.js'),
+    DataLayout = require('../datalayout.js'),
+    debug = require('debug')('box:backupformat/tgz'),
+    { DecryptStream, EncryptStream } = require('../hush.js'),
+    once = require('../once.js'),
+    path = require('path'),
+    progressStream = require('progress-stream'),
+    storage = require('../storage.js'),
+    tar = require('tar-fs'),
+    zlib = require('zlib');
+
+function getBackupFilePath(backupConfig, remotePath) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+
+    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
+
+    const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz';
+    return path.join(rootPath, remotePath + fileType);
+}
+
+function tarPack(dataLayout, encryption) {
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const pack = tar.pack('/', {
+        dereference: false, // pack the symlink and not what it points to
+        entries: dataLayout.localPaths(),
+        ignoreStatError: (path, err) => {
+            debug(`tarPack: error stat'ing ${path} - ${err.code}`);
+            return err.code === 'ENOENT'; // ignore if file or dir got removed (probably some temporary file)
+        },
+        map: function(header) {
+            header.name = dataLayout.toRemotePath(header.name);
+            // the tar pax format allows us to encode filenames > 100 and size > 8GB (see #640)
+            // https://www.systutorials.com/docs/linux/man/5-star/
+            if (header.size > 8589934590 || header.name > 99) header.pax = { size: header.size };
+            return header;
+        },
+        strict: false // do not error for unknown types (skip fifo, char/block devices)
+    });
+
+    const gzip = zlib.createGzip({});
+    const ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds
+
+    pack.on('error', function (error) {
+        debug('tarPack: tar stream error.', error);
+        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    gzip.on('error', function (error) {
+        debug('tarPack: gzip stream error.', error);
+        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    if (encryption) {
+        const encryptStream = new EncryptStream(encryption);
+        encryptStream.on('error', function (error) {
+            debug('tarPack: encrypt stream error.', error);
+            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+        });
+
+        pack.pipe(gzip).pipe(encryptStream).pipe(ps);
+    } else {
+        pack.pipe(gzip).pipe(ps);
+    }
+
+    return ps;
+}
+
+function tarExtract(inStream, dataLayout, encryption) {
+    assert.strictEqual(typeof inStream, 'object');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const gunzip = zlib.createGunzip({});
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+    const extract = tar.extract('/', {
+        map: function (header) {
+            header.name = dataLayout.toLocalPath(header.name);
+            return header;
+        },
+        dmode: 500 // ensure directory is writable
+    });
+
+    const emitError = once((error) => {
+        inStream.destroy();
+        ps.emit('error', error);
+    });
+
+    inStream.on('error', function (error) {
+        debug('tarExtract: input stream error.', error);
+        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    gunzip.on('error', function (error) {
+        debug('tarExtract: gunzip stream error.', error);
+        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    extract.on('error', function (error) {
+        debug('tarExtract: extract stream error.', error);
+        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    extract.on('finish', function () {
+        debug('tarExtract: done.');
+        // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not extract
+        ps.emit('done');
+    });
+
+    if (encryption) {
+        const decrypt = new DecryptStream(encryption);
+        decrypt.on('error', function (error) {
+            debug('tarExtract: decrypt stream error.', error);
+            emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`));
+        });
+        inStream.pipe(ps).pipe(decrypt).pipe(gunzip).pipe(extract);
+    } else {
+        inStream.pipe(ps).pipe(gunzip).pipe(extract);
+    }
+
+    return ps;
+}
+
+async function download(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
+
+    const backupFilePath = getBackupFilePath(backupConfig, remotePath);
+
+    return new Promise((resolve, reject) => {
+        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+            progressCallback({ message: `Downloading backup ${remotePath}` });
+
+            storage.api(backupConfig.provider).download(backupConfig, backupFilePath, function (error, sourceStream) {
+                if (error) return retryCallback(error);
+
+                const ps = tarExtract(sourceStream, dataLayout, backupConfig.encryption);
+
+                ps.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong
+                    progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` });
+                });
+                ps.on('error', retryCallback);
+                ps.on('done', retryCallback);
+            });
+        }, (error) => {
+            if (error) return reject(error);
+            resolve();
+        });
+    });
+}
+
+async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof dataLayout, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    return new Promise((resolve, reject) => {
+        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+            retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error
+
+            const tarStream = tarPack(dataLayout, backupConfig.encryption);
+
+            tarStream.on('progress', function (progress) {
+                const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0MBps looks wrong
+                progressCallback({ message: `Uploading backup ${transferred}M@${speed}MBps` });
+            });
+            tarStream.on('error', retryCallback); // already returns BoxError
+
+            storage.api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, remotePath), tarStream, retryCallback);
+        }, (error) => {
+            if (error) return reject(error);
+            resolve();
+        });
+    });
+}

src/backups.js

@@ -6,6 +6,7 @@ exports = module.exports = {
     getByTypePaged,
     add,
     update,
+    setState,
     list,
     del,
 
@@ -52,29 +53,24 @@ const assert = require('assert'),
     ejs = require('ejs'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
+    hat = require('./hat.js'),
     locker = require('./locker.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     storage = require('./storage.js'),
-    tasks = require('./tasks.js'),
-    util = require('util');
+    tasks = require('./tasks.js');
 
 const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/cloudron-backup.ejs', { encoding: 'utf8' });
 
-const BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
-
-// helper until all storage providers have been ported
-function maybePromisify(func) {
-    if (util.types.isAsyncFunction(func)) return func;
-    return util.promisify(func);
-}
+const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
 
 function postProcess(result) {
     assert.strictEqual(typeof result, 'object');
 
-    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
+    result.dependsOn = result.dependsOnJson ? safe.JSON.parse(result.dependsOnJson) : [];
+    delete result.dependsOnJson;
 
     result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
     delete result.manifestJson;
@@ -116,9 +112,9 @@ function generateEncryptionKeysSync(password) {
     };
 }
 
-async function add(id, data) {
+async function add(data) {
     assert(data && typeof data === 'object');
-    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof data.remotePath, 'string');
     assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number');
     assert.strictEqual(typeof data.packageVersion, 'string');
     assert.strictEqual(typeof data.type, 'string');
@@ -127,15 +123,19 @@ async function add(id, data) {
     assert(Array.isArray(data.dependsOn));
     assert.strictEqual(typeof data.manifest, 'object');
     assert.strictEqual(typeof data.format, 'string');
+    assert.strictEqual(typeof data.preserveSecs, 'number');
 
     const creationTime = data.creationTime || new Date(); // allow tests to set the time
     const manifestJson = JSON.stringify(data.manifest);
+    const id = `${data.type}_${data.identifier}_v${data.packageVersion}_${hat(256)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying
 
-    const [error] = await safe(database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ]));
+    const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, format, preserveSecs) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        [ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.format, data.preserveSecs ]));
 
     if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Backup already exists');
     if (error) throw error;
+
+    return id;
 }
 
 async function getByIdentifierAndStatePaged(identifier, state, page, perPage) {
@@ -172,19 +172,55 @@ async function getByTypePaged(type, page, perPage) {
     return results;
 }
 
-async function update(id, backup) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof backup, 'object');
+function validateLabel(label) {
+    assert.strictEqual(typeof label, 'string');
 
-    let fields = [ ], values = [ ];
-    for (const p in backup) {
-        fields.push(p + ' = ?');
-        values.push(backup[p]);
+    if (label.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'label too long');
+    if (/[^a-zA-Z0-9._()-]/.test(label)) return new BoxError(BoxError.BAD_FIELD, 'label can only contain alphanumerals, dot, hyphen, brackets or underscore');
+
+    return null;
+}
+
+// this is called by REST API
+async function update(id, data) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof data, 'object');
+
+    let error;
+    if ('label' in data) {
+        error = validateLabel(data.label);
+        if (error) throw error;
+    }
+
+    const fields = [], values = [];
+    for (const p in data) {
+        if (p === 'label' || p === 'preserveSecs') {
+            fields.push(p + ' = ?');
+            values.push(data[p]);
+        }
     }
     values.push(id);
 
+    const backup = await get(id);
+    if (backup === null) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
+
     const result = await database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values);
     if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
+
+    if ('preserveSecs' in data) {
+        // update the dependancies
+        for (const depId of backup.dependsOn) {
+            await database.query('UPDATE backups SET preserveSecs=? WHERE id = ?', [ data.preserveSecs, depId]);
+        }
+    }
+}
+
+async function setState(id, state) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof state, 'string');
+
+    const result = await database.query('UPDATE backups SET state = ? WHERE id = ?', [state, id]);
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
 }
 
 async function startBackupTask(auditSource) {
@@ -205,7 +241,8 @@ async function startBackupTask(auditSource) {
         const errorMessage = error ? error.message : '';
         const timedOut = error ? error.code === tasks.ETIMEOUT : false;
 
-        await safe(eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId, errorMessage, timedOut, backupId }), { debug });
+        const backup = await get(backupId);
+        await safe(eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId, errorMessage, timedOut, backupId, remotePath: backup?.remotePath }), { debug });
     });
 
     return taskId;
@@ -268,15 +305,15 @@ async function startCleanupTask(auditSource) {
 
     const taskId = await tasks.add(tasks.TASK_CLEAN_BACKUPS, []);
 
-    tasks.startTask(taskId, {}, (error, result) => { // result is { removedBoxBackupIds, removedAppBackupIds, removedMailBackupIds, missingBackupIds }
-        eventlog.add(eventlog.ACTION_BACKUP_CLEANUP_FINISH, auditSource, {
+    tasks.startTask(taskId, {}, async (error, result) => { // result is { removedBoxBackupPaths, removedAppBackupPaths, removedMailBackupPaths, missingBackupPaths }
+        await safe(eventlog.add(eventlog.ACTION_BACKUP_CLEANUP_FINISH, auditSource, {
             taskId,
             errorMessage: error ? error.message : null,
-            removedBoxBackupIds: result ? result.removedBoxBackupIds : [],
-            removedMailBackupIds: result ? result.removedMailBackupIds : [],
-            removedAppBackupIds: result ? result.removedAppBackupIds : [],
-            missingBackupIds: result ? result.missingBackupIds : []
-        });
+            removedBoxBackupPaths: result ? result.removedBoxBackupPaths : [],
+            removedMailBackupPaths: result ? result.removedMailBackupPaths : [],
+            removedAppBackupPaths: result ? result.removedAppBackupPaths : [],
+            missingBackupPaths: result ? result.missingBackupPaths : []
+        }), { debug });
     });
 
     return taskId;
@@ -318,8 +355,7 @@ async function testConfig(backupConfig) {
     if ('keepMonthly' in policy && typeof policy.keepMonthly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepMonthly must be a number');
     if ('keepYearly' in policy && typeof policy.keepYearly !== 'number') return new BoxError(BoxError.BAD_FIELD, 'keepYearly must be a number');
 
-    const [error] = await safe(util.promisify(storage.api(backupConfig.provider).testConfig)(backupConfig));
-    return error;
+    await storage.api(backupConfig.provider).testConfig(backupConfig);
 }
 
 // this skips password check since that policy is only at creation time
@@ -329,8 +365,7 @@ async function testProviderConfig(backupConfig) {
     const func = storage.api(backupConfig.provider);
     if (!func) return new BoxError(BoxError.BAD_FIELD, 'unknown storage provider');
 
-    const [error] = await safe(util.promisify(storage.api(backupConfig.provider).testConfig)(backupConfig));
-    return error;
+    await storage.api(backupConfig.provider).testConfig(backupConfig);
 }
 
 async function remount(auditSource) {
@@ -341,5 +376,5 @@ async function remount(auditSource) {
     const func = storage.api(backupConfig.provider);
     if (!func) throw new BoxError(BoxError.BAD_FIELD, 'unknown storage provider');
 
-    await maybePromisify(storage.api(backupConfig.provider).remount)(backupConfig);
+    await storage.api(backupConfig.provider).remount(backupConfig);
 }

src/boxerror.js

@@ -52,7 +52,7 @@ BoxError.INACTIVE = 'Inactive'; // service/volume/mount
 BoxError.INTERNAL_ERROR = 'Internal Error';
 BoxError.INVALID_CREDENTIALS = 'Invalid Credentials';
 BoxError.IPTABLES_ERROR = 'IPTables Error';
-BoxError.LICENSE_ERROR = 'License Error';
+BoxError.LICENSE_ERROR = 'License Error'; // billing or subscription expired
 BoxError.LOGROTATE_ERROR = 'Logrotate Error';
 BoxError.MAIL_ERROR = 'Mail Error';
 BoxError.MOUNT_ERROR = 'Mount Error';

src/changelog.js

@@ -1,6 +1,6 @@
 'use strict';
 
-let assert = require('assert'),
+const assert = require('assert'),
     fs = require('fs'),
     path = require('path');
 
@@ -11,7 +11,7 @@ exports = module.exports = {
 function getChanges(version) {
     assert.strictEqual(typeof version, 'string');
 
-    let changelog = [ ];
+    const changelog = [];
     const lines = fs.readFileSync(path.join(__dirname, '../CHANGES'), 'utf8').split('\n');
 
     version = version.replace(/[+-].*/, ''); // strip prerelease

src/cloudron.js

@@ -32,7 +32,7 @@ const apps = require('./apps.js'),
     constants = require('./constants.js'),
     cron = require('./cron.js'),
     debug = require('debug')('box:cloudron'),
-    delay = require('delay'),
+    delay = require('./delay.js'),
     dns = require('./dns.js'),
     dockerProxy = require('./dockerproxy.js'),
     domains = require('./domains.js'),
@@ -90,10 +90,13 @@ async function notifyUpdate() {
     const version = safe.fs.readFileSync(paths.VERSION_FILE, 'utf8');
     if (version === constants.VERSION) return;
 
-    await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
-
-    const [error] = await safe(tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }));
-    if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist
+    if (!version) {
+        await eventlog.add(eventlog.ACTION_INSTALL_FINISH, AuditSource.CRON, { version: constants.VERSION });
+    } else {
+        await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
+        const [error] = await safe(tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }));
+        if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist
+    }
 
     safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
 }
@@ -152,6 +155,7 @@ async function getConfig() {
     return {
         apiServerOrigin: settings.apiServerOrigin(),
         webServerOrigin: settings.webServerOrigin(),
+        consoleServerOrigin: settings.consoleServerOrigin(),
         adminDomain: settings.dashboardDomain(),
         adminFqdn: settings.dashboardFqdn(),
         mailFqdn: settings.mailFqdn(),
@@ -215,7 +219,7 @@ async function getLogs(unit, options) {
     assert.strictEqual(typeof options.format, 'string');
     assert.strictEqual(typeof options.follow, 'boolean');
 
-    var lines = options.lines === -1 ? '+1' : options.lines,
+    const lines = options.lines === -1 ? '+1' : options.lines,
         format = options.format || 'json',
         follow = options.follow;
 
@@ -290,7 +294,7 @@ async function setDashboardDomain(domain, auditSource) {
 
     await settings.setDashboardLocation(domain, fqdn);
 
-    await safe(appstore.updateCloudron({ domain }));
+    await safe(appstore.updateCloudron({ domain }), { debug });
 
     await eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain, fqdn });
 }

src/collectd/app_cgroup_v2.ejs

@@ -0,0 +1,42 @@
+LoadPlugin "table"
+<Plugin table>
+  <Table "/sys/fs/cgroup/docker/<%= containerId %>/memory.stat">
+    Instance "<%= appId %>-memory"
+    Separator " \\n"
+    <Result>
+      Type gauge
+      InstancesFrom 0
+      ValuesFrom 1
+    </Result>
+  </Table>
+
+  <Table "/sys/fs/cgroup/docker/<%= containerId %>/memory.max">
+    Instance "<%= appId %>-memory"
+    Separator "\\n"
+    <Result>
+      Type gauge
+      InstancePrefix "max_usage_in_bytes"
+      ValuesFrom 0
+    </Result>
+  </Table>
+
+  <Table "/sys/fs/cgroup/docker/<%= containerId %>/cpu.stat">
+    Instance "<%= appId %>-cpu"
+    Separator " \\n"
+    <Result>
+      Type gauge
+      InstancesFrom 0
+      ValuesFrom 1
+    </Result>
+  </Table>
+</Plugin>
+
+<Plugin python>
+  <Module du>
+    <Path>
+        Instance "<%= appId %>"
+        Dir "<%= appDataDir %>"
+    </Path>
+  </Module>
+</Plugin>
+

src/constants.js

@@ -1,6 +1,6 @@
 'use strict';
 
-let fs = require('fs'),
+const fs = require('fs'),
     path = require('path');
 
 const CLOUDRON = process.env.BOX_ENV === 'cloudron',
@@ -63,12 +63,15 @@ exports = module.exports = {
 
     PORT25_CHECK_SERVER: 'port25check.cloudron.io',
 
+    FORUM_URL: 'https://forum.cloudron.io',
+
+    SUPPORT_USERNAME: 'cloudron-support',
     SUPPORT_EMAIL: 'support@cloudron.io',
 
     USER_DIRECTORY_LDAP_DN: 'cn=admin,ou=system,dc=cloudron',
 
     FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
 
-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '7.0.0-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '7.2.0-test'
 };
 

src/cron.js

@@ -66,7 +66,7 @@ async function startJobs() {
 
     const randomTick = Math.floor(60*Math.random());
     gJobs.systemChecks = new CronJob({
-        cronTime: '00 30 2 * * *', // once a day. if you change this interval, change the notification messages with correct duration
+        cronTime: `${randomTick} ${randomTick} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
         onTick: async () => await safe(cloudron.runSystemChecks(), { debug }),
         start: true
     });

src/database.js

@@ -143,9 +143,11 @@ async function importFromFile(file) {
 async function exportToFile(file) {
     assert.strictEqual(typeof file, 'string');
 
-    // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
-    // this option must not be set in production cloudrons which still use the old mysqldump
-    const colStats = (!constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
+    // latest mysqldump enables column stats by default which is not present in 5.7 util
+    const mysqlDumpHelp = safe.child_process.execSync('/usr/bin/mysqldump --help', { encoding: 'utf8' });
+    if (!mysqlDumpHelp) throw new BoxError(BoxError.DATABASE_ERROR, safe.error);
+    const hasColStats = mysqlDumpHelp.includes('column-statistics');
+    const colStats = hasColStats ? '--column-statistics=0' : '';
 
     const cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${colStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
 

src/delay.js

@@ -0,0 +1,13 @@
+'use strict';
+
+exports = module.exports = delay;
+
+const assert = require('assert');
+
+function delay(msecs) {
+    assert.strictEqual(typeof msecs, 'number');
+
+    return new Promise(function (resolve) {
+        setTimeout(resolve, msecs);
+    });
+}

src/dig.js

@@ -34,8 +34,7 @@ async function resolve(hostname, rrtype, options) {
     if (error && error.code === 'ECANCELLED') error.code = 'TIMEOUT';
     if (error) throw error;
 
-    // result is an empty array if there was no error but there is no record. when you query a random
-    // domain, it errors with ENOTFOUND. But if you query an existing domain (A record) but with different
-    // type (CNAME) it is not an error and empty array. for TXT records, result is 2d array of strings
+    // when you query a random record, it errors with ENOTFOUND. But, if you query a record which has a different type
+    // we sometimes get empty array and sometimes ENODATA. for TXT records, result is 2d array of strings
     return result;
 }

src/dns.js

@@ -51,6 +51,7 @@ function api(provider) {
     case 'namecom': return require('./dns/namecom.js');
     case 'namecheap': return require('./dns/namecheap.js');
     case 'netcup': return require('./dns/netcup.js');
+    case 'hetzner': return require('./dns/hetzner.js');
     case 'noop': return require('./dns/noop.js');
     case 'manual': return require('./dns/manual.js');
     case 'wildcard': return require('./dns/wildcard.js');
@@ -84,7 +85,7 @@ function validateHostname(subdomain, domainObject) {
     if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, `subdomain '${subdomain}' is reserved`);
 
     // workaround https://github.com/oncletom/tld.js/issues/73
-    var tmp = hostname.replace('_', '-');
+    const tmp = hostname.replace('_', '-');
     if (!tld.isValid(tmp)) return new BoxError(BoxError.BAD_FIELD, 'Hostname is not a valid domain name');
 
     if (hostname.length > 253) return new BoxError(BoxError.BAD_FIELD, 'Hostname length exceeds 253 characters');
@@ -122,6 +123,9 @@ async function checkDnsRecords(subdomain, domain) {
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
 
+    const cnameRecords = await getDnsRecords(subdomain, domain, 'CNAME');
+    if (cnameRecords.length !== 0) return { needsOverwrite: true };
+
     const ipv4Records = await getDnsRecords(subdomain, domain, 'A');
     const ipv4 = await sysinfo.getServerIPv4();
 
@@ -158,7 +162,7 @@ async function removeDnsRecords(subdomain, domain, type, values) {
     assert.strictEqual(typeof type, 'string');
     assert(Array.isArray(values));
 
-    debug('removeDNSRecord: %s on %s type %s values', subdomain, domain, type, values);
+    debug('removeDNSRecords: %s on %s type %s values', subdomain, domain, type, values);
 
     const domainObject = await domains.get(domain);
     const [error] = await safe(api(domainObject.provider).del(domainObject, subdomain, type, values));
@@ -195,8 +199,8 @@ async function registerLocation(location, options, recordType, recordValue) {
     // get the current record before updating it
     const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, recordType));
     if (getError) {
-        const retryable = getError.reason !== BoxError.ACCESS_DENIED && getError.reason !== BoxError.NOT_FOUND;
-        debug(`registerLocation: Get error. retryable: ${retryable}. ${upsertError.message}`);
+        const retryable = getError.reason !== BoxError.ACCESS_DENIED && getError.reason !== BoxError.NOT_FOUND;  // NOT_FOUND is when zone is not found
+        debug(`registerLocation: Get error. retryable: ${retryable}. ${getError.message}`);
         throw new BoxError(getError.reason, getError.message, { domain: location, retryable });
     }
 
@@ -224,9 +228,18 @@ async function registerLocations(locations, options, progressCallback) {
     const ipv6 = await sysinfo.getServerIPv6();
 
     for (const location of locations) {
-        progressCallback({ message: `Registering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+        const fqdn = `${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}`;
+        progressCallback({ message: `Registering location: ${fqdn}` });
 
         await promiseRetry({ times: 200, interval: 5000, debug, retry: (error) => error.retryable }, async function () {
+            // cname records cannot co-exist with other records
+            const [getError, values] = await safe(getDnsRecords(location.subdomain, location.domain, 'CNAME'));
+            if (!getError && values.length === 1) {
+                if (!options.overwriteDns) throw new BoxError(BoxError.ALREADY_EXISTS, 'DNS CNAME record already exists', { domain: location, retryable: false });
+                debug(`registerLocations: removing CNAME record of ${fqdn}`);
+                await removeDnsRecords(location.subdomain, location.domain, 'CNAME', values);
+            }
+
             await registerLocation(location, options, 'A', ipv4);
             if (ipv6) await registerLocation(location, options, 'AAAA', ipv6);
         });

src/dns/cloudflare.js

@@ -34,7 +34,7 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-async function translateRequestError(result) {
+function translateRequestError(result) {
     assert.strictEqual(typeof result, 'object');
 
     if (result.statusCode === 404) return new BoxError(BoxError.NOT_FOUND, util.format('%s %j', result.statusCode, 'API does not exist'));

src/dns/digitalocean.js

@@ -18,13 +18,12 @@ const assert = require('assert'),
     dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
-    util = require('util'),
     waitForDns = require('./waitfordns.js');
 
 const DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com';
 
 function formatError(response) {
-    return util.format('DigitalOcean DNS error [%s] %j', response.statusCode, response.body);
+    return `DigitalOcean DNS error ${response.statusCode} ${JSON.stringify(response.body)}`;
 }
 
 function removePrivateFields(domainObject) {

src/dns/godaddy.js

@@ -24,12 +24,6 @@ const assert = require('assert'),
 // const GODADDY_API_OTE = 'https://api.ote-godaddy.com/v1/domains';
 const GODADDY_API = 'https://api.godaddy.com/v1/domains';
 
-// this is a workaround for godaddy not having a delete API
-// https://stackoverflow.com/questions/39347464/delete-record-libcloud-godaddy-api
-const GODADDY_INVALID_IP = '0.0.0.0';
-const GODADDY_INVALID_IPv6 = '0:0:0:0:0:0:0:0';
-const GODADDY_INVALID_TXT = '""';
-
 function formatError(response) {
     return util.format(`GoDaddy DNS error [${response.statusCode}] ${response.body.message}`);
 }
@@ -106,6 +100,12 @@ async function get(domainObject, location, type) {
     const values = response.body.map(function (record) { return record.data; });
 
     if (values.length === 1) {
+        // legacy: this was a workaround for godaddy not having a delete API
+        // https://stackoverflow.com/questions/39347464/delete-record-libcloud-godaddy-api
+        const GODADDY_INVALID_IP = '0.0.0.0';
+        const GODADDY_INVALID_IPv6 = '0:0:0:0:0:0:0:0';
+        const GODADDY_INVALID_TXT = '""';
+
         if ((type === 'A' && values[0] === GODADDY_INVALID_IP)
             || (type === 'AAAA' && values[0] === GODADDY_INVALID_IPv6)
             || (type === 'TXT' && values[0] === GODADDY_INVALID_TXT)) return []; // pretend this record doesn't exist
@@ -124,30 +124,24 @@ async function del(domainObject, location, type, values) {
         zoneName = domainObject.zoneName,
         name = dns.getName(domainObject, location, type) || '@';
 
-    debug(`get: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
+    debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
-    if (type !== 'A' && type !== 'AAAA' && type !== 'TXT') throw new BoxError(BoxError.EXTERNAL_ERROR, 'Record deletion is not supported by GoDaddy API');
+    const result = await get(domainObject, location, type);
+    if (result.length === 0) return;
 
-    // check if the record exists at all so that we don't insert the "Dead" record for no reason
-    const existingRecords = await get(domainObject, location, type);
-    if (existingRecords.length === 0) return;
+    const tmp = result.filter(r => !values.includes(r));
 
-    // godaddy does not have a delete API. so fill it up with an invalid IP that we can ignore in future get()
-    const records = [{
-        ttl: 600,
-        data: type === 'A' ? GODADDY_INVALID_IP : (type === 'AAAA' ? GODADDY_INVALID_IPv6 : GODADDY_INVALID_TXT)
-    }];
+    if (tmp.length) return await upsert(domainObject, location, type, tmp); // only remove 'values'
 
-    const [error, response] = await safe(superagent.put(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
+    const [error, response] = await safe(superagent.del(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
         .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
-        .send(records)
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.statusCode === 404) return;
     if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
-    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    if (response.statusCode !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
 }
 
 async function wait(domainObject, subdomain, type, value, options) {

src/dns/hetzner.js

@@ -0,0 +1,259 @@
+'use strict';
+
+exports = module.exports = {
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
+};
+
+const assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    constants = require('../constants.js'),
+    debug = require('debug')('box:dns/digitalocean'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
+    superagent = require('superagent'),
+    waitForDns = require('./waitfordns.js');
+
+const ENDPOINT = 'https://dns.hetzner.com/api/v1';
+
+function formatError(response) {
+    return `Hetzner DNS error ${response.statusCode} ${JSON.stringify(response.body)}`;
+}
+
+function removePrivateFields(domainObject) {
+    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    return domainObject;
+}
+
+function injectPrivateFields(newConfig, currentConfig) {
+    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+}
+
+async function getZone(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+
+    const [error, response] = await safe(superagent.get(`${ENDPOINT}/zones`)
+        .set('Auth-API-Token', domainConfig.token)
+        .query({ search_name: zoneName })
+        .timeout(30 * 1000)
+        .retry(5)
+        .ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 401 || response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+
+    if (!Array.isArray(response.body.zones)) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+
+    const zone = response.body.zones.filter(z => z.name === zoneName);
+    if (zone.length === 0) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+    return zone[0];
+}
+
+async function getZoneRecords(domainConfig, zone, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
+    assert.strictEqual(typeof zone, 'object');
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof type, 'string');
+
+    let page = 1, matchingRecords = [];
+
+    debug(`getInternal: getting dns records of ${zone.name} with ${name} and type ${type}`);
+
+    const perPage = 50;
+
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const [error, response] = await safe(superagent.get(`${ENDPOINT}/records`)
+            .set('Auth-API-Token', domainConfig.token)
+            .query({ zone_id: zone.id, page, per_page: perPage })
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
+
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+        if (response.statusCode === 401 || response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+
+        matchingRecords = matchingRecords.concat(response.body.records.filter(function (record) {
+            return (record.type === type && record.name === name);
+        }));
+
+        if (response.body.records.length < perPage) break;
+
+        ++page;
+    }
+
+    return matchingRecords;
+}
+
+async function upsert(domainObject, location, type, values) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = dns.getName(domainObject, location, type) || '@';
+
+    debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
+
+    const zone = await getZone(domainConfig, zoneName);
+    const records = await getZoneRecords(domainConfig, zone, name, type);
+
+    // used to track available records to update instead of create
+    let i = 0;
+
+    for (let value of values) {
+        const data = {
+            type,
+            name,
+            value,
+            ttl: 60,
+            zone_id: zone.id
+        };
+
+        if (i >= records.length) {
+            const [error, response] = await safe(superagent.post(`${ENDPOINT}/records`)
+                .set('Auth-API-Token', domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
+
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+        } else {
+            const [error, response] = await safe(superagent.put(`${ENDPOINT}/records/${records[i].id}`)
+                .set('Auth-API-Token', domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
+
+            ++i;
+
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+        }
+    }
+
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] = await safe(superagent.del(`${ENDPOINT}/records/${records[j].id}`)
+            .set('Auth-API-Token', domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
+
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
+
+    debug('upsert: completed');
+}
+
+async function get(domainObject, location, type) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = dns.getName(domainObject, location, type) || '@';
+
+    const zone = await getZone(domainConfig, zoneName);
+    const result = await getZoneRecords(domainConfig, zone, name, type);
+
+    return result.map(function (record) { return record.value; });
+}
+
+async function del(domainObject, location, type, values) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = dns.getName(domainObject, location, type) || '@';
+
+    const zone = await getZone(domainConfig, zoneName);
+    const records = await getZoneRecords(domainConfig, zone, name, type);
+    if (records.length === 0) return;
+
+    const matchingRecords = records.filter(function (record) { return values.some(function (value) { return value === record.value; }); });
+    if (matchingRecords.length === 0) return;
+
+    for (const r of matchingRecords) {
+        const [error, response] = await safe(superagent.del(`${ENDPOINT}/records/${r.id}`)
+            .set('Auth-API-Token', domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
+
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) return;
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    }
+}
+
+async function wait(domainObject, subdomain, type, value, options) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+
+    const fqdn = dns.fqdn(subdomain, domainObject);
+
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
+}
+
+async function verifyDomainConfig(domainObject) {
+    assert.strictEqual(typeof domainObject, 'object');
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
+
+    const ip = '127.0.0.1';
+
+    const credentials = {
+        token: domainConfig.token
+    };
+
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
+
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
+
+    // https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are
+    if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('oxygen.ns.hetzner.com') === -1) {
+        debug('verifyDomainConfig: %j does not contain Hetzner NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner');
+    }
+
+    const location = 'cloudrontestdns';
+
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
+
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
+
+    return credentials;
+}

src/dns/namecheap.js

@@ -90,7 +90,7 @@ async function setZone(domainConfig, zoneName, hosts) {
 
     // Map to query params https://www.namecheap.com/support/api/methods/domains-dns/set-hosts.aspx
     hosts.forEach(function (host, i) {
-        var n = i+1; // api starts with 1 not 0
+        const n = i+1; // api starts with 1 not 0
         query['TTL' + n] = '300'; // keep it low
         query['HostName' + n] = host.HostName || host.Name;
         query['RecordType' + n] = host.RecordType || host.Type;

src/dns/route53.js

@@ -255,6 +255,9 @@ async function verifyDomainConfig(domainObject) {
     await upsert(newDomainObject, location, 'A', [ ip ]);
     debug('verifyDomainConfig: Test A record added');
 
+    await get(newDomainObject, location, 'A');
+    debug('verifyDomainConfig: Can list record sets');
+
     await del(newDomainObject, location, 'A', [ ip ]);
     debug('verifyDomainConfig: Test A record removed again');
 

src/dns/vultr.js

@@ -99,7 +99,7 @@ async function upsert(domainObject, location, type, values) {
     for (const value of values) {
         const data = {
             type,
-            ttl: 300 // lowest
+            ttl: 120 // lowest
         };
 
         if (type === 'MX') {

src/dns/waitfordns.js

@@ -25,8 +25,8 @@ async function resolveIp(hostname, type, options) {
     if (cnameResults.length === 0) return cnameResults;
 
     // recurse lookup the CNAME record
-    debug(`resolveIp: Resolving ${hostname}'s CNAME record ${results[0]}`);
-    await dig.resolve(results[0], type, options);
+    debug(`resolveIp: Resolving ${hostname}'s CNAME record ${cnameResults[0]}`);
+    return await dig.resolve(cnameResults[0], type, options);
 }
 
 async function isChangeSynced(hostname, type, value, nameserver) {
@@ -83,15 +83,16 @@ async function waitForDns(hostname, zoneName, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    debug(`waitForDns: ${hostname} to be ${value} in zone ${zoneName}`);
+    debug(`waitForDns: waiting for ${hostname} to be ${value} in zone ${zoneName}`);
 
     await promiseRetry(Object.assign({ debug }, options), async function () {
         const nameservers = await dig.resolve(zoneName, 'NS', { timeout: 5000 });
         if (!nameservers) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to get nameservers');
+        debug(`waitForDns: nameservers are ${JSON.stringify(nameservers)}`);
 
         for (const nameserver of nameservers) {
             const synced = await isChangeSynced(hostname, type, value, nameserver);
-            debug('waitForDns: %s %s ns: %j', hostname, synced ? 'done' : 'not done', nameservers);
+            debug(`waitForDns: ${hostname} at ns ${nameserver}: ${synced ? 'done' : 'not done'} `);
             if (!synced) throw new BoxError(BoxError.EXTERNAL_ERROR, 'ETRYAGAIN');
         }
     });

src/dns/wildcard.js

@@ -80,8 +80,9 @@ async function verifyDomainConfig(domainObject) {
     const fqdn = dns.fqdn(location, domainObject);
 
     const [ipv4Error, ipv4Result] = await safe(dig.resolve(fqdn, 'A', { server: '127.0.0.1', timeout: 5000 }));
-    if (ipv4Error && ipv4Error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv4 of ${fqdn}. Please check if you have set up *.${domainObject.domain} to point to this server's IP`);
-    if (ipv4Error || !ipv4Result) throw new BoxError(BoxError.BAD_FIELD, ipv4Error ? ipv4Error.message : `Unable to resolve IPv4 of ${fqdn}`);
+    if (ipv4Error && (ipv4Error.code === 'ENOTFOUND' || ipv4Error.code === 'ENODATA')) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv4 of ${fqdn}. Please check if you have set up *.${domainObject.domain} to point to this server's IP`);
+    if (ipv4Error) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv4 of ${fqdn}: ${ipv4Error.message}`);
+    if (!ipv4Result) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv4 of ${fqdn}`);
 
     const ipv4 = await sysinfo.getServerIPv4();
     if (ipv4Result.length !== 1 || ipv4 !== ipv4Result[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(ipv4Result)} instead of IPv4 ${ipv4}`);
@@ -89,8 +90,9 @@ async function verifyDomainConfig(domainObject) {
     const ipv6 = await sysinfo.getServerIPv6(); // both should be RFC 5952 format
     if (ipv6) {
         const [ipv6Error, ipv6Result] = await safe(dig.resolve(fqdn, 'AAAA', { server: '127.0.0.1', timeout: 5000 }));
-        if (ipv6Error && ipv6Error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv6 of ${fqdn}`);
-        if (ipv6Error || !ipv6Result) throw new BoxError(BoxError.BAD_FIELD, ipv6Error ? ipv6Error.message : `Unable to resolve IPv6 of ${fqdn}`);
+        if (ipv6Error && (ipv6Error.code === 'ENOTFOUND' || ipv6Error.code === 'ENODATA')) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv6 of ${fqdn}`);
+        if (ipv6Error) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv6 of ${fqdn}: ${ipv6Error.message}`);
+        if (!ipv6Result) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv6 of ${fqdn}`);
 
         if (ipv6Result.length !== 1 || ipv6 !== ipv6Result[0]) throw new BoxError(BoxError.EXTERNAL_ERROR, `Domain resolves to ${JSON.stringify(ipv6Result)} instead of IPv6 ${ipv6}`);
     }

src/docker.js

@@ -20,13 +20,15 @@ exports = module.exports = {
     createSubcontainer,
     inspect,
     getContainerIp,
-    execContainer,
     getEvents,
     memoryUsage,
-    createVolume,
-    removeVolume,
-    clearVolume,
+
     update,
+
+    createExec,
+    startExec,
+    getExec,
+    resizeExec
 };
 
 const apps = require('./apps.js'),
@@ -34,9 +36,8 @@ const apps = require('./apps.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:docker'),
-    delay = require('delay'),
+    delay = require('./delay.js'),
     Docker = require('dockerode'),
-    path = require('path'),
     reverseProxy = require('./reverseproxy.js'),
     services = require('./services.js'),
     settings = require('./settings.js'),
@@ -46,9 +47,6 @@ const apps = require('./apps.js'),
     volumes = require('./volumes.js'),
     _ = require('underscore');
 
-const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
-    MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');
-
 const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
 const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
 
@@ -117,7 +115,7 @@ async function pullImage(manifest) {
         // https://github.com/dotcloud/docker/issues/1074 says each status message
         // is emitted as a chunk
         stream.on('data', function (chunk) {
-            var data = safe.JSON.parse(chunk) || { };
+            const data = safe.JSON.parse(chunk) || { };
             debug('pullImage: %j', data);
 
             // The data.status here is useless because this is per layer as opposed to per image
@@ -194,15 +192,17 @@ async function getAddonMounts(app) {
 
     for (const addon of Object.keys(addons)) {
         switch (addon) {
-        case 'localstorage':
+        case 'localstorage': {
+            const storageDir = await apps.getStorageDir(app);
             mounts.push({
                 Target: '/app/data',
-                Source: `${app.id}-localstorage`,
-                Type: 'volume',
+                Source: storageDir,
+                Type: 'bind',
                 ReadOnly: false
             });
 
             break;
+        }
         case 'tls': {
             const bundle = await reverseProxy.getCertificatePath(app.fqdn, app.domain);
 
@@ -247,12 +247,12 @@ function getAddresses() {
 
     const addresses = [];
     for (const phy of physicalDevices) {
-        const inet = safe.JSON.parse(safe.child_process.execSync(`ip -f inet -j addr show ${phy.name}`, { encoding: 'utf8' }));
+        const inet = safe.JSON.parse(safe.child_process.execSync(`ip -f inet -j addr show dev ${phy.name} scope global`, { encoding: 'utf8' }));
         for (const r of inet) {
             const address = safe.query(r, 'addr_info[0].local');
             if (address) addresses.push(address);
         }
-        const inet6 = safe.JSON.parse(safe.child_process.execSync(`ip -f inet6 -j addr show ${phy.name}`, { encoding: 'utf8' }));
+        const inet6 = safe.JSON.parse(safe.child_process.execSync(`ip -f inet6 -j addr show dev ${phy.name} scope global`, { encoding: 'utf8' }));
         for (const r of inet6) {
             const address = safe.query(r, 'addr_info[0].local');
             if (address) addresses.push(address);
@@ -355,7 +355,8 @@ async function createSubcontainer(app, name, cmd, options) {
             VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ],
             SecurityOpt: [ 'apparmor=docker-cloudron-app' ],
             CapAdd: [],
-            CapDrop: []
+            CapDrop: [],
+            Sysctls: {}
         }
     };
 
@@ -388,7 +389,13 @@ async function createSubcontainer(app, name, cmd, options) {
     const capabilities = manifest.capabilities || [];
 
     // https://docs-stage.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
-    if (capabilities.includes('net_admin')) containerOptions.HostConfig.CapAdd.push('NET_ADMIN', 'NET_RAW');
+    if (capabilities.includes('net_admin')) {
+        containerOptions.HostConfig.CapAdd.push('NET_ADMIN', 'NET_RAW');
+        // ipv6 for new interfaces is disabled in the container. this prevents the openvpn tun device having ipv6
+        // See https://github.com/moby/moby/issues/20569 and https://github.com/moby/moby/issues/33099
+        containerOptions.HostConfig.Sysctls['net.ipv6.conf.all.disable_ipv6'] = '0';
+        containerOptions.HostConfig.Sysctls['net.ipv6.conf.all.forwarding'] = '1';
+    }
     if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping
     if (!capabilities.includes('ping')) containerOptions.HostConfig.CapDrop.push('NET_RAW'); // NET_RAW is included by default by Docker
 
@@ -506,7 +513,7 @@ async function deleteImage(manifest) {
 
     const dockerImage = manifest ? manifest.dockerImage : null;
     if (!dockerImage) return;
-    if (dockerImage.includes('//')) return; // a common mistake is to paste a https:// as docker image. this results in a crash at runtime in dockerode module
+    if (dockerImage.includes('//') || dockerImage.startsWith('/')) return; // a common mistake is to paste a https:// as docker image. this results in a crash at runtime in dockerode module (https://github.com/apocas/dockerode/issues/548)
 
     const removeOptions = {
         force: false, // might be shared with another instance of this app
@@ -552,30 +559,51 @@ async function getContainerIp(containerId) {
     return ip;
 }
 
-async function execContainer(containerId, options) {
+async function createExec(containerId, options) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof options, 'object');
 
     const container = gConnection.getContainer(containerId);
-
-    const [error, exec] = await safe(container.exec(options.execOptions));
+    const [error, exec] = await safe(container.exec(options));
+    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND);
     if (error && error.statusCode === 409) throw new BoxError(BoxError.BAD_STATE, error.message); // container restarting/not running
     if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
 
-    const [startError, stream] = await safe(exec.start(options.startOptions)); /* in hijacked mode, stream is a net.socket */
-    if (startError) throw new BoxError(BoxError.DOCKER_ERROR, startError);
+    return exec.id;
+}
 
-    if (options.rows && options.columns) {
-        // there is a race where resizing too early results in a 404 "no such exec"
-        // https://git.cloudron.io/cloudron/box/issues/549
-        setTimeout(function () {
-            exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
-        }, 2000);
-    }
+async function startExec(execId, options) {
+    assert.strictEqual(typeof execId, 'string');
+    assert.strictEqual(typeof options, 'object');
 
+    const exec = gConnection.getExec(execId);
+    const [error, stream] = await safe(exec.start(options)); /* in hijacked mode, stream is a net.socket */
+    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND);
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
     return stream;
 }
 
+async function getExec(execId) {
+    assert.strictEqual(typeof execId, 'string');
+
+    const exec = gConnection.getExec(execId);
+    const [error, result] = await safe(exec.inspect());
+    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to find exec container ${execId}`);
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
+
+    return { exitCode: result.ExitCode, running: result.Running };
+}
+
+async function resizeExec(execId, options) {
+    assert.strictEqual(typeof execId, 'string');
+    assert.strictEqual(typeof options, 'object');
+
+    const exec = gConnection.getExec(execId);
+    const [error] = await safe(exec.resize(options)); // { h, w }
+    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND);
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
+}
+
 async function getEvents(options) {
     assert.strictEqual(typeof options, 'object');
 
@@ -596,53 +624,6 @@ async function memoryUsage(containerId) {
     return result;
 }
 
-async function createVolume(name, volumeDataDir, labels) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof volumeDataDir, 'string');
-    assert.strictEqual(typeof labels, 'object');
-
-    const volumeOptions = {
-        Name: name,
-        Driver: 'local',
-        DriverOpts: { // https://github.com/moby/moby/issues/19990#issuecomment-248955005
-            type: 'none',
-            device: volumeDataDir,
-            o: 'bind'
-        },
-        Labels: labels
-    };
-
-    // requires sudo because the path can be outside appsdata
-    let [error] = await safe(shell.promises.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}));
-    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`);
-
-    [error] = await safe(gConnection.createVolume(volumeOptions));
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
-}
-
-async function clearVolume(name, options) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof options, 'object');
-
-    let volume = gConnection.getVolume(name);
-    let [error, v] = await safe(volume.inspect());
-    if (error && error.statusCode === 404) return;
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
-
-    const volumeDataDir = v.Options.device;
-    [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, options.removeDirectory ? 'rmdir' : 'clear', volumeDataDir ], {});
-    if (error) throw new BoxError(BoxError.FS_ERROR, error);
-}
-
-// this only removes the volume and not the data
-async function removeVolume(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    let volume = gConnection.getVolume(name);
-    const [error] = await safe(volume.remove());
-    if (error && error.statusCode !== 404) throw new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume: ${error.message}`);
-}
-
 async function info() {
     const [error, result] = await safe(gConnection.info());
     if (error) throw new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker');

src/dockerproxy.js

@@ -41,7 +41,7 @@ async function authorizeApp(req, res, next) {
 }
 
 function attachDockerRequest(req, res, next) {
-    var options = {
+    const options = {
         socketPath: '/var/run/docker.sock',
         method: req.method,
         path: req.url,
@@ -143,8 +143,8 @@ async function start() {
     // eslint-disable-next-line no-unused-vars
     gHttpServer.on('upgrade', function (req, client, head) {
         // Create a new tcp connection to the TCP server
-        var remote = net.connect('/var/run/docker.sock', function () {
-            var upgradeMessage = req.method + ' ' + req.url + ' HTTP/1.1\r\n' +
+        const remote = net.connect('/var/run/docker.sock', function () {
+            let upgradeMessage = req.method + ' ' + req.url + ' HTTP/1.1\r\n' +
                                 `Host: ${req.headers.host}\r\n` +
                                 'Connection: Upgrade\r\n' +
                                 'Upgrade: tcp\r\n';

src/domains.js

@@ -54,6 +54,7 @@ function api(provider) {
     case 'digitalocean': return require('./dns/digitalocean.js');
     case 'gandi': return require('./dns/gandi.js');
     case 'godaddy': return require('./dns/godaddy.js');
+    case 'hetzner': return require('./dns/hetzner.js');
     case 'linode': return require('./dns/linode.js');
     case 'vultr': return require('./dns/vultr.js');
     case 'namecom': return require('./dns/namecom.js');
@@ -165,7 +166,7 @@ async function add(domain, data, auditSource) {
 
     await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
 
-    eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
+    await eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
 
     safe(mail.onDomainAdded(domain)); // background
 }
@@ -247,7 +248,7 @@ async function setConfig(domain, data, auditSource) {
 
     await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
 
-    eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, zoneName, provider });
+    await eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, zoneName, provider });
 }
 
 async function setWellKnown(domain, wellKnown, auditSource) {
@@ -262,7 +263,7 @@ async function setWellKnown(domain, wellKnown, auditSource) {
     if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
     if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 
-    eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, wellKnown });
+    await eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, wellKnown });
 }
 
 async function del(domain, auditSource) {
@@ -287,7 +288,7 @@ async function del(domain, auditSource) {
     if (error) throw error;
     if (results[1].affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
 
-    eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
+    await eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
 
     safe(mail.onDomainRemoved(domain));
 }
@@ -298,13 +299,13 @@ async function clear() {
 
 // removes all fields that are strictly private and should never be returned by API calls
 function removePrivateFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate', 'wellKnown');
+    const result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate', 'wellKnown');
     return api(result.provider).removePrivateFields(result);
 }
 
 // removes all fields that are not accessible by a normal user
 function removeRestrictedFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider');
+    const result = _.pick(domain, 'domain', 'zoneName', 'provider');
 
     result.config = {}; // always ensure config object
 

src/dyndns.js

@@ -27,7 +27,7 @@ async function sync(auditSource) {
         info.ipv4 = info.ip;
         delete info.ip;
     }
-    const ipv4Changed = info.ip !== ipv4;
+    const ipv4Changed = info.ipv4 !== ipv4;
     const ipv6Changed = ipv6 && info.ipv6 !== ipv6; // both should be RFC 5952 format
 
     if (!ipv4Changed && !ipv6Changed) {
@@ -35,7 +35,7 @@ async function sync(auditSource) {
         return;
     }
 
-    debug(`refreshDNS: updating IP from ${info.ip} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`);
+    debug(`refreshDNS: updating IP from ${info.ipv4} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`);
     if (ipv4Changed) await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'A', [ ipv4 ]);
     if (ipv6Changed) await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'AAAA', [ ipv6 ]);
 

src/eventlog.js

@@ -36,6 +36,7 @@ exports = module.exports = {
 
     ACTION_CERTIFICATE_NEW: 'certificate.new',
     ACTION_CERTIFICATE_RENEWAL: 'certificate.renew',
+    ACTION_CERTIFICATE_CLEANUP: 'certificate.cleanup',
 
     ACTION_DASHBOARD_DOMAIN_UPDATE: 'dashboard.domain.update',
 
@@ -43,6 +44,8 @@ exports = module.exports = {
     ACTION_DOMAIN_UPDATE: 'domain.update',
     ACTION_DOMAIN_REMOVE: 'domain.remove',
 
+    ACTION_INSTALL_FINISH: 'cloudron.install.finish',
+
     ACTION_MAIL_LOCATION: 'mail.location',
     ACTION_MAIL_ENABLED: 'mail.enabled',
     ACTION_MAIL_DISABLED: 'mail.disabled',

src/externalldap.js

@@ -20,7 +20,7 @@ const assert = require('assert'),
     debug = require('debug')('box:externalldap'),
     groups = require('./groups.js'),
     ldap = require('ldapjs'),
-    once = require('once'),
+    once = require('./once.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     tasks = require('./tasks.js'),
@@ -42,7 +42,7 @@ function translateUser(ldapConfig, ldapUser) {
 
     // RFC: https://datatracker.ietf.org/doc/html/rfc2798
     return {
-        username: ldapUser[ldapConfig.usernameField],
+        username: ldapUser[ldapConfig.usernameField].toLowerCase(),
         email: ldapUser.mail || ldapUser.mailPrimaryAddress,
         displayName: ldapUser.displayName || ldapUser.cn // user.giveName + ' ' + user.sn
     };
@@ -432,7 +432,7 @@ async function syncGroupUsers(externalLdapConfig, progressCallback) {
 
             debug(`syncGroupUsers: Found member object at ${memberDn} adding to group ${group.name}`);
 
-            const username = result[externalLdapConfig.usernameField];
+            const username = result[externalLdapConfig.usernameField].toLowerCase();
             if (!username) continue;
 
             const [getError, userObject] = await safe(users.getByUsername(username));

src/hush.js

@@ -0,0 +1,225 @@
+'use strict';
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    crypto = require('crypto'),
+    debug = require('debug')('box:hush'),
+    fs = require('fs'),
+    progressStream = require('progress-stream'),
+    TransformStream = require('stream').Transform;
+
+class EncryptStream extends TransformStream {
+    constructor(encryption) {
+        super();
+        this._headerPushed = false;
+        this._iv = crypto.randomBytes(16);
+        this._cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.dataKey, 'hex'), this._iv);
+        this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex'));
+    }
+
+    pushHeaderIfNeeded() {
+        if (!this._headerPushed) {
+            const magic = Buffer.from('CBV2');
+            this.push(magic);
+            this._hmac.update(magic);
+            this.push(this._iv);
+            this._hmac.update(this._iv);
+            this._headerPushed = true;
+        }
+    }
+
+    _transform(chunk, ignoredEncoding, callback) {
+        this.pushHeaderIfNeeded();
+
+        try {
+            const crypt = this._cipher.update(chunk);
+            this._hmac.update(crypt);
+            callback(null, crypt);
+        } catch (error) {
+            callback(new BoxError(BoxError.CRYPTO_ERROR, `Encryption error when updating: ${error.message}`));
+        }
+    }
+
+    _flush(callback) {
+        try {
+            this.pushHeaderIfNeeded(); // for 0-length files
+            const crypt = this._cipher.final();
+            this.push(crypt);
+            this._hmac.update(crypt);
+            callback(null, this._hmac.digest()); // +32 bytes
+        } catch (error) {
+            callback(new BoxError(BoxError.CRYPTO_ERROR, `Encryption error when flushing: ${error.message}`));
+        }
+    }
+}
+
+class DecryptStream extends TransformStream {
+    constructor(encryption) {
+        super();
+        this._key = Buffer.from(encryption.dataKey, 'hex');
+        this._header = Buffer.alloc(0);
+        this._decipher = null;
+        this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex'));
+        this._buffer = Buffer.alloc(0);
+    }
+
+    _transform(chunk, ignoredEncoding, callback) {
+        const needed = 20 - this._header.length; // 4 for magic, 16 for iv
+
+        if (this._header.length !== 20) { // not gotten header yet
+            this._header = Buffer.concat([this._header, chunk.slice(0, needed)]);
+            if (this._header.length !== 20) return callback();
+
+            if (!this._header.slice(0, 4).equals(new Buffer.from('CBV2'))) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid magic in header'));
+
+            const iv = this._header.slice(4);
+            this._decipher = crypto.createDecipheriv('aes-256-cbc', this._key, iv);
+            this._hmac.update(this._header);
+        }
+
+        this._buffer = Buffer.concat([ this._buffer, chunk.slice(needed) ]);
+        if (this._buffer.length < 32) return callback(); // hmac trailer length is 32
+
+        try {
+            const cipherText = this._buffer.slice(0, -32);
+            this._hmac.update(cipherText);
+            const plainText = this._decipher.update(cipherText);
+            this._buffer = this._buffer.slice(-32);
+            callback(null, plainText);
+        } catch (error) {
+            callback(new BoxError(BoxError.CRYPTO_ERROR, `Decryption error: ${error.message}`));
+        }
+    }
+
+    _flush (callback) {
+        if (this._buffer.length !== 32) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (not enough data)'));
+
+        try {
+            if (!this._hmac.digest().equals(this._buffer)) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (mac mismatch)'));
+
+            const plainText = this._decipher.final();
+            callback(null, plainText);
+        } catch (error) {
+            callback(new BoxError(BoxError.CRYPTO_ERROR, `Invalid password or tampered file: ${error.message}`));
+        }
+    }
+}
+
+function encryptFilePath(filePath, encryption) {
+    assert.strictEqual(typeof filePath, 'string');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const encryptedParts = filePath.split('/').map(function (part) {
+        let hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex'));
+        const iv = hmac.update(part).digest().slice(0, 16); // iv has to be deterministic, for our sync (copy) logic to work
+        const cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv);
+        let crypt = cipher.update(part);
+        crypt = Buffer.concat([ iv, crypt, cipher.final() ]);
+
+        return crypt.toString('base64')     // ensures path is valid
+            .replace(/\//g, '-')            // replace '/' of base64 since it conflicts with path separator
+            .replace(/=/g,'');              // strip trailing = padding. this is only needed if we concat base64 strings, which we don't
+    });
+
+    return encryptedParts.join('/');
+}
+
+function decryptFilePath(filePath, encryption) {
+    assert.strictEqual(typeof filePath, 'string');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const decryptedParts = [];
+    for (let part of filePath.split('/')) {
+        part = part + Array(part.length % 4).join('='); // add back = padding
+        part = part.replace(/-/g, '/');                 // replace with '/'
+
+        try {
+            const buffer = Buffer.from(part, 'base64');
+            const iv = buffer.slice(0, 16);
+            let decrypt = crypto.createDecipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv);
+            const plainText = decrypt.update(buffer.slice(16));
+            const plainTextString = Buffer.concat([ plainText, decrypt.final() ]).toString('utf8');
+            const hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex'));
+            if (!hmac.update(plainTextString).digest().slice(0, 16).equals(iv)) return { error: new BoxError(BoxError.CRYPTO_ERROR, `mac error decrypting part ${part} of path ${filePath}`) };
+
+            decryptedParts.push(plainTextString);
+        } catch (error) {
+            debug(`Error decrypting part ${part} of path ${filePath}:`, error);
+            return { error: new BoxError(BoxError.CRYPTO_ERROR, `Error decrypting part ${part} of path ${filePath}: ${error.message}`) };
+        }
+    }
+
+    return { result: decryptedParts.join('/') };
+}
+
+function createReadStream(sourceFile, encryption) {
+    assert.strictEqual(typeof sourceFile, 'string');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const stream = fs.createReadStream(sourceFile);
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+
+    stream.on('error', function (error) {
+        debug(`createReadStream: read stream error at ${sourceFile}`, error);
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, `Error reading ${sourceFile}: ${error.message} ${error.code}`));
+    });
+
+    stream.on('open', () => ps.emit('open'));
+
+    if (encryption) {
+        let encryptStream = new EncryptStream(encryption);
+
+        encryptStream.on('error', function (error) {
+            debug(`createReadStream: encrypt stream error ${sourceFile}`, error);
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Encryption error at ${sourceFile}: ${error.message}`));
+        });
+
+        return stream.pipe(encryptStream).pipe(ps);
+    } else {
+        return stream.pipe(ps);
+    }
+}
+
+function createWriteStream(destFile, encryption) {
+    assert.strictEqual(typeof destFile, 'string');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const stream = fs.createWriteStream(destFile);
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+
+    stream.on('error', function (error) {
+        debug(`createWriteStream: write stream error ${destFile}`, error);
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, `Write error ${destFile}: ${error.message}`));
+    });
+
+    stream.on('finish', function () {
+        debug('createWriteStream: done.');
+        // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not write
+        ps.emit('done');
+    });
+
+    if (encryption) {
+        let decrypt = new DecryptStream(encryption);
+        decrypt.on('error', function (error) {
+            debug(`createWriteStream: decrypt stream error ${destFile}`, error);
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Decryption error at ${destFile}: ${error.message}`));
+        });
+
+        ps.pipe(decrypt).pipe(stream);
+    } else {
+        ps.pipe(stream);
+    }
+
+    return ps;
+}
+
+exports = module.exports = {
+    EncryptStream,
+    DecryptStream,
+
+    encryptFilePath,
+    decryptFilePath,
+
+    createReadStream,
+    createWriteStream
+};

src/infra_version.js

@@ -16,12 +16,12 @@ exports = module.exports = {
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
         'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.4.0@sha256:45817f1631992391d585f171498d257487d872480fd5646723a2b956cc4ef15d' },
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.2.0@sha256:c70d0a1ff7ce8a27dcf7d6f8d1718ddba82ef254079fee5d20fdf074f28cd009' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.3.0@sha256:9f0cfe83310a6f67885c21804c01e73c8d256606217f44dcefb3e05a5402b2c9' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.0@sha256:c8ebdbe2663b26fcd58b1e6b97906b62565adbe4a06256ba0f86114f78b37e6b' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.2.1@sha256:30a5550c41c3be3a01dbf457497ba2f3c05f3121c595a17ef1aacbef931b6114' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.6.0@sha256:67541d29f1ce3ace245b4acdaac28acde3cc15f4f83b98e9b7315930aeb5084c' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.2.1@sha256:75cef64ba4917ba9ec68bc0c9d9ba3a9eeae00a70173cd6d81cc6118038737d9' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.3.1@sha256:b0c564d097b765d4a639330843e2e813d2c87fc8ed34b7df7550bf2c6df0012c' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.1@sha256:f7f689beea07b1c6a9503a48f6fb38ef66e5b22f59fc585a92842a6578b33d46' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.3.0@sha256:89c4e8083631b6d16b5d630d9b27f8ecf301c62f81219d77bd5948a1f4a4375c' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.6.1@sha256:b8b93f007105080d4812a05648e6bc5e15c95c63f511c829cbc14a163d9ea029' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.1.0@sha256:30ec3a01964a1e01396acf265183997c3e17fb07eac1a82b979292cc7719ff4b' },
-        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.6.0@sha256:9c686b10c1a3ba344a743f399d08b4da5426e111f455114980f0ae0229c1ab23' }
+        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.6.1@sha256:ba4b9a1fe274c0ef0a900e5d0deeb8f3da08e118798d1d90fbf995cc0cf6e3a3' }
     }
 };

src/ldap.js

@@ -21,7 +21,7 @@ const addonConfigs = require('./addonconfigs.js'),
     users = require('./users.js'),
     util = require('util');
 
-var gServer = null;
+let gServer = null;
 
 const NOOP = function () {};
 
@@ -113,13 +113,13 @@ function finalSend(results, req, res, next) {
 
     if (cookie && Buffer.isBuffer(cookie)) {
         // we have pagination
-        var first = min;
+        let first = min;
         if (cookie.length !== 0) {
             first = parseInt(cookie.toString(), 10);
         }
-        var last = sendPagedResults(first, first + pageSize);
+        const last = sendPagedResults(first, first + pageSize);
 
-        var resultCookie;
+        let resultCookie;
         if (last < max) {
             resultCookie = Buffer.from(last.toString());
         } else {
@@ -172,7 +172,7 @@ async function userSearch(req, res, next) {
             attributes: {
                 objectclass: ['user', 'inetorgperson', 'person', 'organizationalperson', 'top' ],
                 objectcategory: 'person',
-                cn: user.id,
+                cn: displayName,
                 uid: user.id,
                 entryuuid: user.id, // to support OpenLDAP clients
                 mail: user.email,
@@ -209,37 +209,6 @@ async function groupSearch(req, res, next) {
 
     const results = [];
 
-    // those are the old virtual groups for backwards compat
-    const virtualGroups = [{
-        name: 'users',
-        admin: false
-    }, {
-        name: 'admins',
-        admin: true
-    }];
-
-    virtualGroups.forEach(function (group) {
-        const dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-        const members = group.admin ? usersWithAccess.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : usersWithAccess;
-
-        const obj = {
-            dn: dn.toString(),
-            attributes: {
-                objectclass: ['group'],
-                cn: group.name,
-                memberuid: members.map(function(entry) { return entry.id; }).sort()
-            }
-        };
-
-        // ensure all filter values are also lowercase
-        const lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
-        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
-        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-            results.push(obj);
-        }
-    });
-
     let [errorGroups, resultGroups] = await safe(groups.listWithMembers());
     if (errorGroups) return next(new ldap.OperationsError(errorGroups.toString()));
 
@@ -295,7 +264,7 @@ async function groupAdminsCompare(req, res, next) {
 
     // we only support memberuid here, if we add new group attributes later add them here
     if (req.attribute === 'memberuid') {
-        var user = result.find(function (u) { return u.id === req.value; });
+        const user = result.find(function (u) { return u.id === req.value; });
         if (user && users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) return res.end(true);
     }
 
@@ -558,7 +527,7 @@ async function userSearchSftp(req, res, next) {
     const obj = {
         dn: ldap.parseDN(`cn=${username}@${appFqdn},ou=sftp,dc=cloudron`).toString(),
         attributes: {
-            homeDirectory: app.dataDir ? `/mnt/app-${app.id}` : `/mnt/appsdata/${app.id}/data`, // see also sftp.js
+            homeDirectory: app.storageVolumeId ? `/mnt/app-${app.id}` : `/mnt/appsdata/${app.id}/data`, // see also sftp.js
             objectclass: ['user'],
             objectcategory: 'person',
             cn: user.id,
@@ -635,6 +604,26 @@ async function authenticateMail(req, res, next) {
     await authenticateService(req.dn.rdns[1].attrs.ou.value.toLowerCase(), req.dn, req, res, next);
 }
 
+// https://ldapwiki.com/wiki/RootDSE / RFC 4512 - ldapsearch -x -h "${CLOUDRON_LDAP_SERVER}" -p "${CLOUDRON_LDAP_PORT}" -b  "" -s base
+// ldapjs seems to call this handler for everything when search === ''
+async function maybeRootDSE(req, res, next) {
+    debug(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`);
+
+    if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope
+    if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError());
+
+    res.send({
+        dn: '',
+        attributes: {
+            objectclass: [ 'RootDSE', 'top', 'OpenLDAProotDSE' ],
+            supportedLDAPVersion: '3',
+            vendorName: 'Cloudron LDAP',
+            vendorVersion: '1.0.0'
+        }
+    });
+    res.end();
+}
+
 async function start() {
     const logger = {
         trace: NOOP,
@@ -687,6 +676,16 @@ async function start() {
         res.end();
     });
 
+    // directus looks for the "DN" of the bind user
+    gServer.search('ou=apps,dc=cloudron', function(req, res, next) {
+        const obj = {
+            dn: req.dn.toString(),
+        };
+        finalSend([obj], req, res, next);
+    });
+
+    gServer.search('', maybeRootDSE); // when '', it seems the callback is called for everything else
+
     // just log that an attempt was made to unknown route, this helps a lot during app packaging
     gServer.use(function(req, res, next) {
         debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);

src/locker.js

@@ -1,6 +1,6 @@
 'use strict';
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:locker'),
     EventEmitter = require('events').EventEmitter,
@@ -32,8 +32,7 @@ Locker.prototype.lock = function (operation) {
     this._operation = operation;
     ++this._lockDepth;
     this._timestamp = new Date();
-    var that = this;
-    this._watcherId = setInterval(function () { debug('Lock unreleased %s', that._operation); }, 1000 * 60 * 5);
+    this._watcherId = setInterval(() => { debug('Lock unreleased %s', this._operation); }, 1000 * 60 * 5);
 
     debug('Acquired : %s', this._operation);
 

src/mail.js

@@ -22,6 +22,7 @@ exports = module.exports = {
     setDnsRecords,
 
     validateName,
+    validateDisplayName,
 
     setMailFromValidation,
     setCatchAllAddress,
@@ -65,7 +66,6 @@ exports = module.exports = {
     TYPE_LIST: 'list',
     TYPE_ALIAS: 'alias',
 
-    _validateName: validateName,
     _delByDomain: delByDomain,
     _updateDomain: updateDomain
 };
@@ -169,6 +169,16 @@ function validateName(name) {
     return null;
 }
 
+function validateDisplayName(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name must be atleast 1 char');
+    if (name.length >= 100) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name too long');
+    if (/["<>@]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name is not valid');
+
+    return null;
+}
+
 async function checkOutboundPort25() {
     const relay = {
         value: 'OK',
@@ -197,7 +207,7 @@ async function checkOutboundPort25() {
             relay.status = false;
             relay.value = `Connect to ${constants.PORT25_CHECK_SERVER} failed: ${error.message}. Check if port 25 (outbound) is blocked`;
             client.destroy();
-            relay.errorMessage = `Connect to ${constants.PORT25_CHECK_SERVER} failed.`;
+            relay.errorMessage = `Connect to ${constants.PORT25_CHECK_SERVER} failed: ${error.message}`;
             resolve(relay);
         });
     });
@@ -360,9 +370,9 @@ async function checkMx(domain, mailFqdn) {
 }
 
 function txtToDict(txt) {
-    var dict = {};
+    const dict = {};
     txt.split(';').forEach(function(v) {
-        var p = v.trim().split('=');
+        const p = v.trim().split('=');
         dict[p[0]]=p[1];
     });
     return dict;
@@ -516,7 +526,7 @@ async function checkRblStatus(domain) {
         blacklistedServers.push(result);
     }
 
-    debug(`checkRblStatus: ${domain} (ip: ${ip}) servers: ${JSON.stringify(blacklistedServers)})`);
+    debug(`checkRblStatus: ${domain} (ip: ${ip}) blacklistedServers: ${JSON.stringify(blacklistedServers)})`);
 
     return { status: blacklistedServers.length === 0, ip, servers: blacklistedServers };
 }
@@ -1171,7 +1181,7 @@ async function addMailbox(name, domain, data, auditSource) {
     if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.includes('mailboxes_domain_constraint')) throw new BoxError(BoxError.NOT_FOUND, `no such domain '${domain}'`);
     if (error) throw error;
 
-    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active });
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active });
 }
 
 async function updateMailbox(name, domain, data, auditSource) {
@@ -1196,7 +1206,7 @@ async function updateMailbox(name, domain, data, auditSource) {
     const result = await database.query('UPDATE mailboxes SET ownerId = ?, ownerType = ?, active = ?, enablePop3 = ? WHERE name = ? AND domain = ?', [ ownerId, ownerType, active, enablePop3, name, domain ]);
     if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
 
-    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: mailbox.userId, ownerId, ownerType, active });
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: mailbox.userId, ownerId, ownerType, active });
 }
 
 async function removeSolrIndex(mailbox) {
@@ -1233,7 +1243,7 @@ async function delMailbox(name, domain, options, auditSource) {
     const [error] = await safe(removeSolrIndex(mailbox));
     if (error) debug(`delMailbox: failed to remove solr index: ${error.message}`);
 
-    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
 }
 
 async function getAlias(name, domain) {
@@ -1257,10 +1267,11 @@ async function getAliases(name, domain) {
     return await database.query('SELECT name, domain FROM mailboxes WHERE type = ? AND aliasName = ? AND aliasDomain = ? ORDER BY name', [ exports.TYPE_ALIAS, name, domain ]);
 }
 
-async function setAliases(name, domain, aliases) {
+async function setAliases(name, domain, aliases, auditSource) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert(Array.isArray(aliases));
+    assert.strictEqual(typeof auditSource, 'object');
 
     for (let i = 0; i < aliases.length; i++) {
         const name = aliases[i].name.toLowerCase();
@@ -1278,13 +1289,13 @@ async function setAliases(name, domain, aliases) {
     const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ?', [ name, domain ]);
     if (results.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
 
-    let queries = [];
+    const queries = [];
     // clear existing aliases
     queries.push({ query: 'DELETE FROM mailboxes WHERE aliasName = ? AND aliasDomain = ? AND type = ?', args: [ name, domain, exports.TYPE_ALIAS ] });
-    aliases.forEach(function (alias) {
+    for (const alias of aliases) {
         queries.push({ query: 'INSERT INTO mailboxes (name, domain, type, aliasName, aliasDomain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?, ?, ?)',
             args: [ alias.name, alias.domain, exports.TYPE_ALIAS, name, domain, results[0].ownerId, results[0].ownerType ] });
-    });
+    }
 
     const [error] = await safe(database.transaction(queries));
     if (error && error.code === 'ER_DUP_ENTRY' && error.message.indexOf('mailboxes_name_domain_unique_index') !== -1) {
@@ -1294,6 +1305,8 @@ async function setAliases(name, domain, aliases) {
         throw new BoxError(BoxError.ALREADY_EXISTS, `Mailbox, mailinglist or alias for ${aliasMatch[1]} already exists`);
     }
     if (error) throw error;
+
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, aliases });
 }
 
 async function getLists(domain, search, page, perPage) {
@@ -1348,7 +1361,7 @@ async function addList(name, domain, data, auditSource) {
     if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
     if (error) throw error;
 
-    eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly, active });
+    await eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly, active });
 }
 
 async function updateList(name, domain, data, auditSource) {
@@ -1375,7 +1388,7 @@ async function updateList(name, domain, data, auditSource) {
         [ JSON.stringify(members), membersOnly, active, name, domain ]);
     if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
 
-    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly, active });
+    await eventlog.add(eventlog.ACTION_MAIL_LIST_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly, active });
 }
 
 async function delList(name, domain, auditSource) {
@@ -1387,7 +1400,7 @@ async function delList(name, domain, auditSource) {
     const result = await database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ]);
     if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
 
-    eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
+    await eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
 }
 
 // resolves the members of a list. i.e the lists and aliases

src/middleware/cors.js

@@ -2,7 +2,7 @@
 
 'use strict';
 
-var url = require('url');
+const url = require('url');
 
 /*
  * CORS middleware
@@ -11,19 +11,19 @@ var url = require('url');
  */
 module.exports = function cors(options) {
     options = options || { };
-    var maxAge = options.maxAge || 60 * 60 * 25 * 5; // 5 days
-    var origins = options.origins || [ '*' ];
-    var allowCredentials = options.allowCredentials || false; // cookies
+    const maxAge = options.maxAge || 60 * 60 * 25 * 5; // 5 days
+    const origins = options.origins || [ '*' ];
+    const allowCredentials = options.allowCredentials || false; // cookies
 
     return function (req, res, next) {
-        var requestOrigin = req.headers.origin;
+        let requestOrigin = req.headers.origin;
         if (!requestOrigin) return next();
 
         requestOrigin = url.parse(requestOrigin);
         if (!requestOrigin.host) return res.status(405).send('CORS not allowed from this domain');
 
-        var hostname = requestOrigin.host.split(':')[0]; // remove any port
-        var originAllowed = origins.some(function (o) { return o === '*' || o === hostname; });
+        const hostname = requestOrigin.host.split(':')[0]; // remove any port
+        const originAllowed = origins.some(function (o) { return o === '*' || o === hostname; });
         if (!originAllowed) {
             return res.status(405).send('CORS not allowed from this domain');
         }

src/middleware/multipart.js

@@ -2,19 +2,19 @@
 
 'use strict';
 
-var multiparty = require('multiparty'),
+const multiparty = require('multiparty'),
     timeout = require('connect-timeout');
 
 function _mime(req) {
-  var str = req.headers['content-type'] || '';
-  return str.split(';')[0];
+    const str = req.headers['content-type'] || '';
+    return str.split(';')[0];
 }
 
 module.exports = function multipart(options) {
     return function (req, res, next) {
         if (_mime(req) !== 'multipart/form-data') return res.status(400).send('Invalid content-type. Expecting multipart');
 
-        var form = new multiparty.Form({
+        const form = new multiparty.Form({
             uploadDir: '/tmp',
             keepExtensions: true,
             maxFieldsSize: options.maxFieldsSize || (2 * 1024), // only field size, not files
@@ -29,7 +29,7 @@ module.exports = function multipart(options) {
             req.fields = { };
             req.files = { };
 
-            form.parse(req, function (err, fields, files) {
+            form.parse(req, function (err /*, fields, files */) {
                 if (err) return res.status(400).send('Error parsing request');
                 next(null);
             });

src/mounts.js

@@ -54,6 +54,7 @@ function validateMountOptions(type, options) {
         if (typeof options.remoteDir !== 'string') return new BoxError(BoxError.BAD_FIELD, 'remoteDir is not a string');
         return null;
     case 'ext4':
+    case 'xfs':
         if (typeof options.diskPath !== 'string') return new BoxError(BoxError.BAD_FIELD, 'diskPath is not a string');
         return null;
     default:
@@ -62,7 +63,7 @@ function validateMountOptions(type, options) {
 }
 
 function isManagedProvider(provider) {
-    return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4';
+    return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4' || provider === 'xfs';
 }
 
 function mountObjectFromBackupConfig(backupConfig) {
@@ -83,15 +84,21 @@ function mountObjectFromBackupConfig(backupConfig) {
 function renderMountFile(mount) {
     assert.strictEqual(typeof mount, 'object');
 
-    const {name, hostPath, mountType, mountOptions} = mount;
+    const { name, hostPath, mountType, mountOptions } = mount;
 
     let options, what, type;
     switch (mountType) {
-    case 'cifs':
+    case 'cifs': {
+        const out = safe.child_process.execSync(`systemd-escape -p '${hostPath}'`, { encoding: 'utf8' }); // this ensures uniqueness of creds file
+        if (!out) throw new BoxError(BoxError.FS_ERROR, `Could not determine credentials file name: ${safe.error.message}`);
+        const credentialsFilePath = path.join(paths.CIFS_CREDENTIALS_DIR, `${out.trim()}.cred`);
+        if (!safe.fs.writeFileSync(credentialsFilePath, `username=${mountOptions.username}\npassword=${mountOptions.password}\n`, { mode: 0o600 })) throw new BoxError(BoxError.FS_ERROR, `Could not write credentials file: ${safe.error.message}`);
+
         type = 'cifs';
         what = `//${mountOptions.host}` + path.join('/', mountOptions.remoteDir);
-        options = `username=${mountOptions.username},password=${mountOptions.password},rw,${mountOptions.seal ? 'seal,' : ''}iocharset=utf8,file_mode=0666,dir_mode=0777,uid=yellowtent,gid=yellowtent`;
+        options = `credentials=${credentialsFilePath},rw,${mountOptions.seal ? 'seal,' : ''}iocharset=utf8,file_mode=0666,dir_mode=0777,uid=yellowtent,gid=yellowtent`;
         break;
+    }
     case 'nfs':
         type = 'nfs';
         what = `${mountOptions.host}:${mountOptions.remoteDir}`;
@@ -102,8 +109,15 @@ function renderMountFile(mount) {
         what = mountOptions.diskPath; // like /dev/disk/by-uuid/uuid or /dev/disk/by-id/scsi-id
         options = 'discard,defaults,noatime';
         break;
+    case 'xfs':
+        type = 'xfs';
+        what = mountOptions.diskPath; // like /dev/disk/by-uuid/uuid or /dev/disk/by-id/scsi-id
+        options = 'discard,defaults,noatime';
+        break;
     case 'sshfs': {
         const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${mountOptions.host}`);
+        if (!safe.fs.writeFileSync(keyFilePath, `${mount.mountOptions.privateKey}\n`, { mode: 0o600 })) throw new BoxError(BoxError.FS_ERROR, `Could not write private key: ${safe.error.message}`);
+
         type = 'fuse.sshfs';
         what = `${mountOptions.user}@${mountOptions.host}:${mountOptions.remoteDir}`;
         options = `allow_other,port=${mountOptions.port},IdentityFile=${keyFilePath},StrictHostKeyChecking=no,reconnect`; // allow_other means non-root users can access it
@@ -129,6 +143,11 @@ async function removeMount(mount) {
     if (mountType === 'sshfs') {
         const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${mountOptions.host}`);
         safe.fs.unlinkSync(keyFilePath);
+    } else if (mountType === 'cifs') {
+        const out = safe.child_process.execSync(`systemd-escape -p '${hostPath}'`, { encoding: 'utf8' });
+        if (!out) return;
+        const credentialsFilePath = path.join(paths.CIFS_CREDENTIALS_DIR, `${out.trim()}.cred`);
+        safe.fs.unlinkSync(credentialsFilePath);
     }
 }
 
@@ -181,13 +200,6 @@ async function tryAddMount(mount, options) {
 
     if (constants.TEST) return;
 
-    if (mount.mountType === 'sshfs') {
-        const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${mount.mountOptions.host}`);
-
-        safe.fs.mkdirSync(paths.SSHFS_KEYS_DIR);
-        if (!safe.fs.writeFileSync(keyFilePath, `${mount.mountOptions.privateKey}\n`, { mode: 0o600 })) throw new BoxError(BoxError.FS_ERROR, safe.error);
-    }
-
     const [error] = await safe(shell.promises.sudo('addMount', [ ADD_MOUNT_CMD, renderMountFile(mount), options.timeout ], {}));
     if (error && error.code === 2) throw new BoxError(BoxError.MOUNT_ERROR, 'Failed to unmount existing mount'); // at this point, the old mount config is still there
 

src/nginxconfig.ejs

@@ -88,6 +88,9 @@ server {
     ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256;
     ssl_prefer_server_ciphers off;
 
+    # some apps have underscores in headers. this is apparently disabled by default because of some legacy CGI compat
+    underscores_in_headers on;
+
 <% if (endpoint !== 'ip' && endpoint !== 'setup') { -%>
     # dhparams is generated only after dns setup
     ssl_dhparam /home/yellowtent/platformdata/dhparams.pem;
@@ -116,14 +119,6 @@ server {
     add_header Referrer-Policy $hrp;
     proxy_hide_header Referrer-Policy;
 
-    # workaround caching issue after /logout. if max-age is set, browser uses cache and user thinks they have not logged out
-    # have to keep all the add_header here to avoid repeating all add_header in location block
-    <% if (proxyAuth.enabled) { %>
-    proxy_hide_header Cache-Control;
-    add_header Cache-Control no-cache;
-    add_header Set-Cookie $auth_cookie;
-    <% } %>
-
     # gzip responses that are > 50k and not images
     gzip on;
     gzip_min_length 18k;
@@ -191,10 +186,6 @@ server {
         proxy_pass http://127.0.0.1:3000/well-known-handler/;
     }
 
-<% if (proxyAuth.enabled) { %>
-    proxy_set_header X-App-ID "<%= proxyAuth.id %>";
-<% } %>
-
     # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
     proxy_buffer_size       128k;
     proxy_buffers           4 256k;
@@ -215,7 +206,7 @@ server {
 <% if ( endpoint === 'dashboard' || endpoint === 'setup' ) { %>
     location /api/ {
         proxy_pass   http://127.0.0.1:3000;
-        client_max_body_size 1m;
+        client_max_body_size 2m;
     }
 
     location ~ ^/api/v1/cloudron/login$ {
@@ -225,7 +216,7 @@ server {
     }
 
     # the read timeout is between successive reads and not the whole connection
-    location ~ ^/api/v1/apps/.*/exec$ {
+    location ~ ^/api/v1/apps/.*/exec/.*/start$ {
         proxy_pass   http://127.0.0.1:3000;
         proxy_read_timeout 30m;
     }
@@ -245,6 +236,11 @@ server {
         client_max_body_size 0;
     }
 
+    location ~ ^/api/v1/profile/backgroundImage {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 0;
+    }
+
     # graphite paths (uncomment block below and visit /graphite-web/dashboard)
     # remember to comment out the CSP policy as well to access the graphite dashboard
     # location ~ ^/graphite-web/ {
@@ -288,6 +284,11 @@ server {
     location <%= proxyAuth.location %> {
         auth_request /proxy-auth;
         auth_request_set $auth_cookie $upstream_http_set_cookie;
+
+        auth_request_set $user $upstream_http_x_remote_user;
+        auth_request_set $email $upstream_http_x_remote_email;
+        auth_request_set $name $upstream_http_x_remote_name;
+
         error_page 401 = @proxy-auth-login;
 
         proxy_pass http://<%= ip %>:<%= port %>;
@@ -305,6 +306,20 @@ server {
     }
     <% } %>
 
+    <% if (proxyAuth.enabled) { %>
+    # workaround caching issue after /logout. if max-age is set, browser uses cache and user thinks they have not logged out
+    # IMPORTANT: have to keep all the add_headers at top level here to avoid repeating all the add_headers and proxy_set_headers in location block
+    proxy_hide_header Cache-Control;
+    add_header Cache-Control no-cache;
+    add_header Set-Cookie $auth_cookie;
+
+    # To prevent header spoofing from a client, these variables must always be set (or removed with '') for all proxyAuth routes
+    proxy_set_header X-App-ID "<%= proxyAuth.id %>";
+    proxy_set_header X-Remote-User $user;
+    proxy_set_header X-Remote-Email $email;
+    proxy_set_header X-Remote-Name $name;
+    <% } %>
+
 <% } else if ( endpoint === 'redirect' ) { %>
     location / {
         # redirect everything to the app. this is temporary because there is no way

src/notifications.js

@@ -25,6 +25,7 @@ const assert = require('assert'),
     AuditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     changelog = require('./changelog.js'),
+    constants = require('./constants.js'),
     database = require('./database.js'),
     eventlog = require('./eventlog.js'),
     mailer = require('./mailer.js'),
@@ -160,6 +161,16 @@ async function appUpdated(eventId, app, fromManifest, toManifest) {
     await add(eventId, title, `The application installed at https://${app.fqdn} was updated.\n\nChangelog:\n${toManifest.changelog}\n`);
 }
 
+async function boxInstalled(eventId, version) {
+    assert.strictEqual(typeof eventId, 'string');
+    assert.strictEqual(typeof version, 'string');
+
+    const changes = changelog.getChanges(version.replace(/\.([^.]*)$/, '.0')); // last .0 release
+    const changelogMarkdown = changes.map((m) => `* ${m}\n`).join('');
+
+    await add(eventId, `Cloudron v${version} installed`, `Cloudron v${version} was installed.\n\nPlease join our community at ${constants.FORUM_URL} .\n\nChangelog:\n${changelogMarkdown}\n`);
+}
+
 async function boxUpdated(eventId, oldVersion, newVersion) {
     assert.strictEqual(typeof eventId, 'string');
     assert.strictEqual(typeof oldVersion, 'string');
@@ -267,6 +278,9 @@ async function onEvent(id, action, source, data) {
 
         return await backupFailed(id, data.taskId, data.errorMessage); // only notify for automated backups or timedout
 
+    case eventlog.ACTION_INSTALL_FINISH:
+        return await boxInstalled(id, data.version);
+
     case eventlog.ACTION_UPDATE_FINISH:
         if (!data.errorMessage) return await boxUpdated(id, data.oldVersion, data.newVersion);
         if (data.timedOut) return await boxUpdateError(id, data.errorMessage);

src/once.js

@@ -0,0 +1,14 @@
+'use strict';
+
+exports = module.exports = once;
+
+// https://github.com/isaacs/once/blob/main/LICENSE (ISC)
+function once (fn) {
+    const f = function () {
+        if (f.called) return f.value;
+        f.called = true;
+        return f.value = fn.apply(this, arguments);
+    };
+    f.called = false;
+    return f;
+}

src/paths.js

@@ -1,6 +1,6 @@
 'use strict';
 
-var constants = require('./constants.js'),
+const constants = require('./constants.js'),
     path = require('path');
 
 function baseDir() {
@@ -43,6 +43,7 @@ exports = module.exports = {
     DHPARAMS_FILE: path.join(baseDir(), 'platformdata/dhparams.pem'),
     FEATURES_INFO_FILE: path.join(baseDir(), 'platformdata/features-info.json'),
     VERSION_FILE: path.join(baseDir(), 'platformdata/VERSION'),
+    CIFS_CREDENTIALS_DIR: path.join(baseDir(), 'platformdata/cifs'),
     SSHFS_KEYS_DIR: path.join(baseDir(), 'platformdata/sshfs'),
     SFTP_KEYS_DIR: path.join(baseDir(), 'platformdata/sftp/ssh'),
     SFTP_PUBLIC_KEY_FILE: path.join(baseDir(), 'platformdata/sftp/ssh/ssh_host_rsa_key.pub'),

src/platform.js

@@ -13,7 +13,7 @@ const apps = require('./apps.js'),
     AuditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:platform'),
-    delay = require('delay'),
+    delay = require('./delay.js'),
     fs = require('fs'),
     infra = require('./infra_version.js'),
     locker = require('./locker.js'),

src/promise-retry.js

@@ -3,7 +3,7 @@
 exports = module.exports = promiseRetry;
 
 const assert = require('assert'),
-    delay = require('delay'),
+    delay = require('./delay.js'),
     util = require('util');
 
 async function promiseRetry(options, asyncFunction) {

src/provision.js

@@ -45,10 +45,9 @@ const gProvisionStatus = {
     }
 };
 
-function setProgress(task, message, callback) {
+function setProgress(task, message) {
     debug(`setProgress: ${task} - ${message}`);
     gProvisionStatus[task].message = message;
-    if (callback) callback();
 }
 
 async function ensureDhparams() {
@@ -142,7 +141,7 @@ async function activate(username, password, email, displayName, ip, auditSource)
     const token = { clientId: tokens.ID_WEBADMIN, identifier: ownerId, expires: Date.now() + constants.DEFAULT_TOKEN_EXPIRATION_MSECS };
     const result = await tokens.add(token);
 
-    eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, {});
+    await eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, {});
 
     setImmediate(() => safe(cloudron.onActivated({}), { debug }));
 
@@ -153,21 +152,21 @@ async function activate(username, password, email, displayName, ip, auditSource)
     };
 }
 
-async function restoreTask(backupConfig, backupId, sysinfoConfig, options, auditSource) {
+async function restoreTask(backupConfig, remotePath, sysinfoConfig, options, auditSource) {
     assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof remotePath, 'string');
     assert.strictEqual(typeof sysinfoConfig, 'object');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
     try {
         setProgress('restore', 'Downloading box backup');
-        await backuptask.restore(backupConfig, backupId, (progress) => setProgress('restore', progress.message));
+        await backuptask.restore(backupConfig, remotePath, (progress) => setProgress('restore', progress.message));
 
         setProgress('restore', 'Downloading mail backup');
         const mailBackups = await backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_MAIL, backups.BACKUP_STATE_NORMAL, 1, 1);
         if (mailBackups.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'mail backup not found');
-        const mailRestoreConfig = { backupConfig, backupId: mailBackups[0].id, backupFormat: mailBackups[0].format };
+        const mailRestoreConfig = { backupConfig, remotePath: mailBackups[0].remotePath, backupFormat: mailBackups[0].format };
         await backuptask.downloadMail(mailRestoreConfig, (progress) => setProgress('restore', progress.message));
 
         await ensureDhparams();
@@ -178,7 +177,7 @@ async function restoreTask(backupConfig, backupId, sysinfoConfig, options, audit
         if (!options.skipDnsSetup) await cloudron.setupDnsAndCert(constants.DASHBOARD_LOCATION, dashboardDomain, auditSource, (progress) => setProgress('restore', progress.message));
         await cloudron.setDashboardDomain(dashboardDomain, auditSource);
         await settings.setBackupCredentials(backupConfig); // update just the credentials and not the policy and flags
-        await eventlog.add(eventlog.ACTION_RESTORE, auditSource, { backupId });
+        await eventlog.add(eventlog.ACTION_RESTORE, auditSource, { remotePath });
 
         setImmediate(() => safe(cloudron.onActivated(options), { debug }));
     } catch (error) {
@@ -187,9 +186,9 @@ async function restoreTask(backupConfig, backupId, sysinfoConfig, options, audit
     gProvisionStatus.restore.active = false;
 }
 
-async function restore(backupConfig, backupId, version, sysinfoConfig, options, auditSource) {
+async function restore(backupConfig, remotePath, version, sysinfoConfig, options, auditSource) {
     assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof remotePath, 'string');
     assert.strictEqual(typeof version, 'string');
     assert.strictEqual(typeof sysinfoConfig, 'object');
     assert.strictEqual(typeof options, 'object');
@@ -233,7 +232,7 @@ async function restore(backupConfig, backupId, version, sysinfoConfig, options,
         error = await sysinfo.testIPv4Config(sysinfoConfig);
         if (error) throw error;
 
-        safe(restoreTask(backupConfig, backupId, sysinfoConfig, options, auditSource), { debug }); // now that args are validated run the task in the background
+        safe(restoreTask(backupConfig, remotePath, sysinfoConfig, options, auditSource), { debug }); // now that args are validated run the task in the background
     } catch (error) {
         gProvisionStatus.restore.active = false;
         gProvisionStatus.restore.errorMessage = error ? error.message : '';

src/proxyauth.js

@@ -86,8 +86,12 @@ async function loginPage(req, res, next) {
 
     const title = app.label || app.manifest.title;
 
-    const [iconError, iconBuffer] = await safe(apps.getIcon(app, {}));
-    if (iconError || !iconBuffer) return next(new HttpError(500, 'Icon rendering error'));
+    let [iconError, iconBuffer] = await safe(apps.getIcon(app, {}));
+    if (iconError) return next(new HttpError(500, `Error getting app icon: ${error.message}`));
+    if (!iconBuffer) {
+        iconBuffer = safe.fs.readFileSync(path.join(paths.DASHBOARD_DIR, 'img/appicon_fallback.png'));
+        if (!iconBuffer) return next(new HttpError(500, 'App icon and fallback icon is missing'));
+    }
 
     const icon = 'data:image/png;base64,' + iconBuffer.toString('base64');
     const dashboardOrigin = settings.dashboardOrigin();
@@ -131,6 +135,9 @@ function auth(req, res, next) {
         maxAge: constants.DEFAULT_TOKEN_EXPIRATION_MSECS,
         secure: true
     });
+    res.set('x-remote-user', req.user.username);
+    res.set('x-remote-email', req.user.email);
+    res.set('x-remote-name', req.user.displayName);
 
     return next(new HttpSuccess(200, {}));
 }
@@ -197,11 +204,6 @@ async function logoutPage(req, res, next) {
     res.redirect(302, app.manifest.addons.proxyAuth.path ? '/' : '/login');
 }
 
-function logout(req, res, next) {
-    res.clearCookie('authToken');
-    next(new HttpSuccess(200, {}));
-}
-
 // provides webhooks for the auth wall
 function initializeAuthwallExpressSync() {
     const app = express();
@@ -244,7 +246,7 @@ function initializeAuthwallExpressSync() {
     router.get ('/auth',      jwtVerify, basicAuthVerify, auth); // called by nginx before accessing protected page
     router.post('/login',     json, passwordAuth, authorize);
     router.get ('/logout',    logoutPage);
-    router.post('/logout',    json, logout);
+    router.post('/logout',    json, logoutPage);
 
     return httpServer;
 }

src/reverseproxy.js

@@ -609,7 +609,7 @@ async function renewCerts(options, auditSource, progressCallback) {
     for (const app of allApps) {
         if (app.runState === apps.RSTATE_STOPPED) continue; // do not renew certs of stopped apps
 
-        appDomains = appDomains.concat([{ app, domain: app.domain, fqdn: app.fqdn, type: apps.LOCATION_TYPE_PRIMARY }])
+        appDomains = appDomains.concat([{ app, domain: app.domain, fqdn: app.fqdn, type: apps.LOCATION_TYPE_PRIMARY, nginxConfigFilename: getNginxConfigFilename(app, app.fqdn, apps.LOCATION_TYPE_PRIMARY) }])
             .concat(app.secondaryDomains.map(sd => { return { app, domain: sd.domain, fqdn: sd.fqdn, type: apps.LOCATION_TYPE_SECONDARY, nginxConfigFilename: getNginxConfigFilename(app, sd.fqdn, apps.LOCATION_TYPE_SECONDARY) }; }))
             .concat(app.redirectDomains.map(rd => { return { app, domain: rd.domain, fqdn: rd.fqdn, type: apps.LOCATION_TYPE_REDIRECT, nginxConfigFilename: getNginxConfigFilename(app, rd.fqdn, apps.LOCATION_TYPE_REDIRECT) }; }))
             .concat(app.aliasDomains.map(ad => { return { app, domain: ad.domain, fqdn: ad.fqdn, type: apps.LOCATION_TYPE_ALIAS, nginxConfigFilename: getNginxConfigFilename(app, ad.fqdn, apps.LOCATION_TYPE_ALIAS) }; }));
@@ -657,13 +657,17 @@ async function renewCerts(options, auditSource, progressCallback) {
     }
 }
 
-async function cleanupCerts() {
+async function cleanupCerts(auditSource) {
+    assert.strictEqual(typeof auditSource, 'object');
+
     const filenames = await fs.promises.readdir(paths.NGINX_CERT_DIR);
     const certFilenames = filenames.filter(f => f.endsWith('.cert'));
     const now = new Date();
 
     debug('cleanupCerts: start');
 
+    const fqdns = [];
+
     for (const certFilename of certFilenames) {
         const certFilePath = path.join(paths.NGINX_CERT_DIR, certFilename);
         const notAfter = getExpiryDate(certFilePath);
@@ -681,9 +685,13 @@ async function cleanupCerts() {
             await blobs.del(`${blobs.CERT_PREFIX}-${fqdn}.key`);
             await blobs.del(`${blobs.CERT_PREFIX}-${fqdn}.cert`);
             await blobs.del(`${blobs.CERT_PREFIX}-${fqdn}.csr`);
+
+            fqdns.push(fqdn);
         }
     }
 
+    if (fqdns.length) await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_CLEANUP, auditSource, { domains: fqdns }));
+
     debug('cleanupCerts: done');
 }
 
@@ -693,7 +701,7 @@ async function checkCerts(options, auditSource, progressCallback) {
     assert.strictEqual(typeof progressCallback, 'function');
 
     await renewCerts(options, auditSource, progressCallback);
-    await cleanupCerts();
+    await cleanupCerts(auditSource);
 }
 
 function removeAppConfigs() {

src/routes/apps.js

@@ -35,14 +35,18 @@ exports = module.exports = {
     setMailbox,
     setInbox,
     setLocation,
-    setDataDir,
+    setStorage,
     setMounts,
 
     stop,
     start,
     restart,
-    exec,
-    execWebSocket,
+
+    createExec,
+    startExec,
+    startExecWebSocket,
+    getExec,
+
     checkForUpdates,
 
     clone,
@@ -50,6 +54,8 @@ exports = module.exports = {
     uploadFile,
     downloadFile,
 
+    updateBackup,
+
     getLimits,
 
     load
@@ -367,6 +373,7 @@ async function setMailbox(req, res, next) {
     if (req.body.enable) {
         if (req.body.mailboxName !== null && typeof req.body.mailboxName !== 'string') return next(new HttpError(400, 'mailboxName must be a string'));
         if (typeof req.body.mailboxDomain !== 'string') return next(new HttpError(400, 'mailboxDomain must be a string'));
+        if ('mailboxDisplayName' in req.body && typeof req.body.mailboxDisplayName !== 'string') return next(new HttpError(400, 'mailboxDisplayName must be a string'));
     }
 
     const [error, result] = await safe(apps.setMailbox(req.app, req.body, AuditSource.fromRequest(req)));
@@ -425,13 +432,18 @@ async function setLocation(req, res, next) {
     next(new HttpSuccess(202, { taskId: result.taskId }));
 }
 
-async function setDataDir(req, res, next) {
+async function setStorage(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.app, 'object');
 
-    if (req.body.dataDir !== null && typeof req.body.dataDir !== 'string') return next(new HttpError(400, 'dataDir must be a string'));
+    const { storageVolumeId, storageVolumePrefix } = req.body;
 
-    const [error, result] = await safe(apps.setDataDir(req.app, req.body.dataDir, AuditSource.fromRequest(req)));
+    if (storageVolumeId !== null) {
+        if (typeof storageVolumeId !== 'string') return next(new HttpError(400, 'storageVolumeId must be a string'));
+        if (typeof storageVolumePrefix !== 'string') return next(new HttpError(400, 'storageVolumePrefix must be a string'));
+    }
+
+    const [error, result] = await safe(apps.setStorage(req.app, storageVolumeId, storageVolumePrefix, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -479,8 +491,8 @@ async function importApp(req, res, next) {
 
     const data = req.body;
 
-    if ('backupId' in data) { // if not provided, we import in-place
-        if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string'));
+    if ('remotePath' in data) { // if not provided, we import in-place
+        if (typeof data.remotePath !== 'string') return next(new HttpError(400, 'remotePath must be string'));
         if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
 
         if ('backupConfig' in data && typeof data.backupConfig !== 'object') return next(new HttpError(400, 'backupConfig must be an object'));
@@ -517,7 +529,7 @@ async function clone(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.app, 'object');
 
-    var data = req.body;
+    const data = req.body;
 
     if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be a string'));
     if (typeof data.subdomain !== 'string') return next(new HttpError(400, 'subdomain is required'));
@@ -642,7 +654,7 @@ async function getLogStream(req, res, next) {
     res.write('retry: 3000\n');
     res.on('close', logStream.close);
     logStream.on('data', function (data) {
-        var obj = JSON.parse(data);
+        const obj = JSON.parse(data);
         res.write(sse(obj.realtimeTimestamp, JSON.stringify(obj))); // send timestamp as id
     });
     logStream.on('end', res.end.bind(res));
@@ -674,19 +686,19 @@ async function getLogs(req, res, next) {
 }
 
 function demuxStream(stream, stdin) {
-    var header = null;
+    let header = null;
 
     stream.on('readable', function() {
         header = header || stream.read(4);
 
         while (header !== null) {
-            var length = header.readUInt32BE(0);
+            const length = header.readUInt32BE(0);
             if (length === 0) {
                 header = null;
                 return stdin.end(); // EOF
             }
 
-            var payload = stream.read(length);
+            const payload = stream.read(length);
 
             if (payload === null) break;
             stdin.write(payload);
@@ -695,14 +707,29 @@ function demuxStream(stream, stdin) {
     });
 }
 
-async function exec(req, res, next) {
+async function createExec(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
+    assert.strictEqual(typeof req.body, 'object');
 
-    let cmd = null;
-    if (req.query.cmd) {
-        cmd = safe.JSON.parse(req.query.cmd);
-        if (!Array.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
+    if ('cmd' in req.body) {
+        if (!Array.isArray(req.body.cmd) || req.body.cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
     }
+    const cmd = req.body.cmd || null;
+
+    if ('tty' in req.body && typeof req.body.tty !== 'boolean') return next(new HttpError(400, 'tty must be boolean'));
+    const tty = !!req.body.tty;
+
+    if (safe.query(req.app, 'manifest.addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is requied to exec app with docker addon'));
+
+    const [error, id] = await safe(apps.createExec(req.app, { cmd, tty }));
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, { id }));
+}
+
+async function startExec(req, res, next) {
+    assert.strictEqual(typeof req.app, 'object');
+    assert.strictEqual(typeof req.params.execId, 'string');
 
     const columns = req.query.columns ? parseInt(req.query.columns, 10) : null;
     if (isNaN(columns)) return next(new HttpError(400, 'columns must be a number'));
@@ -717,7 +744,7 @@ async function exec(req, res, next) {
     // in a badly configured reverse proxy, we might be here without an upgrade
     if (req.headers['upgrade'] !== 'tcp') return next(new HttpError(404, 'exec requires TCP upgrade'));
 
-    const [error, duplexStream] = await safe(apps.exec(req.app, { cmd: cmd, rows: rows, columns: columns, tty: tty }));
+    const [error, duplexStream] = await safe(apps.startExec(req.app, req.params.execId, { rows, columns, tty }));
     if (error) return next(BoxError.toHttpError(error));
 
     req.clearTimeout();
@@ -735,14 +762,9 @@ async function exec(req, res, next) {
     }
 }
 
-async function execWebSocket(req, res, next) {
+async function startExecWebSocket(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
-
-    let cmd = null;
-    if (req.query.cmd) {
-        cmd = safe.JSON.parse(req.query.cmd);
-        if (!Array.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
-    }
+    assert.strictEqual(typeof req.params.execId, 'string');
 
     const columns = req.query.columns ? parseInt(req.query.columns, 10) : null;
     if (isNaN(columns)) return next(new HttpError(400, 'columns must be a number'));
@@ -755,7 +777,7 @@ async function execWebSocket(req, res, next) {
     // in a badly configured reverse proxy, we might be here without an upgrade
     if (req.headers['upgrade'] !== 'websocket') return next(new HttpError(404, 'exec requires websocket'));
 
-    const [error, duplexStream] = await safe(apps.exec(req.app, { cmd: cmd, rows: rows, columns: columns, tty: tty }));
+    const [error, duplexStream] = await safe(apps.startExec(req.app, req.params.execId, { rows, columns, tty }));
     if (error) return next(BoxError.toHttpError(error));
 
     req.clearTimeout();
@@ -783,6 +805,15 @@ async function execWebSocket(req, res, next) {
     });
 }
 
+async function getExec(req, res, next) {
+    assert.strictEqual(typeof req.app, 'object');
+    assert.strictEqual(typeof req.params.execId, 'string');
+
+    const [error, result] = await safe(apps.getExec(req.app, req.params.execId));
+    if (error) return next(BoxError.toHttpError(error));
+    next(new HttpSuccess(200, result)); // { exitCode, running }
+}
+
 async function listBackups(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
 
@@ -798,6 +829,21 @@ async function listBackups(req, res, next) {
     next(new HttpSuccess(200, { backups: result }));
 }
 
+async function updateBackup(req, res, next) {
+    assert.strictEqual(typeof req.app, 'object');
+    assert.strictEqual(typeof req.params.backupId, 'string');
+    assert.strictEqual(typeof req.body, 'object');
+
+    const { label, preserveSecs } = req.body;
+    if (typeof label !== 'string') return next(new HttpError(400, 'label must be a string'));
+    if (typeof preserveSecs !== 'number') return next(new HttpError(400, 'preserveSecs must be a number'));
+
+    const [error] = await safe(apps.updateBackup(req.app, req.params.backupId, { label, preserveSecs }));
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, {}));
+}
+
 async function uploadFile(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
 

src/routes/appstore.js

@@ -5,7 +5,7 @@ exports = module.exports = {
     getApp,
     getAppVersion,
 
-    createUserToken,
+    getWebToken,
     registerCloudron,
     getSubscription
 };
@@ -45,11 +45,11 @@ async function getAppVersion(req, res, next) {
     next(new HttpSuccess(200, manifest));
 }
 
-async function createUserToken(req, res, next) {
-    const [error, accessToken] = await safe(appstore.createUserToken());
+async function getWebToken(req, res, next) {
+    const [error, accessToken] = await safe(appstore.getWebToken());
     if (error) return next(BoxError.toHttpError(error));
 
-    next(new HttpSuccess(201, { accessToken }));
+    next(new HttpSuccess(200, { accessToken }));
 }
 
 async function registerCloudron(req, res, next) {

src/routes/backups.js

@@ -2,12 +2,14 @@
 
 exports = module.exports = {
     list,
-    startBackup,
+    update,
+    create,
     cleanup,
-    remount
+    remount,
 };
 
-const AuditSource = require('../auditsource.js'),
+const assert = require('assert'),
+    AuditSource = require('../auditsource.js'),
     backups = require('../backups.js'),
     BoxError = require('../boxerror.js'),
     HttpError = require('connect-lastmile').HttpError,
@@ -27,7 +29,21 @@ async function list(req, res, next) {
     next(new HttpSuccess(200, { backups: result }));
 }
 
-async function startBackup(req, res, next) {
+async function update(req, res, next) {
+    assert.strictEqual(typeof req.params.backupId, 'string');
+    assert.strictEqual(typeof req.body, 'object');
+
+    const { label, preserveSecs } = req.body;
+    if (typeof label !== 'string') return next(new HttpError(400, 'label must be a string'));
+    if (typeof preserveSecs !== 'number') return next(new HttpError(400, 'preserveSecs must be a number'));
+
+    const [error] = await safe(backups.update(req.params.backupId, { label, preserveSecs }));
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, {}));
+}
+
+async function create(req, res, next) {
     const [error, taskId] = await safe(backups.startBackupTask(AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 

src/routes/cloudron.js

@@ -63,7 +63,7 @@ async function login(req, res, next) {
     [error, token] = await safe(tokens.add({ clientId: type, identifier: req.user.id, expires: Date.now() + constants.DEFAULT_TOKEN_EXPIRATION_MSECS }));
     if (error) return next(new HttpError(500, error));
 
-    eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+    await eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
     if (!req.user.ghost) safe(users.notifyLoginLocation(req.user, ip, userAgent, auditSource), { debug });
 
@@ -73,7 +73,7 @@ async function login(req, res, next) {
 async function logout(req, res) {
     assert.strictEqual(typeof req.access_token, 'string');
 
-    eventlog.add(eventlog.ACTION_USER_LOGOUT, AuditSource.fromRequest(req), { userId: req.user.id, user: users.removePrivateFields(req.user) });
+    await eventlog.add(eventlog.ACTION_USER_LOGOUT, AuditSource.fromRequest(req), { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
     await safe(tokens.delByAccessToken(req.access_token));
     res.redirect('/login.html');
@@ -253,7 +253,7 @@ async function getLogStream(req, res, next) {
     res.write('retry: 3000\n');
     res.on('close', logStream.close);
     logStream.on('data', function (data) {
-        var obj = JSON.parse(data);
+        const obj = JSON.parse(data);
         res.write(sse(obj.monotonicTimestamp, JSON.stringify(obj))); // send timestamp as id
     });
     logStream.on('end', res.end.bind(res));

src/routes/graphs.js

@@ -4,7 +4,7 @@ exports = module.exports = {
     getGraphs
 };
 
-var middleware = require('../middleware/index.js'),
+const middleware = require('../middleware/index.js'),
     HttpError = require('connect-lastmile').HttpError,
     url = require('url');
 
@@ -13,7 +13,7 @@ var middleware = require('../middleware/index.js'),
 const graphiteProxy = middleware.proxy(url.parse('http://127.0.0.1:8417'));
 
 function getGraphs(req, res, next) {
-    var parsedUrl = url.parse(req.url, true /* parseQueryString */);
+    const parsedUrl = url.parse(req.url, true /* parseQueryString */);
     delete parsedUrl.query['access_token'];
     delete req.headers['authorization'];
     delete req.headers['cookies'];

src/routes/mail.js

@@ -233,7 +233,7 @@ async function setAliases(req, res, next) {
         if (typeof alias.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
     }
 
-    const [error] = await safe(mail.setAliases(req.params.name, req.params.domain, req.body.aliases));
+    const [error] = await safe(mail.setAliases(req.params.name, req.params.domain, req.body.aliases, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(202));
@@ -307,7 +307,7 @@ async function updateList(req, res, next) {
     if (!Array.isArray(req.body.members)) return next(new HttpError(400, 'members must be a string'));
     if (req.body.members.length === 0) return next(new HttpError(400, 'list must have atleast one member'));
 
-    for (var i = 0; i < req.body.members.length; i++) {
+    for (let i = 0; i < req.body.members.length; i++) {
         if (typeof req.body.members[i] !== 'string') return next(new HttpError(400, 'member must be a string'));
     }
     if (typeof req.body.membersOnly !== 'boolean') return next(new HttpError(400, 'membersOnly must be a boolean'));

src/routes/profile.js

@@ -6,6 +6,8 @@ exports = module.exports = {
     update,
     getAvatar,
     setAvatar,
+    getBackgroundImage,
+    setBackgroundImage,
     setPassword,
     setTwoFactorAuthenticationSecret,
     enableTwoFactorAuthentication,
@@ -37,10 +39,14 @@ async function authorize(req, res, next) {
 async function get(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    const [error, avatarUrl] = await safe(users.getAvatarUrl(req.user));
+    let [error, avatarUrl] = await safe(users.getAvatarUrl(req.user));
     if (error) return next(BoxError.toHttpError(error));
     if (!avatarUrl) return next(new HttpError(404, 'User not found'));
 
+    let backgroundImage;
+    [error, backgroundImage] = await safe(users.getBackgroundImage(req.user.id));
+    if (error) return next(BoxError.toHttpError(error));
+
     next(new HttpSuccess(200, {
         id: req.user.id,
         username: req.user.username,
@@ -50,6 +56,7 @@ async function get(req, res, next) {
         twoFactorAuthenticationEnabled: req.user.twoFactorAuthenticationEnabled,
         role: req.user.role,
         source: req.user.source,
+        hasBackgroundImage: !!backgroundImage,
         avatarUrl
     }));
 }
@@ -107,6 +114,31 @@ async function getAvatar(req, res, next) {
     res.send(avatar);
 }
 
+async function setBackgroundImage(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    let backgroundImage = null;
+
+    if (req.files && req.files.backgroundImage) {
+        backgroundImage = safe.fs.readFileSync(req.files.backgroundImage.path);
+        if (!backgroundImage) return next(BoxError.toHttpError(new BoxError(BoxError.FS_ERROR, safe.error.message)));
+    }
+
+    const [error] = await safe(users.setBackgroundImage(req.user.id, backgroundImage));
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(202, {}));
+}
+
+async function getBackgroundImage(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    const [error, backgroundImage] = await safe(users.getBackgroundImage(req.user.id));
+    if (error) return next(BoxError.toHttpError(error));
+
+    res.send(backgroundImage);
+}
+
 async function setPassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.user, 'object');

src/routes/provision.js

@@ -106,7 +106,7 @@ async function restore(req, res, next) {
     if (typeof backupConfig.format !== 'string') return next(new HttpError(400, 'format must be a string'));
     if ('acceptSelfSignedCerts' in backupConfig && typeof backupConfig.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
 
-    if (typeof req.body.backupId !== 'string') return next(new HttpError(400, 'backupId must be a string'));
+    if (typeof req.body.remotePath !== 'string') return next(new HttpError(400, 'remotePath must be a string'));
     if (typeof req.body.version !== 'string') return next(new HttpError(400, 'version must be a string'));
 
     if ('sysinfoConfig' in req.body && typeof req.body.sysinfoConfig !== 'object') return next(new HttpError(400, 'sysinfoConfig must be an object'));
@@ -116,7 +116,7 @@ async function restore(req, res, next) {
         skipDnsSetup: req.body.skipDnsSetup || false
     };
 
-    const [error] = await safe(provision.restore(backupConfig, req.body.backupId, req.body.version, req.body.sysinfoConfig || { provider: 'generic' }, options, AuditSource.fromRequest(req)));
+    const [error] = await safe(provision.restore(backupConfig, req.body.remotePath, req.body.version, req.body.sysinfoConfig || { provider: 'generic' }, options, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(200, {}));

src/routes/services.js

@@ -107,7 +107,7 @@ async function getLogStream(req, res, next) {
     res.write('retry: 3000\n');
     res.on('close', logStream.close);
     logStream.on('data', function (data) {
-        var obj = JSON.parse(data);
+        const obj = JSON.parse(data);
         res.write(sse(obj.monotonicTimestamp, JSON.stringify(obj))); // send timestamp as id
     });
     logStream.on('end', res.end.bind(res));

src/routes/tasks.js

@@ -104,7 +104,7 @@ async function getLogStream(req, res, next) {
     res.write('retry: 3000\n');
     res.on('close', logStream.close);
     logStream.on('data', function (data) {
-        var obj = JSON.parse(data);
+        const obj = JSON.parse(data);
         res.write(sse(obj.monotonicTimestamp, JSON.stringify(obj))); // send timestamp as id
     });
     logStream.on('end', res.end.bind(res));

src/routes/test/api-test.js

@@ -7,7 +7,7 @@
 'use strict';
 
 const common = require('./common.js'),
-    delay = require('delay'),
+    delay = require('../../delay.js'),
     expect = require('expect.js'),
     superagent = require('superagent'),
     tokens = require('../../tokens.js');

src/routes/test/apps-test.js

@@ -29,14 +29,14 @@ const apps = require('../../apps.js'),
     tokens = require('../../tokens.js'),
     url = require('url');
 
-var SERVER_URL = 'http://localhost:' + constants.PORT;
+const SERVER_URL = 'http://localhost:' + constants.PORT;
 
 const docker = new Docker({ socketPath: '/var/run/docker.sock' });
 
 // Test image information
-var TEST_IMAGE_REPO = 'docker.io/cloudron/io.cloudron.testapp';
-var TEST_IMAGE_TAG = '20201121-223249-985e86ebb';
-var TEST_IMAGE = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
+const TEST_IMAGE_REPO = 'docker.io/cloudron/io.cloudron.testapp';
+const TEST_IMAGE_TAG = '20201121-223249-985e86ebb';
+const TEST_IMAGE = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
 
 const DOMAIN_0 = {
     domain: 'example-apps-test.com',
@@ -48,11 +48,12 @@ const DOMAIN_0 = {
     tlsConfig: { provider: 'fallback' }
 };
 
-var APP_STORE_ID = 'test', APP_ID;
-var APP_SUBDOMAIN = 'appssubdomain';
-var APP_SUBDOMAIN_NEW = 'appssubdomainnew';
+const APP_STORE_ID = 'test';
+let APP_ID;
+const APP_SUBDOMAIN = 'appssubdomain';
+const APP_SUBDOMAIN_NEW = 'appssubdomainnew';
 
-var APP_MANIFEST = JSON.parse(fs.readFileSync(__dirname + '/../../../../test-app/CloudronManifest.json', 'utf8'));
+const APP_MANIFEST = JSON.parse(fs.readFileSync(__dirname + '/../../../../test-app/CloudronManifest.json', 'utf8'));
 APP_MANIFEST.dockerImage = TEST_IMAGE;
 
 const USERNAME = 'superadmin';
@@ -62,17 +63,17 @@ const EMAIL ='admin@me.com';
 const USER_1_APPSTORE_TOKEN = 'appstoretoken';
 const USERNAME_1 = 'user';
 const EMAIL_1 ='user@me.com';
-var user_1_id = null;
+const user_1_id = null;
 
 // authentication token
-var token = null;
-var token_1 = null;
+const token = null;
+const token_1 = null;
 
 let KEY, CERT;
 let appstoreIconServer = hock.createHock({ throwOnUnmatched: false });
 
 function checkRedis(containerId, done) {
-    var redisIp, exportedRedisPort;
+    let redisIp, exportedRedisPort;
 
     docker.getContainer(containerId).inspect(function (error, data) {
         expect(error).to.not.be.ok();
@@ -191,7 +192,7 @@ function startBox(done) {
                 .get('/api/v1/apps/' + APP_STORE_ID + '/versions/' + APP_MANIFEST.version + '/icon')
                 .replyWithFile(200, path.resolve(__dirname, '../../../assets/avatar.png'));
 
-            var port = parseInt(url.parse(settings.apiServerOrigin()).port, 10);
+            const port = parseInt(url.parse(settings.apiServerOrigin()).port, 10);
             http.createServer(appstoreIconServer.handler).listen(port, callback);
         },
 
@@ -226,7 +227,7 @@ function stopBox(done) {
     ], done);
 }
 
-describe('App API', function () {
+xdescribe('App API', function () {
     let taskId = '';
 
     before(startBox);
@@ -384,7 +385,7 @@ describe('App API', function () {
         });
 
         it('app install fails because manifest download fails', function (done) {
-            var fake = nock(settings.apiServerOrigin()).get('/api/v1/apps/test').reply(404, {});
+            const fake = nock(settings.apiServerOrigin()).get('/api/v1/apps/test').reply(404, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
@@ -397,7 +398,7 @@ describe('App API', function () {
         });
 
         it('app install fails due to purchase failure', function (done) {
-            var fake1 = nock(settings.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            const fake1 = nock(settings.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
@@ -410,10 +411,10 @@ describe('App API', function () {
         });
 
         it('app install succeeds with purchase', async function () {
-            var fake1 = nock(settings.apiServerOrigin()).get('/api/v1/apps/' + APP_STORE_ID).reply(200, { manifest: APP_MANIFEST });
-            var fake2 = nock(settings.apiServerOrigin()).post(function (uri) { return uri.indexOf('/api/v1/cloudronapps') >= 0; }, (body) => body.appstoreId === APP_STORE_ID && body.manifestId === APP_MANIFEST.id && body.appId).reply(201, { });
+            const fake1 = nock(settings.apiServerOrigin()).get('/api/v1/apps/' + APP_STORE_ID).reply(200, { manifest: APP_MANIFEST });
+            const fake2 = nock(settings.apiServerOrigin()).post(function (uri) { return uri.indexOf('/api/v1/cloudronapps') >= 0; }, (body) => body.appstoreId === APP_STORE_ID && body.manifestId === APP_MANIFEST.id && body.appId).reply(201, { });
 
-            await settings.setCloudronToken(USER_1_APPSTORE_TOKEN);
+            await settings.setAppstoreApiToken(USER_1_APPSTORE_TOKEN);
 
             const res = await superagent.post(SERVER_URL + '/api/v1/apps/install')
                 .query({ access_token: token })
@@ -549,7 +550,7 @@ describe('App API', function () {
         });
 
         xit('tcp port mapping works', function (done) {
-            var client = net.connect(7171);
+            const client = net.connect(7171);
             client.on('data', function (data) {
                 expect(data.toString()).to.eql('ECHO_SERVER_PORT=7171');
                 done();
@@ -586,7 +587,7 @@ describe('App API', function () {
                 .query({ access_token: token })
                 .buffer(false)
                 .end(function (err, res) {
-                    var data = '';
+                    const data = '';
                     res.on('data', function (d) { data += d.toString('utf8'); });
                     res.on('end', function () {
                         expect(data.length).to.not.be(0);
@@ -606,14 +607,14 @@ describe('App API', function () {
         });
 
         it('logStream - stream logs', function (done) {
-            var options = {
+            const options = {
                 port: constants.PORT, host: 'localhost', path: '/api/v1/apps/' + APP_ID + '/logstream?access_token=' + token,
                 headers: { 'Accept': 'text/event-stream', 'Connection': 'keep-alive' }
             };
 
             // superagent doesn't work. maybe https://github.com/visionmedia/superagent/issues/420
-            var req = http.get(options, function (res) {
-                var data = '';
+            const req = http.get(options, function (res) {
+                const data = '';
                 res.on('data', function (d) { data += d.toString('utf8'); });
                 setTimeout(function checkData() {
                     expect(data.length).to.not.be(0);
@@ -1082,7 +1083,7 @@ describe('App API', function () {
 
         xit('port mapping works after reconfiguration', function (done) {
             setTimeout(function () {
-                var client = net.connect(7172);
+                const client = net.connect(7172);
                 client.on('data', function (data) {
                     expect(data.toString()).to.eql('ECHO_SERVER_PORT=7172');
                     done();
@@ -1499,8 +1500,8 @@ describe('App API', function () {
         });
 
         xit('can uninstall app', function (done) {
-            var fake1 = nock(settings.apiServerOrigin()).get(function (uri) { return uri.indexOf('/api/v1/cloudronapps/') >= 0; }).reply(200, { });
-            var fake2 = nock(settings.apiServerOrigin()).delete(function (uri) { return uri.indexOf('/api/v1/cloudronapps/') >= 0; }).reply(204, { });
+            const fake1 = nock(settings.apiServerOrigin()).get(function (uri) { return uri.indexOf('/api/v1/cloudronapps/') >= 0; }).reply(200, { });
+            const fake2 = nock(settings.apiServerOrigin()).delete(function (uri) { return uri.indexOf('/api/v1/cloudronapps/') >= 0; }).reply(204, { });
 
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/uninstall')
                 .query({ access_token: token })

src/routes/test/appstore-test.js

@@ -5,56 +5,43 @@
 
 'use strict';
 
-const common = require('./common.js'),
+const appstore = require('../../appstore.js'),
+    common = require('./common.js'),
     constants = require('../../constants.js'),
     expect = require('expect.js'),
     nock = require('nock'),
     settings = require('../../settings.js'),
     superagent = require('superagent');
 
-const { setup, cleanup, serverUrl, owner } = common;
+const { setup, cleanup, serverUrl, owner, appstoreToken } = common;
 
 describe('Appstore Apps API', function () {
-
     before(setup);
     after(cleanup);
 
-    it('cannot list apps without subscription', async function () {
+    it('cannot list apps when appstore is down', async function () {
         const response = await superagent.get(`${serverUrl}/api/v1/appstore/apps`)
             .query({ access_token: owner.token })
             .ok(() => true);
-        expect(response.statusCode).to.be(402);
+        expect(response.statusCode).to.be(424);
     });
 
-    it('cannot get app without subscription', async function () {
+    it('cannot get app with bad token', async function () {
+        const scope1 = nock(settings.apiServerOrigin())
+            .get(`/api/v1/apps/org.wordpress.cloudronapp?accessToken=${appstoreToken}`)
+            .reply(402, {});
+
         const response = await superagent.get(`${serverUrl}/api/v1/appstore/apps/org.wordpress.cloudronapp`)
             .query({ access_token: owner.token })
             .ok(() => true);
 
         expect(response.statusCode).to.be(402);
-    });
-
-    it('register cloudron', async function () {
-        const scope1 = nock(settings.apiServerOrigin())
-            .post('/api/v1/login', (body) => body.email && body.password)
-            .reply(200, { userId: 'userId', accessToken: 'SECRET_TOKEN' });
-
-        const scope2 = nock(settings.apiServerOrigin())
-            .post('/api/v1/register_cloudron', (body) => !!body.domain && body.accessToken === 'SECRET_TOKEN')
-            .reply(201, { cloudronId: 'cid', cloudronToken: 'CLOUDRON_TOKEN', licenseKey: 'lkey' });
-
-        const response = await superagent.post(`${serverUrl}/api/v1/appstore/register_cloudron`)
-            .send({ email: 'test@cloudron.io', password: 'secret', signup: false })
-            .query({ access_token: owner.token });
-
-        expect(response.statusCode).to.equal(201);
         expect(scope1.isDone()).to.be.ok();
-        expect(scope2.isDone()).to.be.ok();
     });
 
     it('can list apps', async function () {
         const scope1 = nock(settings.apiServerOrigin())
-            .get(`/api/v1/apps?accessToken=CLOUDRON_TOKEN&boxVersion=${constants.VERSION}&unstable=true`, () => true)
+            .get(`/api/v1/apps?accessToken=${appstoreToken}&boxVersion=${constants.VERSION}&unstable=true`, () => true)
             .reply(200, { apps: [] });
 
         const response = await superagent.get(`${serverUrl}/api/v1/appstore/apps`)
@@ -66,7 +53,7 @@ describe('Appstore Apps API', function () {
 
     it('can get app', async function () {
         const scope1 = nock(settings.apiServerOrigin())
-            .get('/api/v1/apps/org.wordpress.cloudronapp?accessToken=CLOUDRON_TOKEN', () => true)
+            .get(`/api/v1/apps/org.wordpress.cloudronapp?accessToken=${appstoreToken}`, () => true)
             .reply(200, { apps: [] });
 
         const response = await superagent.get(`${serverUrl}/api/v1/appstore/apps/org.wordpress.cloudronapp`)
@@ -77,8 +64,8 @@ describe('Appstore Apps API', function () {
     });
 
     it('can get app version', async function () {
-        var scope1 = nock(settings.apiServerOrigin())
-            .get('/api/v1/apps/org.wordpress.cloudronapp/versions/3.4.2?accessToken=CLOUDRON_TOKEN', () => true)
+        const scope1 = nock(settings.apiServerOrigin())
+            .get(`/api/v1/apps/org.wordpress.cloudronapp/versions/3.4.2?accessToken=${appstoreToken}`, () => true)
             .reply(200, { apps: [] });
 
         const response = await superagent.get(`${serverUrl}/api/v1/appstore/apps/org.wordpress.cloudronapp/versions/3.4.2`)
@@ -89,46 +76,17 @@ describe('Appstore Apps API', function () {
     });
 });
 
-describe('Subscription API - no signup', function () {
-    before(setup);
+describe('Appstore Cloudron Registration API - existing user', function () {
+    before(async function () {
+        await setup();
+        await appstore._unregister();
+    });
     after(cleanup);
 
     it('can setup subscription', async function () {
         const scope1 = nock(settings.apiServerOrigin())
-            .post('/api/v1/login', (body) => body.email && body.password)
-            .reply(200, { userId: 'userId', accessToken: 'SECRET_TOKEN' });
-
-        const scope2 = nock(settings.apiServerOrigin())
-            .post('/api/v1/register_cloudron', (body) => !!body.domain && body.accessToken === 'SECRET_TOKEN')
-            .reply(201, { cloudronId: 'cid', cloudronToken: 'CLOUDRON_TOKEN', licenseKey: 'lkey' });
-
-        const response = await superagent.post(`${serverUrl}/api/v1/appstore/register_cloudron`)
-            .send({ email: 'test@cloudron.io', password: 'secret', signup: false })
-            .query({ access_token: owner.token });
-
-        expect(response.statusCode).to.equal(201);
-        expect(scope1.isDone()).to.be.ok();
-        expect(scope2.isDone()).to.be.ok();
-    });
-
-    it('cannot re-setup subscription - already registered', async function () {
-        const response = await superagent.post(`${serverUrl}/api/v1/appstore/register_cloudron`)
-            .send({ email: 'test@cloudron.io', password: 'secret', signup: false })
-            .query({ access_token: owner.token })
-            .ok(() => true);
-
-        expect(response.statusCode).to.equal(409);
-    });
-});
-
-describe('Subscription API - signup', function () {
-    before(setup);
-    after(cleanup);
-
-    it('can setup subscription', async function () {
-        const scope1 = nock(settings.apiServerOrigin())
-            .post('/api/v1/register_user', (body) => body.email && body.password)
-            .reply(201, { });
+            .post('/api/v1/register_user', (body) => body.email && body.password && body.utmSource)
+            .reply(201, {});
 
         const scope2 = nock(settings.apiServerOrigin())
             .post('/api/v1/login', (body) => body.email && body.password)
@@ -136,16 +94,84 @@ describe('Subscription API - signup', function () {
 
         const scope3 = nock(settings.apiServerOrigin())
             .post('/api/v1/register_cloudron', (body) => !!body.domain && body.accessToken === 'SECRET_TOKEN')
-            .reply(201, { cloudronId: 'cid', cloudronToken: 'CLOUDRON_TOKEN', licenseKey: 'lkey' });
+            .reply(201, { cloudronId: 'cid', cloudronToken: 'CLOUDRON_TOKEN' });
 
         const response = await superagent.post(`${serverUrl}/api/v1/appstore/register_cloudron`)
-            .send({ email: 'test@cloudron.io', password: 'secret', signup: true })
+            .send({ email: 'test@cloudron.io', password: 'secret', signup: false })
             .query({ access_token: owner.token });
 
         expect(response.statusCode).to.equal(201);
-        expect(scope1.isDone()).to.be.ok();
+        expect(scope1.isDone()).to.not.be.ok(); // should not have called register_user since signup is false
         expect(scope2.isDone()).to.be.ok();
         expect(scope3.isDone()).to.be.ok();
+        expect(await settings.getAppstoreApiToken()).to.be('CLOUDRON_TOKEN');
+        expect(await settings.getAppstoreWebToken()).to.be('SECRET_TOKEN');
+        nock.cleanAll();
+    });
+
+    it('cannot re-register - already registered', async function () {
+        const response = await superagent.post(`${serverUrl}/api/v1/appstore/register_cloudron`)
+            .send({ email: 'test@cloudron.io', password: 'secret', signup: false })
+            .query({ access_token: owner.token })
+            .ok(() => true);
+
+        expect(response.statusCode).to.equal(409);
+    });
+
+    it('can get subscription', async function () {
+        const scope1 = nock(settings.apiServerOrigin())
+            .get('/api/v1/subscription?accessToken=CLOUDRON_TOKEN', () => true)
+            .reply(200, { subscription: { plan: { id: 'free' } }, email: 'test@cloudron.io' });
+
+        const response = await superagent.get(`${serverUrl}/api/v1/appstore/subscription`)
+            .query({ access_token: owner.token });
+
+        expect(response.statusCode).to.equal(200);
+        expect(response.body.email).to.be('test@cloudron.io');
+        expect(response.body.subscription).to.be.an('object');
+        expect(scope1.isDone()).to.be.ok();
+    });
+});
+
+describe('Appstore Cloudron Registration API - new user signup', function () {
+    before(async function () {
+        await setup();
+        await appstore._unregister();
+    });
+    after(cleanup);
+
+    it('can setup subscription', async function () {
+        const scope1 = nock(settings.apiServerOrigin())
+            .post('/api/v1/register_user', (body) => body.email && body.password && body.utmSource)
+            .reply(201, {});
+
+        const scope2 = nock(settings.apiServerOrigin())
+            .post('/api/v1/login', (body) => body.email && body.password)
+            .reply(200, { userId: 'userId', accessToken: 'SECRET_TOKEN' });
+
+        const scope3 = nock(settings.apiServerOrigin())
+            .post('/api/v1/register_cloudron', (body) => !!body.domain && body.accessToken === 'SECRET_TOKEN')
+            .reply(201, { cloudronId: 'cid', cloudronToken: 'CLOUDRON_TOKEN' });
+
+        const response = await superagent.post(`${serverUrl}/api/v1/appstore/register_cloudron`)
+            .send({ email: 'test@cloudron.io', password: 'secret', signup: true })
+            .query({ access_token: owner.token });
+
+        expect(response.statusCode).to.equal(201);
+        expect(scope1.isDone()).to.be.ok();
+        expect(scope2.isDone()).to.be.ok();
+        expect(scope3.isDone()).to.be.ok();
+        expect(await settings.getAppstoreApiToken()).to.be('CLOUDRON_TOKEN');
+        expect(await settings.getAppstoreWebToken()).to.be('SECRET_TOKEN');
+    });
+
+    it('cannot re-register - already registered', async function () {
+        const response = await superagent.post(`${serverUrl}/api/v1/appstore/register_cloudron`)
+            .send({ email: 'test@cloudron.io', password: 'secret', signup: false })
+            .query({ access_token: owner.token })
+            .ok(() => true);
+
+        expect(response.statusCode).to.equal(409);
     });
 
     it('can get subscription', async function () {

src/routes/test/backups-test.js

@@ -7,15 +7,27 @@
 
 const common = require('./common.js'),
     expect = require('expect.js'),
+    settings = require('../../settings.js'),
     superagent = require('superagent');
 
 describe('Backups API', function () {
-    const { setup, cleanup, serverUrl, owner } = common;
+    const { setup, cleanup, waitForTask, serverUrl, owner } = common;
 
     before(setup);
     after(cleanup);
 
     describe('create', function () {
+        before(async function () {
+            await settings.setBackupConfig({
+                provider: 'filesystem',
+                backupFolder: '/tmp/backups',
+                format: 'tgz',
+                encryption: null,
+                retentionPolicy: { keepWithinSecs: 2 * 24 * 60 * 60 }, // 2 days
+                schedulePattern: '00 00 23 * * *' // every day at 11pm
+            });
+        });
+
         it('fails due to mising token', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/backups/create`)
                 .ok(() => true);
@@ -34,6 +46,7 @@ describe('Backups API', function () {
                 .query({ access_token: owner.token });
             expect(response.statusCode).to.equal(202);
             expect(response.body.taskId).to.be.a('string');
+            await waitForTask(response.body.taskId);
         });
     });
 
@@ -42,7 +55,44 @@ describe('Backups API', function () {
             const response = await superagent.get(`${serverUrl}/api/v1/backups`)
                 .query({ access_token: owner.token });
             expect(response.statusCode).to.equal(200);
-            expect(response.body.backups).to.be.an('array');
+            expect(response.body.backups.length).to.be(1);
+        });
+    });
+
+    describe('update', function () {
+        let someBackup;
+
+        before(async function () {
+            const response = await superagent.get(`${serverUrl}/api/v1/backups`)
+                .query({ access_token: owner.token });
+            expect(response.statusCode).to.equal(200);
+            expect(response.body.backups.length).to.be(1);
+            someBackup = response.body.backups[0];
+            console.log(someBackup);
+        });
+
+        it('fails for bad param', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/backups/bad_id`)
+                .query({ access_token: owner.token })
+                .send({ preserveSecs: 'not-a-number', label: 'some string' })
+                .ok(() => true);
+            expect(response.statusCode).to.equal(400);
+        });
+
+        it('fails for unknown backup', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/backups/bad_id`)
+                .query({ access_token: owner.token })
+                .send({ preserveSecs: 30, label: 'NewOrleans' })
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(404);
+        });
+
+        it('succeeds', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/backups/${someBackup.id}`)
+                .query({ access_token: owner.token })
+                .send({ preserveSecs: 30, label: 'NewOrleans' });
+            expect(response.statusCode).to.equal(200);
         });
     });
 });

src/routes/test/cloudron-test.js

@@ -316,10 +316,10 @@ describe('Cloudron API', function () {
 
             // superagent doesn't work. maybe https://github.com/visionmedia/superagent/issues/420
             const req = http.get(options, function (res) {
-                var data = '';
+                let data = '';
                 res.on('data', function (d) { data += d.toString('utf8'); });
                 setTimeout(function checkData() {
-                    var dataMessageFound = false;
+                    let dataMessageFound = false;
 
                     expect(data.length).to.not.be(0);
                     data.split('\n').forEach(function (line) {

src/routes/test/common.js

@@ -2,7 +2,7 @@
 
 const constants = require('../../constants.js'),
     database = require('../../database.js'),
-    delay = require('delay'),
+    delay = require('../../delay.js'),
     expect = require('expect.js'),
     fs = require('fs'),
     mailer = require('../../mailer.js'),
@@ -11,6 +11,7 @@ const constants = require('../../constants.js'),
     settings = require('../../settings.js'),
     support = require('../../support.js'),
     superagent = require('superagent'),
+    tasks = require('../../tasks.js'),
     tokens = require('../../tokens.js');
 
 exports = module.exports = {
@@ -19,6 +20,7 @@ exports = module.exports = {
     cleanup,
     clearMailQueue,
     checkMails,
+    waitForTask,
 
     owner: {
         id: null,
@@ -82,7 +84,7 @@ async function setup() {
     const token = await tokens.add({ identifier: user.id, clientId: 'test-client-id', expires: Date.now() + (60 * 60 * 1000), name: 'fromtest' });
     user.token = token.accessToken;
 
-    await settings._set(settings.CLOUDRON_TOKEN_KEY, exports.appstoreToken); // appstore token
+    await settings._set(settings.APPSTORE_API_TOKEN_KEY, exports.appstoreToken); // appstore token
 }
 
 async function cleanup() {
@@ -99,3 +101,15 @@ async function checkMails(number) {
     expect(mailer._mailQueue.length).to.equal(number);
     clearMailQueue();
 }
+
+async function waitForTask(taskId) {
+    // eslint-disable-next-line no-constant-condition
+    for (let i = 0; i < 10; i++) {
+        const result = await tasks.get(taskId);
+        expect(result).to.not.be(null);
+        if (!result.active) return;
+        await delay(2000);
+        console.log(`Waiting for task to ${taskId} finish`);
+    }
+    throw new Error(`Task ${taskId} never finished`);
+}

src/routes/test/provision-test.js

@@ -7,7 +7,7 @@
 
 const common = require('./common.js');
 
-const delay = require('delay'),
+const delay = require('../../delay.js'),
     expect = require('expect.js'),
     superagent = require('superagent');
 

src/routes/test/users-test.js

@@ -211,6 +211,39 @@ describe('Users API', function () {
         });
     });
 
+    describe('make local', function () {
+        let userId;
+
+        before(async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/users`)
+                .query({ access_token: owner.token })
+                .send({ username: 'ldapuser', email: 'ldapuser@example.com' });
+
+            expect(response.statusCode).to.equal(201);
+
+            userId = response.body.id;
+        });
+
+        it('cannot make a local user local', async function () {
+            const response = await superagent.post(`${serverUrl}/api/v1/users/${userId}/make_local`)
+                .query({ access_token: owner.token })
+                .send({})
+                .ok(() => true);
+
+            expect(response.statusCode).to.equal(409);
+        });
+
+        it('succeeds', async function () {
+            await users.update({ id: userId }, { source: 'ldap' }, {});
+
+            const response = await superagent.post(`${serverUrl}/api/v1/users/${userId}/make_local`)
+                .query({ access_token: owner.token })
+                .send({});
+
+            expect(response.statusCode).to.equal(204);
+        });
+    });
+
     describe('admin status', function () {
         it('set second user as admin succeeds', async function () {
             const response = await superagent.post(`${serverUrl}/api/v1/users/${user.id}`)
@@ -607,15 +640,5 @@ describe('Users API', function () {
             expect(response.statusCode).to.equal(409);
         });
     });
-
-    describe('transfer ownership', function () {
-        it('succeeds', async function () {
-            const response = await superagent.post(`${serverUrl}/api/v1/users/${user.id}/make_owner`)
-                .query({ access_token: owner.token })
-                .send({});
-
-            expect(response.statusCode).to.equal(204);
-        });
-    });
 });
 

src/routes/users.js

@@ -10,7 +10,7 @@ exports = module.exports = {
     verifyPassword,
     setGroups,
     setGhost,
-    makeOwner,
+    makeLocal,
 
     getPasswordResetLink,
     sendPasswordResetEmail,
@@ -180,6 +180,7 @@ async function setGhost(req, res, next) {
 
     if (typeof req.body.password !== 'string' || !req.body.password) return next(new HttpError(400, 'password must be non-empty string'));
     if ('expiresAt' in req.body && typeof req.body.password !== 'number') return next(new HttpError(400, 'expiresAt must be a number'));
+    if (users.compareRoles(req.user.role, req.resource.role) < 0) return next(new HttpError(403, `role '${req.resource.role}' is required but user has only '${req.user.role}'`));
 
     const [error] = await safe(users.setGhost(req.resource, req.body.password, req.body.expiresAt || 0));
     if (error) return next(BoxError.toHttpError(error));
@@ -200,18 +201,20 @@ async function setPassword(req, res, next) {
     next(new HttpSuccess(204));
 }
 
-// This route transfers ownership from token user to user specified in path param
-async function makeOwner(req, res, next) {
+async function makeLocal(req, res, next) {
     assert.strictEqual(typeof req.resource, 'object');
 
-    // first make new one owner, then demote current one
-    let [error] = await safe(users.update(req.resource, { role: users.ROLE_OWNER }, AuditSource.fromRequest(req)));
+    if (users.compareRoles(req.user.role, req.resource.role) < 0) return next(new HttpError(403, `role '${req.resource.role}' is required but user has only '${req.user.role}'`));
+
+    if (req.resource.source === '') return next(new HttpError(409, 'user is already local'));
+
+    let [error] = await safe(users.update(req.resource, { source: '', inviteToken: '' }, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
-    [error] = await safe(users.update(req.user, { role: users.ROLE_USER }, AuditSource.fromRequest(req)));
-    if (error) return next(BoxError.toHttpError(error));
+    [error] = await safe(users.sendPasswordResetEmail(req.resource, req.resource.fallbackEmail || req.resource.email, AuditSource.fromRequest(req)));
+    if (error)  return next(BoxError.toHttpError(error));
 
-    next(new HttpSuccess(204));
+    next(new HttpSuccess(204, {}));
 }
 
 // This will always return a reset link, if none is set or expired a new one will be created