data dir: allow sameness of old and new dir

this makes it easy to migrate to a new volume setup

(cherry picked from commit a32166bc9d)

CHANGES

@@ -2470,4 +2470,35 @@
 * proxyAuth: set X-Remote-User (rfc3875)
 * GoDaddy: there is now a delete API
 * nginx: use ubuntu packages for ubuntu 20.04 and 22.04
+* Ubuntu 22.04 LTS support
+* Add Hetzner DNS
+* cron: add support for extensions (@reboot, @weekly etc)
+* Add profile backgroundImage api
+* exec: rework API to get exit code
+* Add update available filter
 
+[7.2.1]
+* Refactor backup code to use async/await
+* mongodb: fix bug where a small timeout prevented import of large backups
+* Add update available filter
+* exec: rework API to get exit code
+* Add profile backgroundImage api
+* cron: add support for extensions (@reboot, @weekly etc)
+
+[7.2.2]
+* Update cloudron-manifestformat for new scheduler patterns
+* collectd: FQDNLookup causes collectd install to fail
+
+[7.2.3]
+* appstore: allow re-registration on server side delete
+* transfer ownership route is not used anymore
+* graphite: fix issue where disk names with '.' do not render
+* dark mode fixes
+* sendmail: mail from display name
+* Use volumes for app data instead of raw path
+* initial xfs support
+
+[7.2.4]
+* volumes: Ensure long volume names do not overflow the table
+* Move all appstore filter to the left
+* app data: allow sameness of old and new dir

migrations/20220505162417-settings-add-console-origin.js

@@ -0,0 +1,20 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM settings WHERE name = ?', [ 'api_server_origin' ], function (error, result) {
+        if (error || result.length === 0) return callback(error);
+
+        let consoleOrigin;
+        switch (result[0].value) {
+        case 'https://api.dev.cloudron.io': consoleOrigin = 'https://console.dev.cloudron.io'; break;
+        case 'https://api.staging.cloudron.io': consoleOrigin = 'https://console.staging.cloudron.io'; break;
+        default: consoleOrigin = 'https://console.cloudron.io'; break;
+        }
+
+        db.runSql('REPLACE INTO settings (name, value) VALUES (?, ?)', [ 'console_server_origin', consoleOrigin ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20220514170234-users-add-backgroundImage.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN backgroundImage MEDIUMBLOB', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN backgroundImage', callback);
+};

migrations/20220601004757-apps-add-mailboxDisplayName.js

@@ -0,0 +1,12 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN mailboxDisplayName VARCHAR(128) DEFAULT "" NOT NULL', [], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN mailboxDisplayName', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20220602050517-apps-add-storageVolumeId.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const path = require('path'),
+    safe = require('safetydance'),
+    uuid = require('uuid');
+
+function getMountPoint(dataDir) {
+    const output = safe.child_process.execSync(`df --output=target "${dataDir}" | tail -1`, { encoding: 'utf8' });
+    if (!output) return dataDir;
+    const mountPoint = output.trim();
+    if (mountPoint === '/') return dataDir;
+    return mountPoint;
+}
+
+exports.up = async function(db) {
+    await db.runSql('ALTER TABLE apps ADD storageVolumeId VARCHAR(128), ADD FOREIGN KEY(storageVolumeId) REFERENCES volumes(id)');
+    await db.runSql('ALTER TABLE apps ADD storageVolumePrefix VARCHAR(128)');
+    await db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_storageVolume UNIQUE (storageVolumeId, storageVolumePrefix)');
+
+    const apps = await db.runSql('SELECT * FROM apps WHERE dataDir IS NOT NULL');
+    const allVolumes = await db.runSql('SELECT * FROM volumes');
+
+    for (const app of apps) {
+        console.log(`data-dir (${app.id}): migrating data dir ${app.dataDir}`);
+
+        const mountPoint = getMountPoint(app.dataDir);
+        const prefix = path.relative(mountPoint, app.dataDir);
+
+        console.log(`data-dir (${app.id}): migrating to mountpoint ${mountPoint} and prefix ${prefix}`);
+
+        const volume = allVolumes.find(v => v.hostPath === mountPoint);
+        if (volume) {
+            console.log(`data-dir (${app.id}): using existing volume ${volume.id}`);
+            await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ volume.id, prefix, app.id ]);
+            continue;
+        }
+
+        const id = uuid.v4().replace(/-/g, ''); // to make systemd mount file names more readable
+        const name = `app-${app.id}`;
+        const type = app.dataDir === mountPoint ? 'filesystem' : 'mountpoint';
+
+        console.log(`data-dir (${app.id}): creating new volume ${id}`);
+        await db.runSql('INSERT INTO volumes (id, name, hostPath, mountType, mountOptionsJson) VALUES (?, ?, ?, ?, ?)', [ id, name, mountPoint, type, JSON.stringify({}) ]);
+        await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ id, prefix, app.id ]);
+    }
+
+    await db.runSql('ALTER TABLE apps DROP COLUMN dataDir');
+};
+
+exports.down = async function(/*db*/) {
+};

migrations/schema.sql

@@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS users(
     resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     active BOOLEAN DEFAULT 1,
     avatar MEDIUMBLOB NOT NULL,
+    backgroundImage MEDIUMBLOB,
     loginLocationsJson MEDIUMTEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
 
     INDEX creationTime_index (creationTime),
@@ -85,13 +86,15 @@ CREATE TABLE IF NOT EXISTS apps(
     enableAutomaticUpdate BOOLEAN DEFAULT 1,
     enableMailbox BOOLEAN DEFAULT 1, // whether sendmail addon is enabled
     mailboxName VARCHAR(128), // mailbox of this app
-    mailboxDomain VARCHAR(128), // mailbox domain of this apps
+    mailboxDomain VARCHAR(128), // mailbox domain of this app
+    mailboxDisplayName VARCHAR(128), // mailbox display name
     enableInbox BOOLEAN DEFAULT 0, // whether recvmail addon is enabled
     inboxName VARCHAR(128), // mailbox of this app
-    inboxDomain VARCHAR(128), // mailbox domain of this apps
+    inboxDomain VARCHAR(128), // mailbox domain of this app
     label VARCHAR(128),     // display name
     tagsJson VARCHAR(2048), // array of tags
-    dataDir VARCHAR(256) UNIQUE,
+    storageVolumeId VARCHAR(128),
+    storageVolumePrefix VARCHAR(128),
     taskId INTEGER, // current task
     errorJson TEXT,
     servicesConfigJson TEXT, // app services configuration
@@ -102,6 +105,8 @@ CREATE TABLE IF NOT EXISTS apps(
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
+    FOREIGN KEY(storageVolumeId) REFERENCES volumes(id),
+    UNIQUE (storageVolumeId, storageVolumePrefix),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(

package-lock.json

@@ -15,7 +15,7 @@
         "aws-sdk": "^2.1115.0",
         "basic-auth": "^2.0.1",
         "body-parser": "^1.20.0",
-        "cloudron-manifestformat": "^5.15.2",
+        "cloudron-manifestformat": "^5.16.0",
         "connect": "^3.7.0",
         "connect-lastmile": "^2.1.1",
         "connect-timeout": "^1.9.0",
@@ -1413,9 +1413,9 @@
       }
     },
     "node_modules/cloudron-manifestformat": {
-      "version": "5.15.2",
-      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.15.2.tgz",
-      "integrity": "sha512-FD7s8IG0SztQjqMn0N0ko15Z7um+4+zGfok8xP21TCdOnuOZRdKvXsGHGU6VQNMCm8vLubhOSIRn2L2fRKzgQw==",
+      "version": "5.16.0",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.16.0.tgz",
+      "integrity": "sha512-YNlHrCMmz/L/wtkGOnB+lBa+aBGDDKZd9+pzxL5S4jZTYOcLgQwaD1+fX1Zqts0GnBk14hRsvJocO76qivYn/A==",
       "dependencies": {
         "cron": "^1.8.2",
         "java-packagename-regex": "^1.0.0",
@@ -9012,9 +9012,9 @@
       }
     },
     "cloudron-manifestformat": {
-      "version": "5.15.2",
-      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.15.2.tgz",
-      "integrity": "sha512-FD7s8IG0SztQjqMn0N0ko15Z7um+4+zGfok8xP21TCdOnuOZRdKvXsGHGU6VQNMCm8vLubhOSIRn2L2fRKzgQw==",
+      "version": "5.16.0",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.16.0.tgz",
+      "integrity": "sha512-YNlHrCMmz/L/wtkGOnB+lBa+aBGDDKZd9+pzxL5S4jZTYOcLgQwaD1+fX1Zqts0GnBk14hRsvJocO76qivYn/A==",
       "requires": {
         "cron": "^1.8.2",
         "java-packagename-regex": "^1.0.0",

package.json

@@ -18,7 +18,7 @@
     "aws-sdk": "^2.1115.0",
     "basic-auth": "^2.0.1",
     "body-parser": "^1.20.0",
-    "cloudron-manifestformat": "^5.15.2",
+    "cloudron-manifestformat": "^5.16.0",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.1.1",
     "connect-timeout": "^1.9.0",

scripts/cloudron-setup

@@ -26,8 +26,8 @@ readonly GREEN='\033[32m'
 readonly DONE='\033[m'
 
 # verify the system has minimum requirements met
-if [[ "${rootfs_type}" != "ext4" ]]; then
-    echo "Error: Cloudron requires '/' to be ext4" # see #364
+if [[ "${rootfs_type}" != "ext4" && "${rootfs_type}" != "xfs" ]]; then
+    echo "Error: Cloudron requires '/' to be ext4 or xfs" # see #364
     exit 1
 fi
 
@@ -62,6 +62,7 @@ requestedVersion=""
 installServerOrigin="https://api.cloudron.io"
 apiServerOrigin="https://api.cloudron.io"
 webServerOrigin="https://cloudron.io"
+consoleServerOrigin="https://console.cloudron.io"
 sourceTarballUrl=""
 rebootServer="true"
 setupToken="" # this is a OTP for securing an installation (https://forum.cloudron.io/topic/6389/add-password-for-initial-configuration)
@@ -80,10 +81,12 @@ while true; do
         if [[ "$2" == "dev" ]]; then
             apiServerOrigin="https://api.dev.cloudron.io"
             webServerOrigin="https://dev.cloudron.io"
+            consoleServerOrigin="https://console.dev.cloudron.io"
             installServerOrigin="https://api.dev.cloudron.io"
         elif [[ "$2" == "staging" ]]; then
             apiServerOrigin="https://api.staging.cloudron.io"
             webServerOrigin="https://staging.cloudron.io"
+            consoleServerOrigin="https://console.staging.cloudron.io"
             installServerOrigin="https://api.staging.cloudron.io"
         elif [[ "$2" == "unstable" ]]; then
             installServerOrigin="https://api.dev.cloudron.io"
@@ -209,9 +212,10 @@ fi
 
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('api_server_origin', '${apiServerOrigin}');" 2>/dev/null
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('web_server_origin', '${webServerOrigin}');" 2>/dev/null
+mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('console_server_origin', '${consoleServerOrigin}');" 2>/dev/null
 
 if [[ -n "${appstoreSetupToken}" ]]; then
-    if ! setupResponse=$(curl -X POST -H "Content-type: application/json" --data "{\"setupToken\": \"${appstoreSetupToken}\"}" "${apiServerOrigin}/api/v1/cloudron_setup_done"); then
+    if ! setupResponse=$(curl -sX POST -H "Content-type: application/json" --data "{\"setupToken\": \"${appstoreSetupToken}\"}" "${apiServerOrigin}/api/v1/cloudron_setup_done"); then
         echo "Could not complete setup. See ${LOG_FILE} for details"
         exit 1
     fi

scripts/cloudron-support

@@ -8,7 +8,7 @@ set -eu -o pipefail
 PASTEBIN="https://paste.cloudron.io"
 OUT="/tmp/cloudron-support.log"
 LINE="\n========================================================\n"
-CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQVilclYAIu+ioDp/sgzzFz6YU0hPcRYY7ze/LiF/lC7uQqK062O54BFXTvQ3ehtFZCx3bNckjlT2e6gB8Qq07OM66De4/S/g+HJW4TReY2ppSPMVNag0TNGxDzVH8pPHOysAm33LqT2b6L/wEXwC6zWFXhOhHjcMqXvi8Ejaj20H1HVVcf/j8qs5Thkp9nAaFTgQTPu8pgwD8wDeYX1hc9d0PYGesTADvo6HF4hLEoEnefLw7PaStEbzk2fD3j7/g5r5HcgQQXBe74xYZ/1gWOX2pFNuRYOBSEIrNfJEjFJsqk3NR1+ZoMGK7j+AZBR4k0xbrmncQLcQzl6MMDzkp support@cloudron.io"
+CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGWS+930b8QdzbchGljt3KSljH9wRhYvht8srrtQHdzg support@cloudron.io"
 HELP_MESSAGE="
 This script collects diagnostic information to help debug server related issues.
 
@@ -86,7 +86,6 @@ if [[ "${enableSSH}" == "true" ]]; then
     echo -e $LINE"SSH"$LINE >> $OUT
     echo "Username: ${ssh_user}" >> $OUT
     echo "Port:     ${ssh_port}" >> $OUT
-    echo "PermitRootLogin: ${permit_root_login}" >> $OUT
     echo "Key file: ${keys_file}" >> $OUT
 
     echo -n "Enabling ssh access for the Cloudron support team..."

scripts/init-ubuntu.sh

@@ -18,14 +18,6 @@ export DEBIAN_FRONTEND=noninteractive
 readonly ubuntu_codename=$(lsb_release -cs)
 readonly ubuntu_version=$(lsb_release -rs)
 
-# enable ubuntu proposed for collectd (https://launchpad.net/ubuntu/+source/collectd)
-if [[ "${ubuntu_version}" == "22.04" ]]; then
-    cat <<EOF >/etc/apt/sources.list.d/ubuntu-$(lsb_release -cs)-proposed.list
-# Enable Ubuntu proposed archive
-deb http://archive.ubuntu.com/ubuntu/ $(lsb_release -cs)-proposed restricted main multiverse universe
-EOF
-fi
-
 # hold grub since updating it breaks on some VPS providers. also, dist-upgrade will trigger it
 apt-mark hold grub* >/dev/null
 apt-get -o Dpkg::Options::="--force-confdef" update -y
@@ -117,17 +109,23 @@ update-grub
 
 echo "==> Install collectd"
 # without this, libnotify4 will install gnome-shell
-apt-get install -y libnotify4 --no-install-recommends
-if ! apt-get install -y --no-install-recommends libcurl3-gnutls collectd collectd-utils; then
-    # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
-    echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
-    sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
-fi
+apt-get install -y libnotify4 libcurl3-gnutls --no-install-recommends
 # https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
-if [[ "${ubuntu_version}" == "20.04" ]]; then
-    echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
-elif [[ "${ubuntu_version}" == "22.04" ]]; then
+if [[ "${ubuntu_version}" == "22.04" ]]; then
+    readonly launchpad="https://launchpad.net/ubuntu/+source/collectd/5.12.0-9/+build/23189375/+files"
+    cd /tmp && wget -q "${launchpad}/collectd_5.12.0-9_amd64.deb" "${launchpad}/collectd-utils_5.12.0-9_amd64.deb" "${launchpad}/collectd-core_5.12.0-9_amd64.deb" "${launchpad}/libcollectdclient1_5.12.0-9_amd64.deb"
+    cd /tmp && apt install -y --no-install-recommends ./libcollectdclient1_5.12.0-9_amd64.deb ./collectd-core_5.12.0-9_amd64.deb ./collectd_5.12.0-9_amd64.deb ./collectd-utils_5.12.0-9_amd64.deb && rm -f /tmp/collectd_*.deb
     echo -e "\nLD_PRELOAD=/usr/lib/python3.10/config-3.10-x86_64-linux-gnu/libpython3.10.so" >> /etc/default/collectd
+else
+    if ! apt-get install -y --no-install-recommends collectd collectd-utils; then
+        # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
+        echo "Failed to install collectd, continuing anyway. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
+        sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
+    fi
+
+    if [[ "${ubuntu_version}" == "20.04" ]]; then
+        echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
+    fi
 fi
 
 # some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
@@ -146,7 +144,7 @@ sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See h
 
 # https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
 echo "==> Disabling motd news"
-if [ -f "/etc/default/motd-news" ]; then
+if [[ -f "/etc/default/motd-news" ]]; then
     sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
 fi
 

setup/start/cloudron-firewall.sh

@@ -77,8 +77,6 @@ if [[ -f "${ldap_allowlist_json}" ]]; then
     done < "${ldap_allowlist_json}"
 
     # ldap server we expose 3004 and also redirect from standard ldaps port 636
-    $iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
-
     $iptables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
     $iptables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist src -p tcp --dport 3004 -j ACCEPT
 
@@ -149,6 +147,7 @@ for port in 3306 5432 6379 27017; do
     $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
+# Add the rate limit chain to input chain
 $iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
 $ip6tables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $ip6tables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
 

setup/start/collectd/collectd.conf

@@ -13,7 +13,7 @@
 ##############################################################################
 
 Hostname "localhost"
-#FQDNLookup true
+FQDNLookup false
 #BaseDir "/var/lib/collectd"
 #PluginDir "/usr/lib/collectd"
 #TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db"
@@ -232,7 +232,7 @@ LoadPlugin swap
 
 <Plugin write_graphite>
    <Node "graphing">
-       Host "localhost"
+       Host "127.0.0.1"
        Port "2003"
        Protocol "tcp"
        LogSendErrors true

setup/start/collectd/df.py

@@ -14,7 +14,7 @@ def read():
     for d in disks:
         device = d[0]
         if 'devicemapper' in d[1] or not device.startswith('/dev/'): continue
-        instance = device[len('/dev/'):].replace('/', '_') # see #348
+        instance = device[len('/dev/'):].replace('/', '_').replace('.', '_') # see #348
 
         try:
             st = os.statvfs(d[1]) # handle disk removal

setup/start/sudoers

@@ -1,6 +1,9 @@
 # sudo logging breaks journalctl output with very long urls (systemd bug)
 Defaults !syslog
 
+Defaults!/home/yellowtent/box/src/scripts/checkvolume.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/checkvolume.sh
+
 Defaults!/home/yellowtent/box/src/scripts/clearvolume.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/clearvolume.sh
 

src/apps.js

@@ -42,7 +42,7 @@ exports = module.exports = {
     setMailbox,
     setInbox,
     setLocation,
-    setDataDir,
+    setStorage,
     repair,
 
     restore,
@@ -66,7 +66,9 @@ exports = module.exports = {
     stop,
     restart,
 
-    exec,
+    createExec,
+    startExec,
+    getExec,
 
     checkManifestConstraints,
     downloadManifest,
@@ -79,7 +81,7 @@ exports = module.exports = {
     schedulePendingTasks,
     restartAppsUsingAddons,
 
-    getDataDir,
+    getStorageDir,
     getIcon,
     getMemoryLimit,
     getLimits,
@@ -166,6 +168,7 @@ const appstore = require('./appstore.js'),
     semver = require('semver'),
     services = require('./services.js'),
     settings = require('./settings.js'),
+    shell = require('./shell.js'),
     spawn = require('child_process').spawn,
     split = require('split'),
     superagent = require('superagent'),
@@ -176,6 +179,7 @@ const appstore = require('./appstore.js'),
     util = require('util'),
     uuid = require('uuid'),
     validator = require('validator'),
+    volumes = require('./volumes.js'),
     _ = require('underscore');
 
 const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
@@ -183,11 +187,13 @@ const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationS
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson', 'apps.operatorsJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', 'apps.crontab',
     'apps.creationTime', 'apps.updateTime', 'apps.enableAutomaticUpdate',
-    'apps.enableMailbox', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain',
-    'apps.dataDir', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
+    'apps.enableMailbox', 'apps.mailboxDisplayName', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain',
+    'apps.storageVolumeId', 'apps.storageVolumePrefix', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
 
 // const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
 
+const CHECKVOLUME_CMD = path.join(__dirname, 'scripts/checkvolume.sh');
+
 function validatePortBindings(portBindings, manifest) {
     assert.strictEqual(typeof portBindings, 'object');
     assert.strictEqual(typeof manifest, 'object');
@@ -300,6 +306,18 @@ function translateSecondaryDomains(secondaryDomains) {
 function parseCrontab(crontab) {
     assert(crontab === null || typeof crontab === 'string');
 
+    // https://www.man7.org/linux/man-pages/man5/crontab.5.html#EXTENSIONS
+    const KNOWN_EXTENSIONS = {
+        '@service'  : '@service', // runs once
+        '@reboot'   : '@service',
+        '@yearly'   : '0 0 1 1 *',
+        '@annually' : '0 0 1 1 *',
+        '@monthly'  : '0 0 1 * *',
+        '@weekly'   : '0 0 * * 0',
+        '@daily'    : '0 0 * * *',
+        '@hourly'   : '0 * * * *',
+    };
+
     const result = [];
     if (!crontab) return result;
 
@@ -307,20 +325,28 @@ function parseCrontab(crontab) {
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i].trim();
         if (!line || line.startsWith('#')) continue;
-        const parts = /^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/.exec(line);
-        if (!parts) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron configuration at line ${i+1}`);
-        const schedule = parts.slice(1, 6).join(' ');
-        const command = parts[6];
+        if (line.startsWith('@')) {
+            const parts = /^(@\S+)\s+(.+)$/.exec(line);
+            if (!parts) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron configuration at line ${i+1}`);
+            const [, extension, command] = parts;
+            if (!KNOWN_EXTENSIONS[extension]) throw new BoxError(BoxError.BAD_FIELD, `Unknown extension pattern at line ${i+1}`);
+            result.push({ schedule: KNOWN_EXTENSIONS[extension], command });
+        } else {
+            const parts = /^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/.exec(line);
+            if (!parts) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron configuration at line ${i+1}`);
+            const schedule = parts.slice(1, 6).join(' ');
+            const command = parts[6];
 
-        try {
-            new CronJob('00 ' + schedule, function() {}); // second is disallowed
-        } catch (ex) {
-            throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern at line ${i+1}`);
+            try {
+                new CronJob('00 ' + schedule, function() {}); // second is disallowed
+            } catch (ex) {
+                throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern at line ${i+1}`);
+            }
+
+            if (command.length === 0) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern. Command must not be empty at line ${i+1}`); // not possible with the regexp we have
+
+            result.push({ schedule, command });
         }
-
-        if (command.length === 0) throw new BoxError(BoxError.BAD_FIELD, `Invalid cron pattern. Command must not be empty at line ${i+1}`); // not possible with the regexp we have
-
-        result.push({ schedule, command });
     }
 
     return result;
@@ -455,28 +481,29 @@ function validateEnv(env) {
     return null;
 }
 
-function validateDataDir(dataDir) {
-    if (dataDir === null) return null;
+async function checkStorage(app, volumeId, prefix) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof volumeId, 'string');
+    assert.strictEqual(typeof prefix, 'string');
 
-    if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`);
-    if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`);
-    if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`);
+    const volume = await volumes.get(volumeId);
+    if (volume === null) throw new BoxError(BoxError.BAD_FIELD, 'Storage volume not found');
 
-    // nfs shares will have the directory mounted already
-    let stat = safe.fs.lstatSync(dataDir);
-    if (stat) {
-        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`);
-        let entries = safe.fs.readdirSync(dataDir);
-        if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`);
-        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`);
-    }
+    const status = await volumes.getStatus(volume);
+    if (status.state !== 'active') throw new BoxError(BoxError.BAD_FIELD, 'Volume is not active');
 
-    // backup logic relies on paths not overlapping (because it recurses)
-    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`);
+    if (path.isAbsolute(prefix)) throw new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" must be a relative path`);
+    if (prefix.endsWith('/')) throw new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" contains trailing slash`);
+    if (prefix !== '' && path.normalize(prefix) !== prefix) throw new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" is not a normalized path`);
 
-    // if we made it this far, it cannot start with any of these realistically
-    const fhs = [ '/bin', '/boot', '/etc', '/lib', '/lib32', '/lib64', '/proc', '/run', '/sbin', '/tmp', '/usr' ];
-    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`);
+    const sourceDir = await getStorageDir(app);
+    const targetDir = path.join(volume.hostPath, prefix);
+    const rel = path.relative(sourceDir, targetDir);
+    if (!rel.startsWith('../') && rel.split('/').length > 1) throw new BoxError(BoxError.BAD_FIELD, 'Only one level subdirectory moves are supported');
+
+    const [error] = await safe(shell.promises.sudo('checkStorage', [ CHECKVOLUME_CMD, targetDir, sourceDir ], {}));
+    if (error && error.code === 2) throw new BoxError(BoxError.BAD_FIELD, `Target directory ${targetDir} is not empty`);
+    if (error && error.code === 3) throw new BoxError(BoxError.BAD_FIELD, `Target directory ${targetDir} does not support chown`);
 
     return null;
 }
@@ -508,17 +535,20 @@ function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, port
         if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is in use`);
     }
 
-    if (match[2] === 'dataDir') {
-        return new BoxError(BoxError.BAD_FIELD, `Data directory ${match[1]} is in use`);
+    if (match[2] === 'apps_storageVolume') {
+        return new BoxError(BoxError.BAD_FIELD, `Storage directory ${match[1]} is in use`);
     }
 
     return new BoxError(BoxError.ALREADY_EXISTS, `${match[2]} '${match[1]}' is in use`);
 }
 
-function getDataDir(app, dataDir) {
-    assert(dataDir === null || typeof dataDir === 'string');
+async function getStorageDir(app) {
+    assert.strictEqual(typeof app, 'object');
 
-    return dataDir || path.join(paths.APPS_DATA_DIR, app.id, 'data');
+    if (!app.storageVolumeId) return path.join(paths.APPS_DATA_DIR, app.id, 'data');
+    const volume = await volumes.get(app.storageVolumeId);
+    if (!volume) throw new BoxError(BoxError.NOT_FOUND, 'Volume not found'); // not possible
+    return path.join(volume.hostPath, app.storageVolumePrefix);
 }
 
 function removeInternalFields(app) {
@@ -527,8 +557,8 @@ function removeInternalFields(app) {
         'subdomain', 'domain', 'fqdn', 'crontab',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'operators',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts',
-        'enableMailbox', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
+        'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'storageVolumeId', 'storageVolumePrefix', 'mounts',
+        'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
 }
 
 // non-admins can only see these
@@ -757,6 +787,7 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
         tagsJson = data.tags ? JSON.stringify(data.tags) : null,
         mailboxName = data.mailboxName || null,
         mailboxDomain = data.mailboxDomain || null,
+        mailboxDisplayName = data.mailboxDisplayName || '',
         reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null,
         servicesConfigJson = data.servicesConfig ? JSON.stringify(data.servicesConfig) : null,
         enableMailbox = data.enableMailbox || false,
@@ -767,10 +798,11 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
     queries.push({
         query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
             + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, '
-            + 'enableMailbox) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+            + 'enableMailbox, mailboxDisplayName) '
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
         args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
-            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox ]
+            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon,
+            enableMailbox, mailboxDisplayName ]
     });
 
     queries.push({
@@ -1571,6 +1603,7 @@ async function setMailbox(app, data, auditSource) {
     const optional = 'optional' in app.manifest.addons.sendmail ? app.manifest.addons.sendmail.optional : false;
     if (!optional && !enableMailbox) throw new BoxError(BoxError.BAD_FIELD, 'App requires sendmail to be enabled');
 
+    const mailboxDisplayName = data.mailboxDisplayName || '';
     let mailboxName = data.mailboxName || null;
     const mailboxDomain = data.mailboxDomain || null;
 
@@ -1583,15 +1616,20 @@ async function setMailbox(app, data, auditSource) {
         } else {
             mailboxName = mailboxNameForSubdomain(app.subdomain, app.domain, app.manifest);
         }
+
+        if (mailboxDisplayName) {
+            error = mail.validateDisplayName(mailboxDisplayName);
+            if (error) throw new BoxError(BoxError.BAD_FIELD, error.message);
+        }
     }
 
     const task = {
         args: {},
-        values: { enableMailbox, mailboxName, mailboxDomain }
+        values: { enableMailbox, mailboxName, mailboxDomain, mailboxDisplayName }
     };
     const taskId = await addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task, auditSource);
 
-    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mailboxName, mailboxDomain, taskId });
+    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mailboxName, mailboxDomain, mailboxDisplayName, taskId });
 
     return { taskId };
 }
@@ -1764,25 +1802,29 @@ async function setLocation(app, data, auditSource) {
     return { taskId };
 }
 
-async function setDataDir(app, dataDir, auditSource) {
+async function setStorage(app, volumeId, volumePrefix, auditSource) {
     assert.strictEqual(typeof app, 'object');
-    assert(dataDir === null || typeof dataDir === 'string');
+    assert(volumeId === null || typeof volumeId === 'string');
+    assert(volumePrefix === null || typeof volumePrefix === 'string');
     assert.strictEqual(typeof auditSource, 'object');
 
     const appId = app.id;
     let error = checkAppState(app, exports.ISTATE_PENDING_DATA_DIR_MIGRATION);
     if (error) throw error;
 
-    error = validateDataDir(dataDir);
-    if (error) throw error;
+    if (volumeId) {
+        await checkStorage(app, volumeId, volumePrefix);
+    } else {
+        volumeId = volumePrefix = null;
+    }
 
     const task = {
-        args: { newDataDir: dataDir },
+        args: { newStorageVolumeId: volumeId, newStorageVolumePrefix: volumePrefix },
         values: {}
     };
     const taskId = await addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, auditSource);
 
-    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, dataDir, taskId });
+    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, volumeId, volumePrefix, taskId });
 
     return { taskId };
 }
@@ -2213,7 +2255,8 @@ async function clone(app, data, user, auditSource) {
         tags: app.tags,
         enableAutomaticUpdate: app.enableAutomaticUpdate,
         icon: icons.icon,
-        enableMailbox: app.enableMailbox
+        enableMailbox: app.enableMailbox,
+        mailboxDisplayName: app.mailboxDisplayName
     };
 
     const [addError] = await safe(add(newAppId, appStoreId, manifest, subdomain, domain, translatePortBindings(portBindings, manifest), obj));
@@ -2335,7 +2378,7 @@ function checkManifestConstraints(manifest) {
     return null;
 }
 
-async function exec(app, options) {
+async function createExec(app, options) {
     assert.strictEqual(typeof app, 'object');
     assert(options && typeof options === 'object');
 
@@ -2346,7 +2389,7 @@ async function exec(app, options) {
         throw new BoxError(BoxError.BAD_STATE, 'App not installed or running');
     }
 
-    const execOptions = {
+    const createOptions = {
         AttachStdin: true,
         AttachStdout: true,
         AttachStderr: true,
@@ -2358,6 +2401,18 @@ async function exec(app, options) {
         Cmd: cmd
     };
 
+    return await docker.createExec(app.containerId, createOptions);
+}
+
+async function startExec(app, execId, options) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof execId, 'string');
+    assert(options && typeof options === 'object');
+
+    if (app.installationState !== exports.ISTATE_INSTALLED || app.runState !== exports.RSTATE_RUNNING) {
+        throw new BoxError(BoxError.BAD_STATE, 'App not installed or running');
+    }
+
     const startOptions = {
         Detach: false,
         Tty: options.tty,
@@ -2373,10 +2428,26 @@ async function exec(app, options) {
         stderr: true
     };
 
-    const stream = await docker.execContainer(app.containerId, { execOptions, startOptions, rows: options.rows, columns: options.columns });
+    const stream = await docker.startExec(execId, startOptions);
+
+    if (options.rows && options.columns) {
+        // there is a race where resizing too early results in a 404 "no such exec"
+        // https://git.cloudron.io/cloudron/box/issues/549
+        setTimeout(async function () {
+            await safe(docker.resizeExec(execId, { h: options.rows, w: options.columns }, { debug }));
+        }, 2000);
+    }
+
     return stream;
 }
 
+async function getExec(app, execId) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof execId, 'string');
+
+    return await docker.getExec(execId);
+}
+
 function canAutoupdateApp(app, updateInfo) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof updateInfo, 'object');
@@ -2601,7 +2672,8 @@ async function downloadFile(app, filePath) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof filePath, 'string');
 
-    const statStream = await exec(app, { cmd: [ 'stat', '--printf=%F-%s', filePath ], tty: true });
+    const statExecId = await createExec(app, { cmd: [ 'stat', '--printf=%F-%s', filePath ], tty: true });
+    const statStream = await startExec(app, statExecId, { tty: true });
     const data = await drainStream(statStream);
 
     const parts = data.split('-');
@@ -2622,7 +2694,8 @@ async function downloadFile(app, filePath) {
         throw new BoxError(BoxError.NOT_FOUND, 'only files or dirs can be downloaded');
     }
 
-    const inputStream = await exec(app, { cmd, tty: false });
+    const execId = await createExec(app, { cmd, tty: false });
+    const inputStream = await startExec(app, execId, { tty: false });
 
     // transforms the docker stream into a normal stream
     const stdoutStream = new TransformStream({
@@ -2663,7 +2736,8 @@ async function uploadFile(app, sourceFilePath, destFilePath) {
     const escapedDestFilePath = safe.child_process.execSync(`printf %q '${destFilePath.replace(/'/g, '\'\\\'\'')}'`, { shell: '/bin/bash', encoding: 'utf8' });
     debug(`uploadFile: ${sourceFilePath} -> ${escapedDestFilePath}`);
 
-    const destStream = await exec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false });
+    const execId = await createExec(app, { cmd: [ 'bash', '-c', `cat - > ${escapedDestFilePath}` ], tty: false });
+    const destStream = await startExec(app, execId, { tty: false });
 
     return new Promise((resolve, reject) => {
         const done = once(error => reject(new BoxError(BoxError.FS_ERROR, error.message)));

src/appstore.js

@@ -89,8 +89,8 @@ async function login(email, password, totpToken) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${response.status}`);
-    if (!response.body.accessToken) throw new BoxError(BoxError.EXTERNAL_ERROR, `login invalid response: ${response.text}`);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. status code: ${response.status}`);
+    if (!response.body.accessToken) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. invalid response: ${response.text}`);
 
     return response.body; // { userId, accessToken }
 }
@@ -100,13 +100,13 @@ async function registerUser(email, password) {
     assert.strictEqual(typeof password, 'string');
 
     const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/register_user`)
-        .send({ email, password })
+        .send({ email, password, utmSource: 'cloudron-dashboard' })
         .timeout(30 * 1000)
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, 'account already exists');
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${response.status}`);
+    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, 'Registration error: account already exists');
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Registration error. invalid response: ${response.status}`);
 }
 
 async function getWebToken() {
@@ -129,7 +129,6 @@ async function getSubscription() {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR);
     if (response.status === 502) throw new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`);
 
@@ -185,8 +184,7 @@ async function unpurchaseApp(appId, data) {
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 404) return;   // was never purchased
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed to get app. status:${response.status}`);
 
     [error, response] = await safe(superagent.del(url)
         .send(data)
@@ -196,7 +194,7 @@ async function unpurchaseApp(appId, data) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
+    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed. status:${response.status}`);
 }
 
 async function getBoxUpdate(options) {
@@ -218,7 +216,6 @@ async function getBoxUpdate(options) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status === 204) return; // no update
     if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
@@ -261,7 +258,6 @@ async function getAppUpdate(app, options) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status === 204) return; // no update
     if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
@@ -285,10 +281,10 @@ async function getAppUpdate(app, options) {
 async function registerCloudron(data) {
     assert.strictEqual(typeof data, 'object');
 
-    const { domain, accessToken, version } = data;
+    const { domain, accessToken, version, existingApps } = data;
 
     const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/register_cloudron`)
-        .send({ domain, accessToken, version })
+        .send({ domain, accessToken, version, existingApps })
         .timeout(30 * 1000)
         .ok(() => true));
 
@@ -326,7 +322,6 @@ async function updateCloudron(data) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
     debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
@@ -335,13 +330,14 @@ async function updateCloudron(data) {
 async function registerWithLoginCredentials(options) {
     assert.strictEqual(typeof options, 'object');
 
-    const token = await settings.getAppstoreApiToken();
-    if (token) throw new BoxError(BoxError.CONFLICT, 'Cloudron is already registered');
-
     if (options.signup) await registerUser(options.email, options.password);
 
     const result = await login(options.email, options.password, options.totpToken || '');
     await registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION });
+
+    for (const app of await apps.list()) {
+        await purchaseApp({ appId: app.id, appstoreId: app.appStoreId, manifestId: app.manifest.id || 'customapp' });
+    }
 }
 
 async function unregister() {
@@ -391,7 +387,6 @@ async function createTicket(info, auditSource) {
     const [error, response] = await safe(request);
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
     await eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
@@ -412,7 +407,6 @@ async function getApps() {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 403 || response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', response.status, response.body));
     if (!response.body.apps) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
@@ -443,7 +437,6 @@ async function getAppVersion(appId, version) {
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 403 || response.statusCode === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
     if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', response.status, response.body));
 
     return response.body;

src/apptask.js

@@ -160,7 +160,8 @@ async function deleteAppDir(app, options) {
 async function addCollectdProfile(app) {
     assert.strictEqual(typeof app, 'object');
 
-    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir: apps.getDataDir(app, app.dataDir) });
+    const appDataDir = await apps.getStorageDir(app);
+    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir });
     await collectd.addProfile(app.id, collectdConf);
 }
 
@@ -268,19 +269,22 @@ async function waitForDnsPropagation(app) {
     }
 }
 
-async function moveDataDir(app, targetDir) {
+async function moveDataDir(app, targetVolumeId, targetVolumePrefix) {
     assert.strictEqual(typeof app, 'object');
-    assert(targetDir === null || typeof targetDir === 'string');
+    assert(targetVolumeId === null || typeof targetVolumeId === 'string');
+    assert(targetVolumePrefix === null || typeof targetVolumePrefix === 'string');
 
-    const resolvedSourceDir = apps.getDataDir(app, app.dataDir);
-    const resolvedTargetDir = apps.getDataDir(app, targetDir);
+    const resolvedSourceDir = await apps.getStorageDir(app);
+    const resolvedTargetDir = await apps.getStorageDir(_.extend({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix }));
 
     debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
 
-    if (resolvedSourceDir === resolvedTargetDir) return;
+    if (resolvedSourceDir !== resolvedTargetDir) {
+        const [error] = await safe(shell.promises.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
+        if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`);
+    }
 
-    const [error] = await safe(shell.promises.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
-    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`);
+    await updateApp(app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix });
 }
 
 async function downloadImage(manifest) {
@@ -520,8 +524,9 @@ async function migrateDataDir(app, args, progressCallback) {
     assert.strictEqual(typeof args, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    const newDataDir = args.newDataDir;
-    assert(newDataDir === null || typeof newDataDir === 'string');
+    const { newStorageVolumeId, newStorageVolumePrefix } = args;
+    assert(newStorageVolumeId === null || typeof newStorageVolumeId === 'string');
+    assert(newStorageVolumePrefix === null || typeof newStorageVolumePrefix === 'string');
 
     await progressCallback({ percent: 10, message: 'Cleaning up old install' });
     await deleteContainers(app, { managedOnly: true });
@@ -529,12 +534,12 @@ async function migrateDataDir(app, args, progressCallback) {
     await progressCallback({ percent: 45, message: 'Ensuring app data directory' });
     await createAppDir(app);
 
-    // re-setup addons since this creates the localStorage volume
+    // re-setup addons since this creates the localStorage destination
     await progressCallback({ percent: 50, message: 'Setting up addons' });
-    await services.setupAddons(_.extend({}, app, { dataDir: newDataDir }), app.manifest.addons);
+    await services.setupAddons(_.extend({}, app, { storageVolumeId: newStorageVolumeId, storageVolumePrefix: newStorageVolumePrefix }), app.manifest.addons);
 
     await progressCallback({ percent: 60, message: 'Moving data dir' });
-    await moveDataDir(app, newDataDir);
+    await moveDataDir(app, newStorageVolumeId, newStorageVolumePrefix);
 
     await progressCallback({ percent: 90, message: 'Creating container' });
     await createContainer(app);
@@ -542,7 +547,7 @@ async function migrateDataDir(app, args, progressCallback) {
     await startApp(app);
 
     await progressCallback({ percent: 100, message: 'Done' });
-    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir });
+    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
 }
 
 // configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local"

src/backupcleaner.js

@@ -10,6 +10,7 @@ exports = module.exports = {
 
 const apps = require('./apps.js'),
     assert = require('assert'),
+    backupFormat = require('./backupformat.js'),
     backups = require('./backups.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:backupcleaner'),
@@ -85,7 +86,7 @@ async function removeBackup(backupConfig, backup, progressCallback) {
     assert.strictEqual(typeof backup, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    const backupFilePath = storage.getBackupFilePath(backupConfig, backup.remotePath, backup.format);
+    const backupFilePath = backupFormat.api(backup.format).getBackupFilePath(backupConfig, backup.remotePath);
 
     let removeError;
     if (backup.format ==='tgz') {
@@ -212,7 +213,7 @@ async function cleanupMissingBackups(backupConfig, progressCallback) {
         result = await backups.list(page, perPage);
 
         for (const backup of result) {
-            let backupFilePath = storage.getBackupFilePath(backupConfig, backup.remotePath, backup.format);
+            let backupFilePath = backupFormat.api(backup.format).getBackupFilePath(backupConfig, backup.remotePath);
             if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
 
             const [existsError, exists] = await safe(storage.api(backupConfig.provider).exists(backupConfig, backupFilePath));
@@ -251,9 +252,9 @@ async function cleanupSnapshots(backupConfig) {
         if (app) continue; // app is still installed
 
         if (info[appId].format ==='tgz') {
-            await safe(storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format)), { debug });
+            await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`)), { debug });
         } else {
-            await safe(storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), progressCallback), { debug });
+            await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`), progressCallback), { debug });
         }
 
         safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));

src/backupformat.js

@@ -0,0 +1,12 @@
+'use strict';
+
+exports = module.exports = {
+    api
+};
+
+function api(format) {
+    switch (format) {
+    case 'tgz': return require('./backupformat/tgz.js');
+    case 'rsync': return require('./backupformat/rsync.js');
+    }
+}

src/backupformat/rsync.js

@@ -0,0 +1,245 @@
+'use strict';
+
+exports = module.exports = {
+    getBackupFilePath,
+    download,
+    upload,
+
+    _saveFsMetadata: saveFsMetadata,
+    _restoreFsMetadata: restoreFsMetadata
+};
+
+const assert = require('assert'),
+    async = require('async'),
+    BoxError = require('../boxerror.js'),
+    DataLayout = require('../datalayout.js'),
+    debug = require('debug')('box:backupformat/rsync'),
+    fs = require('fs'),
+    hush = require('../hush.js'),
+    once = require('../once.js'),
+    path = require('path'),
+    safe = require('safetydance'),
+    storage = require('../storage.js'),
+    syncer = require('../syncer.js'),
+    util = require('util');
+
+function getBackupFilePath(backupConfig, remotePath) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+
+    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
+    return path.join(rootPath, remotePath);
+}
+
+function sync(backupConfig, remotePath, dataLayout, progressCallback, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB
+    const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10);
+    const removeDir = util.callbackify(storage.api(backupConfig.provider).removeDir);
+    const remove = util.callbackify(storage.api(backupConfig.provider).remove);
+
+    syncer.sync(dataLayout, function processTask(task, iteratorCallback) {
+        debug('sync: processing task: %j', task);
+        // the empty task.path is special to signify the directory
+        const destPath = task.path && backupConfig.encryption ? hush.encryptFilePath(task.path, backupConfig.encryption) : task.path;
+        const backupFilePath = path.join(getBackupFilePath(backupConfig, remotePath), destPath);
+
+        if (task.operation === 'removedir') {
+            debug(`Removing directory ${backupFilePath}`);
+            return removeDir(backupConfig, backupFilePath, progressCallback, iteratorCallback);
+        } else if (task.operation === 'remove') {
+            debug(`Removing ${backupFilePath}`);
+            return remove(backupConfig, backupFilePath, iteratorCallback);
+        }
+
+        let retryCount = 0;
+        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+            retryCallback = once(retryCallback); // protect again upload() erroring much later after read stream error
+
+            ++retryCount;
+            if (task.operation === 'add') {
+                progressCallback({ message: `Adding ${task.path}` + (retryCount > 1 ?  ` (Try ${retryCount})` : '') });
+                debug(`Adding ${task.path} position ${task.position} try ${retryCount}`);
+                const stream = hush.createReadStream(dataLayout.toLocalPath('./' + task.path), backupConfig.encryption);
+                stream.on('error', (error) => retryCallback(error.message.includes('ENOENT') ? null : error)); // ignore error if file disappears
+                stream.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0MBps looks wrong
+                    progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}MBps` }); // 0M@0MBps looks wrong
+                });
+                // only create the destination path when we have confirmation that the source is available. otherwise, we end up with
+                // files owned as 'root' and the cp later will fail
+                stream.on('open', function () {
+                    storage.api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, function (error) {
+                        debug(error ? `Error uploading ${task.path} try ${retryCount}: ${error.message}` : `Uploaded ${task.path}`);
+                        retryCallback(error);
+                    });
+                });
+            }
+        }, iteratorCallback);
+    }, concurrency, function (error) {
+        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+
+        callback();
+    });
+}
+
+// this is not part of 'snapshotting' because we need root access to traverse
+async function saveFsMetadata(dataLayout, metadataFile) {
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof metadataFile, 'string');
+
+    // contains paths prefixed with './'
+    const metadata = {
+        emptyDirs: [],
+        execFiles: [],
+        symlinks: []
+    };
+
+    // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
+    for (let lp of dataLayout.localPaths()) {
+        const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (emptyDirs === null) throw new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`);
+        if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
+
+        const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (execFiles === null) throw new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`);
+        if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
+
+        const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (symlinks === null) throw new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`);
+        if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => {
+            const target = safe.fs.readlinkSync(sl);
+            return { path: dataLayout.toRemotePath(sl), target };
+        }));
+    }
+
+    if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) throw new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`);
+}
+
+async function restoreFsMetadata(dataLayout, metadataFile) {
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof metadataFile, 'string');
+
+    debug(`Recreating empty directories in ${dataLayout.toString()}`);
+
+    const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
+    if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
+    const metadata = safe.JSON.parse(metadataJson);
+    if (metadata === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message);
+
+    for (const emptyDir of metadata.emptyDirs) {
+        const [mkdirError] = await safe(fs.promises.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }));
+        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to create path: ${mkdirError.message}`);
+    }
+
+    for (const execFile of metadata.execFiles) {
+        const [chmodError] = await safe(fs.promises.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8)));
+        if (chmodError) throw new BoxError(BoxError.FS_ERROR, `unable to chmod: ${chmodError.message}`);
+    }
+
+    for (const symlink of (metadata.symlinks || [])) {
+        if (!symlink.target) continue;
+        // the path may not exist if we had a directory full of symlinks
+        const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }));
+        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink (mkdir): ${mkdirError.message}`);
+        const [symlinkError] = await safe(fs.promises.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file'));
+        if (symlinkError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink: ${symlinkError.message}`);
+    }
+}
+
+function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
+
+    function downloadFile(entry, done) {
+        let relativePath = path.relative(backupFilePath, entry.fullPath);
+        if (backupConfig.encryption) {
+            const { error, result } = hush.decryptFilePath(relativePath, backupConfig.encryption);
+            if (error) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file'));
+            relativePath = result;
+        }
+        const destFilePath = dataLayout.toLocalPath('./' + relativePath);
+
+        fs.mkdir(path.dirname(destFilePath), { recursive: true }, function (error) {
+            if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));
+
+            async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+                storage.api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) {
+                    if (error) {
+                        progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                        return retryCallback(error);
+                    }
+
+                    let destStream = hush.createWriteStream(destFilePath, backupConfig.encryption);
+
+                    // protect against multiple errors. must destroy the write stream so that a previous retry does not write
+                    let closeAndRetry = once((error) => {
+                        if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                        else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` });
+                        sourceStream.destroy();
+                        destStream.destroy();
+                        retryCallback(error);
+                    });
+
+                    destStream.on('progress', function (progress) {
+                        const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                        if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0MBps looks wrong
+                        progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}MBps` });
+                    });
+                    destStream.on('error', closeAndRetry);
+
+                    sourceStream.on('error', closeAndRetry);
+
+                    progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
+
+                    sourceStream.pipe(destStream, { end: true }).on('done', closeAndRetry);
+                });
+            }, done);
+        });
+    }
+
+    storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
+        // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
+        const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
+
+        async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
+    }, callback);
+}
+
+async function download(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
+
+    const backupFilePath = getBackupFilePath(backupConfig, remotePath);
+    const downloadDirAsync = util.promisify(downloadDir);
+
+    await downloadDirAsync(backupConfig, backupFilePath, dataLayout, progressCallback);
+    await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
+}
+
+async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof dataLayout, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const syncAsync = util.promisify(sync);
+
+    await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
+    await syncAsync(backupConfig, remotePath, dataLayout, progressCallback);
+}

src/backupformat/tgz.js

@@ -0,0 +1,195 @@
+'use strict';
+
+exports = module.exports = {
+    getBackupFilePath,
+    download,
+    upload
+};
+
+const assert = require('assert'),
+    async = require('async'),
+    BoxError = require('../boxerror.js'),
+    DataLayout = require('../datalayout.js'),
+    debug = require('debug')('box:backupformat/tgz'),
+    { DecryptStream, EncryptStream } = require('../hush.js'),
+    once = require('../once.js'),
+    path = require('path'),
+    progressStream = require('progress-stream'),
+    storage = require('../storage.js'),
+    tar = require('tar-fs'),
+    zlib = require('zlib');
+
+function getBackupFilePath(backupConfig, remotePath) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+
+    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
+
+    const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz';
+    return path.join(rootPath, remotePath + fileType);
+}
+
+function tarPack(dataLayout, encryption) {
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const pack = tar.pack('/', {
+        dereference: false, // pack the symlink and not what it points to
+        entries: dataLayout.localPaths(),
+        ignoreStatError: (path, err) => {
+            debug(`tarPack: error stat'ing ${path} - ${err.code}`);
+            return err.code === 'ENOENT'; // ignore if file or dir got removed (probably some temporary file)
+        },
+        map: function(header) {
+            header.name = dataLayout.toRemotePath(header.name);
+            // the tar pax format allows us to encode filenames > 100 and size > 8GB (see #640)
+            // https://www.systutorials.com/docs/linux/man/5-star/
+            if (header.size > 8589934590 || header.name > 99) header.pax = { size: header.size };
+            return header;
+        },
+        strict: false // do not error for unknown types (skip fifo, char/block devices)
+    });
+
+    const gzip = zlib.createGzip({});
+    const ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds
+
+    pack.on('error', function (error) {
+        debug('tarPack: tar stream error.', error);
+        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    gzip.on('error', function (error) {
+        debug('tarPack: gzip stream error.', error);
+        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    if (encryption) {
+        const encryptStream = new EncryptStream(encryption);
+        encryptStream.on('error', function (error) {
+            debug('tarPack: encrypt stream error.', error);
+            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+        });
+
+        pack.pipe(gzip).pipe(encryptStream).pipe(ps);
+    } else {
+        pack.pipe(gzip).pipe(ps);
+    }
+
+    return ps;
+}
+
+function tarExtract(inStream, dataLayout, encryption) {
+    assert.strictEqual(typeof inStream, 'object');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const gunzip = zlib.createGunzip({});
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+    const extract = tar.extract('/', {
+        map: function (header) {
+            header.name = dataLayout.toLocalPath(header.name);
+            return header;
+        },
+        dmode: 500 // ensure directory is writable
+    });
+
+    const emitError = once((error) => {
+        inStream.destroy();
+        ps.emit('error', error);
+    });
+
+    inStream.on('error', function (error) {
+        debug('tarExtract: input stream error.', error);
+        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    gunzip.on('error', function (error) {
+        debug('tarExtract: gunzip stream error.', error);
+        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    extract.on('error', function (error) {
+        debug('tarExtract: extract stream error.', error);
+        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    extract.on('finish', function () {
+        debug('tarExtract: done.');
+        // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not extract
+        ps.emit('done');
+    });
+
+    if (encryption) {
+        const decrypt = new DecryptStream(encryption);
+        decrypt.on('error', function (error) {
+            debug('tarExtract: decrypt stream error.', error);
+            emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`));
+        });
+        inStream.pipe(ps).pipe(decrypt).pipe(gunzip).pipe(extract);
+    } else {
+        inStream.pipe(ps).pipe(gunzip).pipe(extract);
+    }
+
+    return ps;
+}
+
+async function download(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
+
+    const backupFilePath = getBackupFilePath(backupConfig, remotePath);
+
+    return new Promise((resolve, reject) => {
+        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+            progressCallback({ message: `Downloading backup ${remotePath}` });
+
+            storage.api(backupConfig.provider).download(backupConfig, backupFilePath, function (error, sourceStream) {
+                if (error) return retryCallback(error);
+
+                const ps = tarExtract(sourceStream, dataLayout, backupConfig.encryption);
+
+                ps.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong
+                    progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` });
+                });
+                ps.on('error', retryCallback);
+                ps.on('done', retryCallback);
+            });
+        }, (error) => {
+            if (error) return reject(error);
+            resolve();
+        });
+    });
+}
+
+async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof dataLayout, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    return new Promise((resolve, reject) => {
+        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+            retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error
+
+            const tarStream = tarPack(dataLayout, backupConfig.encryption);
+
+            tarStream.on('progress', function (progress) {
+                const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0MBps looks wrong
+                progressCallback({ message: `Uploading backup ${transferred}M@${speed}MBps` });
+            });
+            tarStream.on('error', retryCallback); // already returns BoxError
+
+            storage.api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, remotePath), tarStream, retryCallback);
+        }, (error) => {
+            if (error) return reject(error);
+            resolve();
+        });
+    });
+}

src/backuptask.js

@@ -12,40 +12,26 @@ exports = module.exports = {
     downloadMail,
 
     upload,
-
-    _restoreFsMetadata: restoreFsMetadata,
-    _saveFsMetadata: saveFsMetadata,
 };
 
 const apps = require('./apps.js'),
     assert = require('assert'),
-    async = require('async'),
+    backupFormat = require('./backupformat.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
-    crypto = require('crypto'),
     DataLayout = require('./datalayout.js'),
     database = require('./database.js'),
     debug = require('debug')('box:backuptask'),
-    fs = require('fs'),
-    once = require('./once.js'),
     path = require('path'),
     paths = require('./paths.js'),
-    progressStream = require('progress-stream'),
     safe = require('safetydance'),
     services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
-    storage = require('./storage.js'),
-    syncer = require('./syncer.js'),
-    tar = require('tar-fs'),
-    TransformStream = require('stream').Transform,
-    zlib = require('zlib'),
-    util = require('util');
+    storage = require('./storage.js');
 
 const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');
-const getBackupConfig = util.callbackify(settings.getBackupConfig);
-const runBackupUploadAsync = util.promisify(runBackupUpload);
 
 function canBackupApp(app) {
     // only backup apps that are installed or specific pending states
@@ -61,587 +47,32 @@ function canBackupApp(app) {
             app.installationState === apps.ISTATE_PENDING_UPDATE; // called from apptask
 }
 
-function encryptFilePath(filePath, encryption) {
-    assert.strictEqual(typeof filePath, 'string');
-    assert.strictEqual(typeof encryption, 'object');
-
-    const encryptedParts = filePath.split('/').map(function (part) {
-        let hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex'));
-        const iv = hmac.update(part).digest().slice(0, 16); // iv has to be deterministic, for our sync (copy) logic to work
-        const cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv);
-        let crypt = cipher.update(part);
-        crypt = Buffer.concat([ iv, crypt, cipher.final() ]);
-
-        return crypt.toString('base64')     // ensures path is valid
-            .replace(/\//g, '-')            // replace '/' of base64 since it conflicts with path separator
-            .replace(/=/g,'');              // strip trailing = padding. this is only needed if we concat base64 strings, which we don't
-    });
-
-    return encryptedParts.join('/');
-}
-
-function decryptFilePath(filePath, encryption) {
-    assert.strictEqual(typeof filePath, 'string');
-    assert.strictEqual(typeof encryption, 'object');
-
-    const decryptedParts = [];
-    for (let part of filePath.split('/')) {
-        part = part + Array(part.length % 4).join('='); // add back = padding
-        part = part.replace(/-/g, '/');                 // replace with '/'
-
-        try {
-            const buffer = Buffer.from(part, 'base64');
-            const iv = buffer.slice(0, 16);
-            let decrypt = crypto.createDecipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv);
-            const plainText = decrypt.update(buffer.slice(16));
-            const plainTextString = Buffer.concat([ plainText, decrypt.final() ]).toString('utf8');
-            const hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex'));
-            if (!hmac.update(plainTextString).digest().slice(0, 16).equals(iv)) return { error: new BoxError(BoxError.CRYPTO_ERROR, `mac error decrypting part ${part} of path ${filePath}`) };
-
-            decryptedParts.push(plainTextString);
-        } catch (error) {
-            debug(`Error decrypting part ${part} of path ${filePath}:`, error);
-            return { error: new BoxError(BoxError.CRYPTO_ERROR, `Error decrypting part ${part} of path ${filePath}: ${error.message}`) };
-        }
-    }
-
-    return { result: decryptedParts.join('/') };
-}
-
-class EncryptStream extends TransformStream {
-    constructor(encryption) {
-        super();
-        this._headerPushed = false;
-        this._iv = crypto.randomBytes(16);
-        this._cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.dataKey, 'hex'), this._iv);
-        this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex'));
-    }
-
-    pushHeaderIfNeeded() {
-        if (!this._headerPushed) {
-            const magic = Buffer.from('CBV2');
-            this.push(magic);
-            this._hmac.update(magic);
-            this.push(this._iv);
-            this._hmac.update(this._iv);
-            this._headerPushed = true;
-        }
-    }
-
-    _transform(chunk, ignoredEncoding, callback) {
-        this.pushHeaderIfNeeded();
-
-        try {
-            const crypt = this._cipher.update(chunk);
-            this._hmac.update(crypt);
-            callback(null, crypt);
-        } catch (error) {
-            callback(error);
-        }
-    }
-
-    _flush(callback) {
-        try {
-            this.pushHeaderIfNeeded(); // for 0-length files
-            const crypt = this._cipher.final();
-            this.push(crypt);
-            this._hmac.update(crypt);
-            callback(null, this._hmac.digest()); // +32 bytes
-        } catch (error) {
-            callback(error);
-        }
-    }
-}
-
-class DecryptStream extends TransformStream {
-    constructor(encryption) {
-        super();
-        this._key = Buffer.from(encryption.dataKey, 'hex');
-        this._header = Buffer.alloc(0);
-        this._decipher = null;
-        this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex'));
-        this._buffer = Buffer.alloc(0);
-    }
-
-    _transform(chunk, ignoredEncoding, callback) {
-        const needed = 20 - this._header.length; // 4 for magic, 16 for iv
-
-        if (this._header.length !== 20) { // not gotten header yet
-            this._header = Buffer.concat([this._header, chunk.slice(0, needed)]);
-            if (this._header.length !== 20) return callback();
-
-            if (!this._header.slice(0, 4).equals(new Buffer.from('CBV2'))) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid magic in header'));
-
-            const iv = this._header.slice(4);
-            this._decipher = crypto.createDecipheriv('aes-256-cbc', this._key, iv);
-            this._hmac.update(this._header);
-        }
-
-        this._buffer = Buffer.concat([ this._buffer, chunk.slice(needed) ]);
-        if (this._buffer.length < 32) return callback(); // hmac trailer length is 32
-
-        try {
-            const cipherText = this._buffer.slice(0, -32);
-            this._hmac.update(cipherText);
-            const plainText = this._decipher.update(cipherText);
-            this._buffer = this._buffer.slice(-32);
-            callback(null, plainText);
-        } catch (error) {
-            callback(error);
-        }
-    }
-
-    _flush (callback) {
-        if (this._buffer.length !== 32) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (not enough data)'));
-
-        try {
-            if (!this._hmac.digest().equals(this._buffer)) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (mac mismatch)'));
-
-            const plainText = this._decipher.final();
-            callback(null, plainText);
-        } catch (error) {
-            callback(error);
-        }
-    }
-}
-
-function createReadStream(sourceFile, encryption) {
-    assert.strictEqual(typeof sourceFile, 'string');
-    assert.strictEqual(typeof encryption, 'object');
-
-    const stream = fs.createReadStream(sourceFile);
-    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
-
-    stream.on('error', function (error) {
-        debug(`createReadStream: read stream error at ${sourceFile}`, error);
-        ps.emit('error', new BoxError(BoxError.FS_ERROR, `Error reading ${sourceFile}: ${error.message} ${error.code}`));
-    });
-
-    stream.on('open', () => ps.emit('open'));
-
-    if (encryption) {
-        let encryptStream = new EncryptStream(encryption);
-
-        encryptStream.on('error', function (error) {
-            debug(`createReadStream: encrypt stream error ${sourceFile}`, error);
-            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Encryption error at ${sourceFile}: ${error.message}`));
-        });
-
-        return stream.pipe(encryptStream).pipe(ps);
-    } else {
-        return stream.pipe(ps);
-    }
-}
-
-function createWriteStream(destFile, encryption) {
-    assert.strictEqual(typeof destFile, 'string');
-    assert.strictEqual(typeof encryption, 'object');
-
-    const stream = fs.createWriteStream(destFile);
-    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
-
-    stream.on('error', function (error) {
-        debug(`createWriteStream: write stream error ${destFile}`, error);
-        ps.emit('error', new BoxError(BoxError.FS_ERROR, `Write error ${destFile}: ${error.message}`));
-    });
-
-    stream.on('finish', function () {
-        debug('createWriteStream: done.');
-        // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not write
-        ps.emit('done');
-    });
-
-    if (encryption) {
-        let decrypt = new DecryptStream(encryption);
-        decrypt.on('error', function (error) {
-            debug(`createWriteStream: decrypt stream error ${destFile}`, error);
-            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Decryption error at ${destFile}: ${error.message}`));
-        });
-
-        ps.pipe(decrypt).pipe(stream);
-    } else {
-        ps.pipe(stream);
-    }
-
-    return ps;
-}
-
-function tarPack(dataLayout, encryption, callback) {
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof encryption, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const pack = tar.pack('/', {
-        dereference: false, // pack the symlink and not what it points to
-        entries: dataLayout.localPaths(),
-        ignoreStatError: (path, err) => {
-            debug(`tarPack: error stat'ing ${path} - ${err.code}`);
-            return err.code === 'ENOENT'; // ignore if file or dir got removed (probably some temporary file)
-        },
-        map: function(header) {
-            header.name = dataLayout.toRemotePath(header.name);
-            // the tar pax format allows us to encode filenames > 100 and size > 8GB (see #640)
-            // https://www.systutorials.com/docs/linux/man/5-star/
-            if (header.size > 8589934590 || header.name > 99) header.pax = { size: header.size };
-            return header;
-        },
-        strict: false // do not error for unknown types (skip fifo, char/block devices)
-    });
-
-    const gzip = zlib.createGzip({});
-    const ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds
-
-    pack.on('error', function (error) {
-        debug('tarPack: tar stream error.', error);
-        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    gzip.on('error', function (error) {
-        debug('tarPack: gzip stream error.', error);
-        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    if (encryption) {
-        const encryptStream = new EncryptStream(encryption);
-        encryptStream.on('error', function (error) {
-            debug('tarPack: encrypt stream error.', error);
-            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-        });
-
-        pack.pipe(gzip).pipe(encryptStream).pipe(ps);
-    } else {
-        pack.pipe(gzip).pipe(ps);
-    }
-
-    return callback(null, ps);
-}
-
-function sync(backupConfig, remotePath, dataLayout, progressCallback, callback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB
-    const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10);
-    const removeDir = util.callbackify(storage.api(backupConfig.provider).removeDir);
-    const remove = util.callbackify(storage.api(backupConfig.provider).remove);
-
-    syncer.sync(dataLayout, function processTask(task, iteratorCallback) {
-        debug('sync: processing task: %j', task);
-        // the empty task.path is special to signify the directory
-        const destPath = task.path && backupConfig.encryption ? encryptFilePath(task.path, backupConfig.encryption) : task.path;
-        const backupFilePath = path.join(storage.getBackupFilePath(backupConfig, remotePath, backupConfig.format), destPath);
-
-        if (task.operation === 'removedir') {
-            debug(`Removing directory ${backupFilePath}`);
-            return removeDir(backupConfig, backupFilePath, progressCallback, iteratorCallback);
-        } else if (task.operation === 'remove') {
-            debug(`Removing ${backupFilePath}`);
-            return remove(backupConfig, backupFilePath, iteratorCallback);
-        }
-
-        let retryCount = 0;
-        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-            retryCallback = once(retryCallback); // protect again upload() erroring much later after read stream error
-
-            ++retryCount;
-            if (task.operation === 'add') {
-                progressCallback({ message: `Adding ${task.path}` + (retryCount > 1 ?  ` (Try ${retryCount})` : '') });
-                debug(`Adding ${task.path} position ${task.position} try ${retryCount}`);
-                const stream = createReadStream(dataLayout.toLocalPath('./' + task.path), backupConfig.encryption);
-                stream.on('error', (error) => retryCallback(error.message.includes('ENOENT') ? null : error)); // ignore error if file disappears
-                stream.on('progress', function (progress) {
-                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                    if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0MBps looks wrong
-                    progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}MBps` }); // 0M@0MBps looks wrong
-                });
-                // only create the destination path when we have confirmation that the source is available. otherwise, we end up with
-                // files owned as 'root' and the cp later will fail
-                stream.on('open', function () {
-                    storage.api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, function (error) {
-                        debug(error ? `Error uploading ${task.path} try ${retryCount}: ${error.message}` : `Uploaded ${task.path}`);
-                        retryCallback(error);
-                    });
-                });
-            }
-        }, iteratorCallback);
-    }, concurrency, function (error) {
-        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-
-        callback();
-    });
-}
-
-// this is not part of 'snapshotting' because we need root access to traverse
-async function saveFsMetadata(dataLayout, metadataFile) {
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof metadataFile, 'string');
-
-    // contains paths prefixed with './'
-    const metadata = {
-        emptyDirs: [],
-        execFiles: [],
-        symlinks: []
-    };
-
-    // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
-    for (let lp of dataLayout.localPaths()) {
-        const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
-        if (emptyDirs === null) throw new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`);
-        if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
-
-        const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
-        if (execFiles === null) throw new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`);
-        if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
-
-        const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
-        if (symlinks === null) throw new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`);
-        if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => {
-            const target = safe.fs.readlinkSync(sl);
-            return { path: dataLayout.toRemotePath(sl), target };
-        }));
-    }
-
-    if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) throw new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`);
-}
-
 // this function is called via backupupload (since it needs root to traverse app's directory)
-function upload(remotePath, format, dataLayoutString, progressCallback, callback) {
+async function upload(remotePath, format, dataLayoutString, progressCallback) {
     assert.strictEqual(typeof remotePath, 'string');
     assert.strictEqual(typeof format, 'string');
     assert.strictEqual(typeof dataLayoutString, 'string');
     assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`upload: path ${remotePath} format ${format} dataLayout ${dataLayoutString}`);
 
     const dataLayout = DataLayout.fromString(dataLayoutString);
+    const backupConfig = await settings.getBackupConfig();
+    await safe(storage.api(backupConfig.provider).checkPreconditions(backupConfig, dataLayout));
 
-    getBackupConfig(async function (error, backupConfig) {
-        if (error) return callback(error);
-
-        const [preconditionError] = await safe(storage.api(backupConfig.provider).checkPreconditions(backupConfig, dataLayout));
-        if (preconditionError) return callback(preconditionError);
-
-        if (format === 'tgz') {
-            async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-                retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error
-
-                tarPack(dataLayout, backupConfig.encryption, function (error, tarStream) {
-                    if (error) return retryCallback(error);
-
-                    tarStream.on('progress', function (progress) {
-                        const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                        if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0MBps looks wrong
-                        progressCallback({ message: `Uploading backup ${transferred}M@${speed}MBps` });
-                    });
-                    tarStream.on('error', retryCallback); // already returns BoxError
-
-                    storage.api(backupConfig.provider).upload(backupConfig, storage.getBackupFilePath(backupConfig, remotePath, format), tarStream, retryCallback);
-                });
-            }, callback);
-        } else {
-            async.series([
-                saveFsMetadata.bind(null, dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`),
-                sync.bind(null, backupConfig, remotePath, dataLayout, progressCallback)
-            ], callback);
-        }
-    });
+    await backupFormat.api(format).upload(backupConfig, remotePath, dataLayout, progressCallback);
 }
 
-function tarExtract(inStream, dataLayout, encryption, callback) {
-    assert.strictEqual(typeof inStream, 'object');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof encryption, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const gunzip = zlib.createGunzip({});
-    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
-    const extract = tar.extract('/', {
-        map: function (header) {
-            header.name = dataLayout.toLocalPath(header.name);
-            return header;
-        },
-        dmode: 500 // ensure directory is writable
-    });
-
-    const emitError = once((error) => {
-        inStream.destroy();
-        ps.emit('error', error);
-    });
-
-    inStream.on('error', function (error) {
-        debug('tarExtract: input stream error.', error);
-        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    gunzip.on('error', function (error) {
-        debug('tarExtract: gunzip stream error.', error);
-        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    extract.on('error', function (error) {
-        debug('tarExtract: extract stream error.', error);
-        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    extract.on('finish', function () {
-        debug('tarExtract: done.');
-        // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not extract
-        ps.emit('done');
-    });
-
-    if (encryption) {
-        let decrypt = new DecryptStream(encryption);
-        decrypt.on('error', function (error) {
-            debug('tarExtract: decrypt stream error.', error);
-            emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`));
-        });
-        inStream.pipe(ps).pipe(decrypt).pipe(gunzip).pipe(extract);
-    } else {
-        inStream.pipe(ps).pipe(gunzip).pipe(extract);
-    }
-
-    callback(null, ps);
-}
-
-async function restoreFsMetadata(dataLayout, metadataFile) {
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof metadataFile, 'string');
-
-    debug(`Recreating empty directories in ${dataLayout.toString()}`);
-
-    const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
-    if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
-    const metadata = safe.JSON.parse(metadataJson);
-    if (metadata === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message);
-
-    for (const emptyDir of metadata.emptyDirs) {
-        const [mkdirError] = await safe(fs.promises.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }));
-        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to create path: ${mkdirError.message}`);
-    }
-
-    for (const execFile of metadata.execFiles) {
-        const [chmodError] = await safe(fs.promises.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8)));
-        if (chmodError) throw new BoxError(BoxError.FS_ERROR, `unable to chmod: ${chmodError.message}`);
-    }
-
-    for (const symlink of (metadata.symlinks || [])) {
-        if (!symlink.target) continue;
-        // the path may not exist if we had a directory full of symlinks
-        const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }));
-        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink (mkdir): ${mkdirError.message}`);
-        const [symlinkError] = await safe(fs.promises.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file'));
-        if (symlinkError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink: ${symlinkError.message}`);
-    }
-}
-
-function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof backupFilePath, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
-
-    function downloadFile(entry, done) {
-        let relativePath = path.relative(backupFilePath, entry.fullPath);
-        if (backupConfig.encryption) {
-            const { error, result } = decryptFilePath(relativePath, backupConfig.encryption);
-            if (error) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file'));
-            relativePath = result;
-        }
-        const destFilePath = dataLayout.toLocalPath('./' + relativePath);
-
-        fs.mkdir(path.dirname(destFilePath), { recursive: true }, function (error) {
-            if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));
-
-            async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-                storage.api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) {
-                    if (error) {
-                        progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
-                        return retryCallback(error);
-                    }
-
-                    let destStream = createWriteStream(destFilePath, backupConfig.encryption);
-
-                    // protect against multiple errors. must destroy the write stream so that a previous retry does not write
-                    let closeAndRetry = once((error) => {
-                        if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
-                        else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` });
-                        sourceStream.destroy();
-                        destStream.destroy();
-                        retryCallback(error);
-                    });
-
-                    destStream.on('progress', function (progress) {
-                        const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                        if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0MBps looks wrong
-                        progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}MBps` });
-                    });
-                    destStream.on('error', closeAndRetry);
-
-                    sourceStream.on('error', closeAndRetry);
-
-                    progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
-
-                    sourceStream.pipe(destStream, { end: true }).on('done', closeAndRetry);
-                });
-            }, done);
-        });
-    }
-
-    storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
-        // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
-        const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
-
-        async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
-    }, callback);
-}
-
-function download(backupConfig, remotePath, format, dataLayout, progressCallback, callback) {
+async function download(backupConfig, remotePath, format, dataLayout, progressCallback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof remotePath, 'string');
     assert.strictEqual(typeof format, 'string');
     assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
     assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`download: Downloading ${remotePath} of format ${format} to ${dataLayout.toString()}`);
 
-    const backupFilePath = storage.getBackupFilePath(backupConfig, remotePath, format);
-
-    if (format === 'tgz') {
-        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-            progressCallback({ message: `Downloading backup ${remotePath}` });
-
-            storage.api(backupConfig.provider).download(backupConfig, backupFilePath, function (error, sourceStream) {
-                if (error) return retryCallback(error);
-
-                tarExtract(sourceStream, dataLayout, backupConfig.encryption, function (error, ps) {
-                    if (error) return retryCallback(error);
-
-                    ps.on('progress', function (progress) {
-                        const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                        if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong
-                        progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` });
-                    });
-                    ps.on('error', retryCallback);
-                    ps.on('done', retryCallback);
-                });
-            });
-        }, callback);
-    } else {
-        downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, async function (error) {
-            if (error) return callback(error);
-
-            [error] = await safe(restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`));
-            callback(error);
-        });
-    }
+    await backupFormat.api(format).download(backupConfig, remotePath, dataLayout, progressCallback);
 }
 
 async function restore(backupConfig, remotePath, progressCallback) {
@@ -653,7 +84,7 @@ async function restore(backupConfig, remotePath, progressCallback) {
     if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
     const dataLayout = new DataLayout(boxDataDir, []);
 
-    await util.promisify(download)(backupConfig, remotePath, backupConfig.format, dataLayout, progressCallback);
+    await download(backupConfig, remotePath, backupConfig.format, dataLayout, progressCallback);
 
     debug('restore: download completed, importing database');
 
@@ -671,20 +102,18 @@ async function downloadApp(app, restoreConfig, progressCallback) {
 
     const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
     if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
-    const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
+    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);
 
     const startTime = new Date();
     const backupConfig = restoreConfig.backupConfig || await settings.getBackupConfig();
 
-    const downloadAsync = util.promisify(download);
-    await downloadAsync(backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
+    await download(backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
     debug('downloadApp: time: %s', (new Date() - startTime)/1000);
 }
 
-function runBackupUpload(uploadConfig, progressCallback, callback) {
+async function runBackupUpload(uploadConfig, progressCallback) {
     assert.strictEqual(typeof uploadConfig, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
 
     const { remotePath, backupConfig, dataLayout, progressTag } = uploadConfig;
     assert.strictEqual(typeof remotePath, 'string');
@@ -692,8 +121,6 @@ function runBackupUpload(uploadConfig, progressCallback, callback) {
     assert.strictEqual(typeof progressTag, 'string');
     assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
 
-    let result = ''; // the script communicates error result as a string
-
     // https://stackoverflow.com/questions/48387040/node-js-recommended-max-old-space-size
     const envCopy = Object.assign({}, process.env);
     if (backupConfig.memoryLimit && backupConfig.memoryLimit >= 2*1024*1024*1024) {
@@ -702,19 +129,19 @@ function runBackupUpload(uploadConfig, progressCallback, callback) {
         envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`;
     }
 
-    shell.sudo(`backup-${remotePath}`, [ BACKUP_UPLOAD_CMD, remotePath, backupConfig.format, dataLayout.toString() ], { env: envCopy, preserveEnv: true, ipc: true }, function (error) {
-        if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
-            return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed'));
-        } else if (error && error.code === 50) { // exited with error
-            return callback(new BoxError(BoxError.EXTERNAL_ERROR, result));
-        }
-
-        callback();
-    }).on('message', function (progress) { // this is { message } or { result }
+    let result = ''; // the script communicates error result as a string
+    function onMessage(progress) { // this is { message } or { result }
         if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
         debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
         result = progress.result;
-    });
+    }
+
+    const [error] = await safe(shell.promises.sudo(`backup-${remotePath}`, [ BACKUP_UPLOAD_CMD, remotePath, backupConfig.format, dataLayout.toString() ], { env: envCopy, preserveEnv: true, ipc: true, onMessage }));
+    if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
+        throw new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed');
+    } else if (error && error.code === 50) { // exited with error
+        throw new BoxError(BoxError.EXTERNAL_ERROR, result);
+    }
 }
 
 async function snapshotBox(progressCallback) {
@@ -748,7 +175,7 @@ async function uploadBoxSnapshot(backupConfig, progressCallback) {
 
     const startTime = new Date();
 
-    await runBackupUploadAsync(uploadConfig, progressCallback);
+    await runBackupUpload(uploadConfig, progressCallback);
 
     debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);
 
@@ -762,18 +189,12 @@ async function copy(backupConfig, srcRemotePath, destRemotePath, progressCallbac
     assert.strictEqual(typeof progressCallback, 'function');
 
     const { provider, format } = backupConfig;
+    const oldFilePath = backupFormat.api(format).getBackupFilePath(backupConfig, srcRemotePath);
+    const newFilePath = backupFormat.api(format).getBackupFilePath(backupConfig, destRemotePath);
 
-    return new Promise((resolve, reject) => {
-        const startTime = new Date();
-
-        const copyEvents = storage.api(provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, srcRemotePath, format), storage.getBackupFilePath(backupConfig, destRemotePath, format));
-        copyEvents.on('progress', (message) => progressCallback({ message }));
-        copyEvents.on('done', function (error) {
-            if (error) return reject(error);
-            debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
-            resolve();
-        });
-    });
+    const startTime = new Date();
+    await safe(storage.api(provider).copy(backupConfig, oldFilePath, newFilePath, progressCallback));
+    debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
 }
 
 async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback) {
@@ -897,7 +318,7 @@ async function uploadAppSnapshot(backupConfig, app, progressCallback) {
     const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
     if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`);
 
-    const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
+    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);
 
     progressCallback({ message: `Uploading app snapshot ${app.fqdn}`});
 
@@ -910,7 +331,7 @@ async function uploadAppSnapshot(backupConfig, app, progressCallback) {
 
     const startTime = new Date();
 
-    await runBackupUploadAsync(uploadConfig, progressCallback);
+    await runBackupUpload(uploadConfig, progressCallback);
 
     debug(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`);
 
@@ -954,7 +375,7 @@ async function uploadMailSnapshot(backupConfig, progressCallback) {
 
     const startTime = new Date();
 
-    await runBackupUploadAsync(uploadConfig, progressCallback);
+    await runBackupUpload(uploadConfig, progressCallback);
 
     debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`);
 
@@ -1025,8 +446,7 @@ async function downloadMail(restoreConfig, progressCallback) {
 
     const startTime = new Date();
 
-    const downloadAsync = util.promisify(download);
-    await downloadAsync(restoreConfig.backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
+    await download(restoreConfig.backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
     debug('downloadMail: time: %s', (new Date() - startTime)/1000);
 }
 

src/cloudron.js

@@ -155,6 +155,7 @@ async function getConfig() {
     return {
         apiServerOrigin: settings.apiServerOrigin(),
         webServerOrigin: settings.webServerOrigin(),
+        consoleServerOrigin: settings.consoleServerOrigin(),
         adminDomain: settings.dashboardDomain(),
         adminFqdn: settings.dashboardFqdn(),
         mailFqdn: settings.mailFqdn(),

src/constants.js

@@ -72,6 +72,6 @@ exports = module.exports = {
 
     FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
 
-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '7.0.0-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '7.2.0-test'
 };
 

src/dns.js

@@ -51,6 +51,7 @@ function api(provider) {
     case 'namecom': return require('./dns/namecom.js');
     case 'namecheap': return require('./dns/namecheap.js');
     case 'netcup': return require('./dns/netcup.js');
+    case 'hetzner': return require('./dns/hetzner.js');
     case 'noop': return require('./dns/noop.js');
     case 'manual': return require('./dns/manual.js');
     case 'wildcard': return require('./dns/wildcard.js');

src/dns/digitalocean.js

@@ -18,13 +18,12 @@ const assert = require('assert'),
     dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
-    util = require('util'),
     waitForDns = require('./waitfordns.js');
 
 const DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com';
 
 function formatError(response) {
-    return util.format('DigitalOcean DNS error [%s] %j', response.statusCode, response.body);
+    return `DigitalOcean DNS error ${response.statusCode} ${JSON.stringify(response.body)}`;
 }
 
 function removePrivateFields(domainObject) {

src/dns/godaddy.js

@@ -126,6 +126,13 @@ async function del(domainObject, location, type, values) {
 
     debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
+    const result = await get(domainObject, location, type);
+    if (result.length === 0) return;
+
+    const tmp = result.filter(r => !values.includes(r));
+
+    if (tmp.length) return await upsert(domainObject, location, type, tmp); // only remove 'values'
+
     const [error, response] = await safe(superagent.del(`${GODADDY_API}/${zoneName}/records/${type}/${name}`)
         .set('Authorization', `sso-key ${domainConfig.apiKey}:${domainConfig.apiSecret}`)
         .timeout(30 * 1000)

src/dns/hetzner.js

@@ -0,0 +1,259 @@
+'use strict';
+
+exports = module.exports = {
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDomainConfig
+};
+
+const assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    constants = require('../constants.js'),
+    debug = require('debug')('box:dns/digitalocean'),
+    dig = require('../dig.js'),
+    dns = require('../dns.js'),
+    safe = require('safetydance'),
+    superagent = require('superagent'),
+    waitForDns = require('./waitfordns.js');
+
+const ENDPOINT = 'https://dns.hetzner.com/api/v1';
+
+function formatError(response) {
+    return `Hetzner DNS error ${response.statusCode} ${JSON.stringify(response.body)}`;
+}
+
+function removePrivateFields(domainObject) {
+    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    return domainObject;
+}
+
+function injectPrivateFields(newConfig, currentConfig) {
+    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+}
+
+async function getZone(domainConfig, zoneName) {
+    assert.strictEqual(typeof domainConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+
+    const [error, response] = await safe(superagent.get(`${ENDPOINT}/zones`)
+        .set('Auth-API-Token', domainConfig.token)
+        .query({ search_name: zoneName })
+        .timeout(30 * 1000)
+        .retry(5)
+        .ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.statusCode === 401 || response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+    if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+
+    if (!Array.isArray(response.body.zones)) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+
+    const zone = response.body.zones.filter(z => z.name === zoneName);
+    if (zone.length === 0) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+    return zone[0];
+}
+
+async function getZoneRecords(domainConfig, zone, name, type) {
+    assert.strictEqual(typeof domainConfig, 'object');
+    assert.strictEqual(typeof zone, 'object');
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof type, 'string');
+
+    let page = 1, matchingRecords = [];
+
+    debug(`getInternal: getting dns records of ${zone.name} with ${name} and type ${type}`);
+
+    const perPage = 50;
+
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const [error, response] = await safe(superagent.get(`${ENDPOINT}/records`)
+            .set('Auth-API-Token', domainConfig.token)
+            .query({ zone_id: zone.id, page, per_page: perPage })
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
+
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, formatError(response));
+        if (response.statusCode === 401 || response.statusCode === 403) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+
+        matchingRecords = matchingRecords.concat(response.body.records.filter(function (record) {
+            return (record.type === type && record.name === name);
+        }));
+
+        if (response.body.records.length < perPage) break;
+
+        ++page;
+    }
+
+    return matchingRecords;
+}
+
+async function upsert(domainObject, location, type, values) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = dns.getName(domainObject, location, type) || '@';
+
+    debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
+
+    const zone = await getZone(domainConfig, zoneName);
+    const records = await getZoneRecords(domainConfig, zone, name, type);
+
+    // used to track available records to update instead of create
+    let i = 0;
+
+    for (let value of values) {
+        const data = {
+            type,
+            name,
+            value,
+            ttl: 60,
+            zone_id: zone.id
+        };
+
+        if (i >= records.length) {
+            const [error, response] = await safe(superagent.post(`${ENDPOINT}/records`)
+                .set('Auth-API-Token', domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
+
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+        } else {
+            const [error, response] = await safe(superagent.put(`${ENDPOINT}/records/${records[i].id}`)
+                .set('Auth-API-Token', domainConfig.token)
+                .send(data)
+                .timeout(30 * 1000)
+                .retry(5)
+                .ok(() => true));
+
+            ++i;
+
+            if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+            if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+            if (response.statusCode === 422) throw new BoxError(BoxError.BAD_FIELD, response.body.message);
+            if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+        }
+    }
+
+    for (let j = values.length + 1; j < records.length; j++) {
+        const [error] = await safe(superagent.del(`${ENDPOINT}/records/${records[j].id}`)
+            .set('Auth-API-Token', domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
+
+        if (error) debug(`upsert: error removing record ${records[j].id}: ${error.message}`);
+    }
+
+    debug('upsert: completed');
+}
+
+async function get(domainObject, location, type) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = dns.getName(domainObject, location, type) || '@';
+
+    const zone = await getZone(domainConfig, zoneName);
+    const result = await getZoneRecords(domainConfig, zone, name, type);
+
+    return result.map(function (record) { return record.value; });
+}
+
+async function del(domainObject, location, type, values) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = dns.getName(domainObject, location, type) || '@';
+
+    const zone = await getZone(domainConfig, zoneName);
+    const records = await getZoneRecords(domainConfig, zone, name, type);
+    if (records.length === 0) return;
+
+    const matchingRecords = records.filter(function (record) { return values.some(function (value) { return value === record.value; }); });
+    if (matchingRecords.length === 0) return;
+
+    for (const r of matchingRecords) {
+        const [error, response] = await safe(superagent.del(`${ENDPOINT}/records/${r.id}`)
+            .set('Auth-API-Token', domainConfig.token)
+            .timeout(30 * 1000)
+            .retry(5)
+            .ok(() => true));
+
+        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+        if (response.statusCode === 404) return;
+        if (response.statusCode === 403 || response.statusCode === 401) throw new BoxError(BoxError.ACCESS_DENIED, formatError(response));
+        if (response.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, formatError(response));
+    }
+}
+
+async function wait(domainObject, subdomain, type, value, options) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+
+    const fqdn = dns.fqdn(subdomain, domainObject);
+
+    await waitForDns(fqdn, domainObject.zoneName, type, value, options);
+}
+
+async function verifyDomainConfig(domainObject) {
+    assert.strictEqual(typeof domainObject, 'object');
+
+    const domainConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
+
+    const ip = '127.0.0.1';
+
+    const credentials = {
+        token: domainConfig.token
+    };
+
+    if (process.env.BOX_ENV === 'test') return credentials; // this shouldn't be here
+
+    const [error, nameservers] = await safe(dig.resolve(zoneName, 'NS', { timeout: 5000 }));
+    if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
+    if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
+
+    // https://docs.hetzner.com/dns-console/dns/general/dns-overview#the-hetzner-online-name-servers-are
+    if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('oxygen.ns.hetzner.com') === -1) {
+        debug('verifyDomainConfig: %j does not contain Hetzner NS', nameservers);
+        throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Hetzner');
+    }
+
+    const location = 'cloudrontestdns';
+
+    await upsert(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record added');
+
+    await del(domainObject, location, 'A', [ ip ]);
+    debug('verifyDomainConfig: Test A record removed again');
+
+    return credentials;
+}

src/dns/vultr.js

@@ -99,7 +99,7 @@ async function upsert(domainObject, location, type, values) {
     for (const value of values) {
         const data = {
             type,
-            ttl: 300 // lowest
+            ttl: 120 // lowest
         };
 
         if (type === 'MX') {

src/docker.js

@@ -20,13 +20,15 @@ exports = module.exports = {
     createSubcontainer,
     inspect,
     getContainerIp,
-    execContainer,
     getEvents,
     memoryUsage,
-    createVolume,
-    removeVolume,
-    clearVolume,
+
     update,
+
+    createExec,
+    startExec,
+    getExec,
+    resizeExec
 };
 
 const apps = require('./apps.js'),
@@ -36,7 +38,6 @@ const apps = require('./apps.js'),
     debug = require('debug')('box:docker'),
     delay = require('./delay.js'),
     Docker = require('dockerode'),
-    path = require('path'),
     reverseProxy = require('./reverseproxy.js'),
     services = require('./services.js'),
     settings = require('./settings.js'),
@@ -46,9 +47,6 @@ const apps = require('./apps.js'),
     volumes = require('./volumes.js'),
     _ = require('underscore');
 
-const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
-    MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');
-
 const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
 const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
 
@@ -194,15 +192,17 @@ async function getAddonMounts(app) {
 
     for (const addon of Object.keys(addons)) {
         switch (addon) {
-        case 'localstorage':
+        case 'localstorage': {
+            const storageDir = await apps.getStorageDir(app);
             mounts.push({
                 Target: '/app/data',
-                Source: `${app.id}-localstorage`,
-                Type: 'volume',
+                Source: storageDir,
+                Type: 'bind',
                 ReadOnly: false
             });
 
             break;
+        }
         case 'tls': {
             const bundle = await reverseProxy.getCertificatePath(app.fqdn, app.domain);
 
@@ -394,6 +394,7 @@ async function createSubcontainer(app, name, cmd, options) {
         // ipv6 for new interfaces is disabled in the container. this prevents the openvpn tun device having ipv6
         // See https://github.com/moby/moby/issues/20569 and https://github.com/moby/moby/issues/33099
         containerOptions.HostConfig.Sysctls['net.ipv6.conf.all.disable_ipv6'] = '0';
+        containerOptions.HostConfig.Sysctls['net.ipv6.conf.all.forwarding'] = '1';
     }
     if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping
     if (!capabilities.includes('ping')) containerOptions.HostConfig.CapDrop.push('NET_RAW'); // NET_RAW is included by default by Docker
@@ -558,30 +559,51 @@ async function getContainerIp(containerId) {
     return ip;
 }
 
-async function execContainer(containerId, options) {
+async function createExec(containerId, options) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof options, 'object');
 
     const container = gConnection.getContainer(containerId);
-
-    const [error, exec] = await safe(container.exec(options.execOptions));
+    const [error, exec] = await safe(container.exec(options));
+    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND);
     if (error && error.statusCode === 409) throw new BoxError(BoxError.BAD_STATE, error.message); // container restarting/not running
     if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
 
-    const [startError, stream] = await safe(exec.start(options.startOptions)); /* in hijacked mode, stream is a net.socket */
-    if (startError) throw new BoxError(BoxError.DOCKER_ERROR, startError);
+    return exec.id;
+}
 
-    if (options.rows && options.columns) {
-        // there is a race where resizing too early results in a 404 "no such exec"
-        // https://git.cloudron.io/cloudron/box/issues/549
-        setTimeout(function () {
-            exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
-        }, 2000);
-    }
+async function startExec(execId, options) {
+    assert.strictEqual(typeof execId, 'string');
+    assert.strictEqual(typeof options, 'object');
 
+    const exec = gConnection.getExec(execId);
+    const [error, stream] = await safe(exec.start(options)); /* in hijacked mode, stream is a net.socket */
+    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND);
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
     return stream;
 }
 
+async function getExec(execId) {
+    assert.strictEqual(typeof execId, 'string');
+
+    const exec = gConnection.getExec(execId);
+    const [error, result] = await safe(exec.inspect());
+    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to find exec container ${execId}`);
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
+
+    return { exitCode: result.ExitCode, running: result.Running };
+}
+
+async function resizeExec(execId, options) {
+    assert.strictEqual(typeof execId, 'string');
+    assert.strictEqual(typeof options, 'object');
+
+    const exec = gConnection.getExec(execId);
+    const [error] = await safe(exec.resize(options)); // { h, w }
+    if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND);
+    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
+}
+
 async function getEvents(options) {
     assert.strictEqual(typeof options, 'object');
 
@@ -602,53 +624,6 @@ async function memoryUsage(containerId) {
     return result;
 }
 
-async function createVolume(name, volumeDataDir, labels) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof volumeDataDir, 'string');
-    assert.strictEqual(typeof labels, 'object');
-
-    const volumeOptions = {
-        Name: name,
-        Driver: 'local',
-        DriverOpts: { // https://github.com/moby/moby/issues/19990#issuecomment-248955005
-            type: 'none',
-            device: volumeDataDir,
-            o: 'bind'
-        },
-        Labels: labels
-    };
-
-    // requires sudo because the path can be outside appsdata
-    let [error] = await safe(shell.promises.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}));
-    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`);
-
-    [error] = await safe(gConnection.createVolume(volumeOptions));
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
-}
-
-async function clearVolume(name, options) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof options, 'object');
-
-    let volume = gConnection.getVolume(name);
-    let [error, v] = await safe(volume.inspect());
-    if (error && error.statusCode === 404) return;
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
-
-    const volumeDataDir = v.Options.device;
-    [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, options.removeDirectory ? 'rmdir' : 'clear', volumeDataDir ], {});
-    if (error) throw new BoxError(BoxError.FS_ERROR, error);
-}
-
-// this only removes the volume and not the data
-async function removeVolume(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    let volume = gConnection.getVolume(name);
-    const [error] = await safe(volume.remove());
-    if (error && error.statusCode !== 404) throw new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume: ${error.message}`);
-}
-
 async function info() {
     const [error, result] = await safe(gConnection.info());
     if (error) throw new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker');

src/domains.js

@@ -54,6 +54,7 @@ function api(provider) {
     case 'digitalocean': return require('./dns/digitalocean.js');
     case 'gandi': return require('./dns/gandi.js');
     case 'godaddy': return require('./dns/godaddy.js');
+    case 'hetzner': return require('./dns/hetzner.js');
     case 'linode': return require('./dns/linode.js');
     case 'vultr': return require('./dns/vultr.js');
     case 'namecom': return require('./dns/namecom.js');

src/hush.js

@@ -0,0 +1,225 @@
+'use strict';
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    crypto = require('crypto'),
+    debug = require('debug')('box:hush'),
+    fs = require('fs'),
+    progressStream = require('progress-stream'),
+    TransformStream = require('stream').Transform;
+
+class EncryptStream extends TransformStream {
+    constructor(encryption) {
+        super();
+        this._headerPushed = false;
+        this._iv = crypto.randomBytes(16);
+        this._cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.dataKey, 'hex'), this._iv);
+        this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex'));
+    }
+
+    pushHeaderIfNeeded() {
+        if (!this._headerPushed) {
+            const magic = Buffer.from('CBV2');
+            this.push(magic);
+            this._hmac.update(magic);
+            this.push(this._iv);
+            this._hmac.update(this._iv);
+            this._headerPushed = true;
+        }
+    }
+
+    _transform(chunk, ignoredEncoding, callback) {
+        this.pushHeaderIfNeeded();
+
+        try {
+            const crypt = this._cipher.update(chunk);
+            this._hmac.update(crypt);
+            callback(null, crypt);
+        } catch (error) {
+            callback(new BoxError(BoxError.CRYPTO_ERROR, `Encryption error when updating: ${error.message}`));
+        }
+    }
+
+    _flush(callback) {
+        try {
+            this.pushHeaderIfNeeded(); // for 0-length files
+            const crypt = this._cipher.final();
+            this.push(crypt);
+            this._hmac.update(crypt);
+            callback(null, this._hmac.digest()); // +32 bytes
+        } catch (error) {
+            callback(new BoxError(BoxError.CRYPTO_ERROR, `Encryption error when flushing: ${error.message}`));
+        }
+    }
+}
+
+class DecryptStream extends TransformStream {
+    constructor(encryption) {
+        super();
+        this._key = Buffer.from(encryption.dataKey, 'hex');
+        this._header = Buffer.alloc(0);
+        this._decipher = null;
+        this._hmac = crypto.createHmac('sha256', Buffer.from(encryption.dataHmacKey, 'hex'));
+        this._buffer = Buffer.alloc(0);
+    }
+
+    _transform(chunk, ignoredEncoding, callback) {
+        const needed = 20 - this._header.length; // 4 for magic, 16 for iv
+
+        if (this._header.length !== 20) { // not gotten header yet
+            this._header = Buffer.concat([this._header, chunk.slice(0, needed)]);
+            if (this._header.length !== 20) return callback();
+
+            if (!this._header.slice(0, 4).equals(new Buffer.from('CBV2'))) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid magic in header'));
+
+            const iv = this._header.slice(4);
+            this._decipher = crypto.createDecipheriv('aes-256-cbc', this._key, iv);
+            this._hmac.update(this._header);
+        }
+
+        this._buffer = Buffer.concat([ this._buffer, chunk.slice(needed) ]);
+        if (this._buffer.length < 32) return callback(); // hmac trailer length is 32
+
+        try {
+            const cipherText = this._buffer.slice(0, -32);
+            this._hmac.update(cipherText);
+            const plainText = this._decipher.update(cipherText);
+            this._buffer = this._buffer.slice(-32);
+            callback(null, plainText);
+        } catch (error) {
+            callback(new BoxError(BoxError.CRYPTO_ERROR, `Decryption error: ${error.message}`));
+        }
+    }
+
+    _flush (callback) {
+        if (this._buffer.length !== 32) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (not enough data)'));
+
+        try {
+            if (!this._hmac.digest().equals(this._buffer)) return callback(new BoxError(BoxError.CRYPTO_ERROR, 'Invalid password or tampered file (mac mismatch)'));
+
+            const plainText = this._decipher.final();
+            callback(null, plainText);
+        } catch (error) {
+            callback(new BoxError(BoxError.CRYPTO_ERROR, `Invalid password or tampered file: ${error.message}`));
+        }
+    }
+}
+
+function encryptFilePath(filePath, encryption) {
+    assert.strictEqual(typeof filePath, 'string');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const encryptedParts = filePath.split('/').map(function (part) {
+        let hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex'));
+        const iv = hmac.update(part).digest().slice(0, 16); // iv has to be deterministic, for our sync (copy) logic to work
+        const cipher = crypto.createCipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv);
+        let crypt = cipher.update(part);
+        crypt = Buffer.concat([ iv, crypt, cipher.final() ]);
+
+        return crypt.toString('base64')     // ensures path is valid
+            .replace(/\//g, '-')            // replace '/' of base64 since it conflicts with path separator
+            .replace(/=/g,'');              // strip trailing = padding. this is only needed if we concat base64 strings, which we don't
+    });
+
+    return encryptedParts.join('/');
+}
+
+function decryptFilePath(filePath, encryption) {
+    assert.strictEqual(typeof filePath, 'string');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const decryptedParts = [];
+    for (let part of filePath.split('/')) {
+        part = part + Array(part.length % 4).join('='); // add back = padding
+        part = part.replace(/-/g, '/');                 // replace with '/'
+
+        try {
+            const buffer = Buffer.from(part, 'base64');
+            const iv = buffer.slice(0, 16);
+            let decrypt = crypto.createDecipheriv('aes-256-cbc', Buffer.from(encryption.filenameKey, 'hex'), iv);
+            const plainText = decrypt.update(buffer.slice(16));
+            const plainTextString = Buffer.concat([ plainText, decrypt.final() ]).toString('utf8');
+            const hmac = crypto.createHmac('sha256', Buffer.from(encryption.filenameHmacKey, 'hex'));
+            if (!hmac.update(plainTextString).digest().slice(0, 16).equals(iv)) return { error: new BoxError(BoxError.CRYPTO_ERROR, `mac error decrypting part ${part} of path ${filePath}`) };
+
+            decryptedParts.push(plainTextString);
+        } catch (error) {
+            debug(`Error decrypting part ${part} of path ${filePath}:`, error);
+            return { error: new BoxError(BoxError.CRYPTO_ERROR, `Error decrypting part ${part} of path ${filePath}: ${error.message}`) };
+        }
+    }
+
+    return { result: decryptedParts.join('/') };
+}
+
+function createReadStream(sourceFile, encryption) {
+    assert.strictEqual(typeof sourceFile, 'string');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const stream = fs.createReadStream(sourceFile);
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+
+    stream.on('error', function (error) {
+        debug(`createReadStream: read stream error at ${sourceFile}`, error);
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, `Error reading ${sourceFile}: ${error.message} ${error.code}`));
+    });
+
+    stream.on('open', () => ps.emit('open'));
+
+    if (encryption) {
+        let encryptStream = new EncryptStream(encryption);
+
+        encryptStream.on('error', function (error) {
+            debug(`createReadStream: encrypt stream error ${sourceFile}`, error);
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Encryption error at ${sourceFile}: ${error.message}`));
+        });
+
+        return stream.pipe(encryptStream).pipe(ps);
+    } else {
+        return stream.pipe(ps);
+    }
+}
+
+function createWriteStream(destFile, encryption) {
+    assert.strictEqual(typeof destFile, 'string');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const stream = fs.createWriteStream(destFile);
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+
+    stream.on('error', function (error) {
+        debug(`createWriteStream: write stream error ${destFile}`, error);
+        ps.emit('error', new BoxError(BoxError.FS_ERROR, `Write error ${destFile}: ${error.message}`));
+    });
+
+    stream.on('finish', function () {
+        debug('createWriteStream: done.');
+        // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not write
+        ps.emit('done');
+    });
+
+    if (encryption) {
+        let decrypt = new DecryptStream(encryption);
+        decrypt.on('error', function (error) {
+            debug(`createWriteStream: decrypt stream error ${destFile}`, error);
+            ps.emit('error', new BoxError(BoxError.CRYPTO_ERROR, `Decryption error at ${destFile}: ${error.message}`));
+        });
+
+        ps.pipe(decrypt).pipe(stream);
+    } else {
+        ps.pipe(stream);
+    }
+
+    return ps;
+}
+
+exports = module.exports = {
+    EncryptStream,
+    DecryptStream,
+
+    encryptFilePath,
+    decryptFilePath,
+
+    createReadStream,
+    createWriteStream
+};

src/infra_version.js

@@ -18,7 +18,7 @@ exports = module.exports = {
         'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.4.0@sha256:45817f1631992391d585f171498d257487d872480fd5646723a2b956cc4ef15d' },
         'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.2.1@sha256:75cef64ba4917ba9ec68bc0c9d9ba3a9eeae00a70173cd6d81cc6118038737d9' },
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.3.1@sha256:b0c564d097b765d4a639330843e2e813d2c87fc8ed34b7df7550bf2c6df0012c' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.0@sha256:c8ebdbe2663b26fcd58b1e6b97906b62565adbe4a06256ba0f86114f78b37e6b' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.1@sha256:f7f689beea07b1c6a9503a48f6fb38ef66e5b22f59fc585a92842a6578b33d46' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.3.0@sha256:89c4e8083631b6d16b5d630d9b27f8ecf301c62f81219d77bd5948a1f4a4375c' },
         'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.6.1@sha256:b8b93f007105080d4812a05648e6bc5e15c95c63f511c829cbc14a163d9ea029' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.1.0@sha256:30ec3a01964a1e01396acf265183997c3e17fb07eac1a82b979292cc7719ff4b' },

src/ldap.js

@@ -209,37 +209,6 @@ async function groupSearch(req, res, next) {
 
     const results = [];
 
-    // those are the old virtual groups for backwards compat
-    const virtualGroups = [{
-        name: 'users',
-        admin: false
-    }, {
-        name: 'admins',
-        admin: true
-    }];
-
-    virtualGroups.forEach(function (group) {
-        const dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-        const members = group.admin ? usersWithAccess.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : usersWithAccess;
-
-        const obj = {
-            dn: dn.toString(),
-            attributes: {
-                objectclass: ['group'],
-                cn: group.name,
-                memberuid: members.map(function(entry) { return entry.id; }).sort()
-            }
-        };
-
-        // ensure all filter values are also lowercase
-        const lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
-        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
-        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-            results.push(obj);
-        }
-    });
-
     let [errorGroups, resultGroups] = await safe(groups.listWithMembers());
     if (errorGroups) return next(new ldap.OperationsError(errorGroups.toString()));
 
@@ -558,7 +527,7 @@ async function userSearchSftp(req, res, next) {
     const obj = {
         dn: ldap.parseDN(`cn=${username}@${appFqdn},ou=sftp,dc=cloudron`).toString(),
         attributes: {
-            homeDirectory: app.dataDir ? `/mnt/app-${app.id}` : `/mnt/appsdata/${app.id}/data`, // see also sftp.js
+            homeDirectory: app.storageVolumeId ? `/mnt/app-${app.id}` : `/mnt/appsdata/${app.id}/data`, // see also sftp.js
             objectclass: ['user'],
             objectcategory: 'person',
             cn: user.id,

src/mail.js

@@ -22,6 +22,7 @@ exports = module.exports = {
     setDnsRecords,
 
     validateName,
+    validateDisplayName,
 
     setMailFromValidation,
     setCatchAllAddress,
@@ -65,7 +66,6 @@ exports = module.exports = {
     TYPE_LIST: 'list',
     TYPE_ALIAS: 'alias',
 
-    _validateName: validateName,
     _delByDomain: delByDomain,
     _updateDomain: updateDomain
 };
@@ -169,6 +169,16 @@ function validateName(name) {
     return null;
 }
 
+function validateDisplayName(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name must be atleast 1 char');
+    if (name.length >= 100) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name too long');
+    if (/["<>@]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name is not valid');
+
+    return null;
+}
+
 async function checkOutboundPort25() {
     const relay = {
         value: 'OK',

src/mounts.js

@@ -54,6 +54,7 @@ function validateMountOptions(type, options) {
         if (typeof options.remoteDir !== 'string') return new BoxError(BoxError.BAD_FIELD, 'remoteDir is not a string');
         return null;
     case 'ext4':
+    case 'xfs':
         if (typeof options.diskPath !== 'string') return new BoxError(BoxError.BAD_FIELD, 'diskPath is not a string');
         return null;
     default:
@@ -62,7 +63,7 @@ function validateMountOptions(type, options) {
 }
 
 function isManagedProvider(provider) {
-    return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4';
+    return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4' || provider === 'xfs';
 }
 
 function mountObjectFromBackupConfig(backupConfig) {
@@ -108,6 +109,11 @@ function renderMountFile(mount) {
         what = mountOptions.diskPath; // like /dev/disk/by-uuid/uuid or /dev/disk/by-id/scsi-id
         options = 'discard,defaults,noatime';
         break;
+    case 'xfs':
+        type = 'xfs';
+        what = mountOptions.diskPath; // like /dev/disk/by-uuid/uuid or /dev/disk/by-id/scsi-id
+        options = 'discard,defaults,noatime';
+        break;
     case 'sshfs': {
         const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${mountOptions.host}`);
         if (!safe.fs.writeFileSync(keyFilePath, `${mount.mountOptions.privateKey}\n`, { mode: 0o600 })) throw new BoxError(BoxError.FS_ERROR, `Could not write private key: ${safe.error.message}`);

src/nginxconfig.ejs

@@ -216,7 +216,7 @@ server {
     }
 
     # the read timeout is between successive reads and not the whole connection
-    location ~ ^/api/v1/apps/.*/exec$ {
+    location ~ ^/api/v1/apps/.*/exec/.*/start$ {
         proxy_pass   http://127.0.0.1:3000;
         proxy_read_timeout 30m;
     }
@@ -236,6 +236,11 @@ server {
         client_max_body_size 0;
     }
 
+    location ~ ^/api/v1/profile/backgroundImage {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 0;
+    }
+
     # graphite paths (uncomment block below and visit /graphite-web/dashboard)
     # remember to comment out the CSP policy as well to access the graphite dashboard
     # location ~ ^/graphite-web/ {

src/proxyauth.js

@@ -204,11 +204,6 @@ async function logoutPage(req, res, next) {
     res.redirect(302, app.manifest.addons.proxyAuth.path ? '/' : '/login');
 }
 
-function logout(req, res, next) {
-    res.clearCookie('authToken');
-    next(new HttpSuccess(200, {}));
-}
-
 // provides webhooks for the auth wall
 function initializeAuthwallExpressSync() {
     const app = express();
@@ -251,7 +246,7 @@ function initializeAuthwallExpressSync() {
     router.get ('/auth',      jwtVerify, basicAuthVerify, auth); // called by nginx before accessing protected page
     router.post('/login',     json, passwordAuth, authorize);
     router.get ('/logout',    logoutPage);
-    router.post('/logout',    json, logout);
+    router.post('/logout',    json, logoutPage);
 
     return httpServer;
 }

src/routes/apps.js

@@ -35,14 +35,18 @@ exports = module.exports = {
     setMailbox,
     setInbox,
     setLocation,
-    setDataDir,
+    setStorage,
     setMounts,
 
     stop,
     start,
     restart,
-    exec,
-    execWebSocket,
+
+    createExec,
+    startExec,
+    startExecWebSocket,
+    getExec,
+
     checkForUpdates,
 
     clone,
@@ -369,6 +373,7 @@ async function setMailbox(req, res, next) {
     if (req.body.enable) {
         if (req.body.mailboxName !== null && typeof req.body.mailboxName !== 'string') return next(new HttpError(400, 'mailboxName must be a string'));
         if (typeof req.body.mailboxDomain !== 'string') return next(new HttpError(400, 'mailboxDomain must be a string'));
+        if ('mailboxDisplayName' in req.body && typeof req.body.mailboxDisplayName !== 'string') return next(new HttpError(400, 'mailboxDisplayName must be a string'));
     }
 
     const [error, result] = await safe(apps.setMailbox(req.app, req.body, AuditSource.fromRequest(req)));
@@ -427,13 +432,18 @@ async function setLocation(req, res, next) {
     next(new HttpSuccess(202, { taskId: result.taskId }));
 }
 
-async function setDataDir(req, res, next) {
+async function setStorage(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.app, 'object');
 
-    if (req.body.dataDir !== null && typeof req.body.dataDir !== 'string') return next(new HttpError(400, 'dataDir must be a string'));
+    const { storageVolumeId, storageVolumePrefix } = req.body;
 
-    const [error, result] = await safe(apps.setDataDir(req.app, req.body.dataDir, AuditSource.fromRequest(req)));
+    if (storageVolumeId !== null) {
+        if (typeof storageVolumeId !== 'string') return next(new HttpError(400, 'storageVolumeId must be a string'));
+        if (typeof storageVolumePrefix !== 'string') return next(new HttpError(400, 'storageVolumePrefix must be a string'));
+    }
+
+    const [error, result] = await safe(apps.setStorage(req.app, storageVolumeId, storageVolumePrefix, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -697,14 +707,29 @@ function demuxStream(stream, stdin) {
     });
 }
 
-async function exec(req, res, next) {
+async function createExec(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
+    assert.strictEqual(typeof req.body, 'object');
 
-    let cmd = null;
-    if (req.query.cmd) {
-        cmd = safe.JSON.parse(req.query.cmd);
-        if (!Array.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
+    if ('cmd' in req.body) {
+        if (!Array.isArray(req.body.cmd) || req.body.cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
     }
+    const cmd = req.body.cmd || null;
+
+    if ('tty' in req.body && typeof req.body.tty !== 'boolean') return next(new HttpError(400, 'tty must be boolean'));
+    const tty = !!req.body.tty;
+
+    if (safe.query(req.app, 'manifest.addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is requied to exec app with docker addon'));
+
+    const [error, id] = await safe(apps.createExec(req.app, { cmd, tty }));
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(200, { id }));
+}
+
+async function startExec(req, res, next) {
+    assert.strictEqual(typeof req.app, 'object');
+    assert.strictEqual(typeof req.params.execId, 'string');
 
     const columns = req.query.columns ? parseInt(req.query.columns, 10) : null;
     if (isNaN(columns)) return next(new HttpError(400, 'columns must be a number'));
@@ -719,7 +744,7 @@ async function exec(req, res, next) {
     // in a badly configured reverse proxy, we might be here without an upgrade
     if (req.headers['upgrade'] !== 'tcp') return next(new HttpError(404, 'exec requires TCP upgrade'));
 
-    const [error, duplexStream] = await safe(apps.exec(req.app, { cmd: cmd, rows: rows, columns: columns, tty: tty }));
+    const [error, duplexStream] = await safe(apps.startExec(req.app, req.params.execId, { rows, columns, tty }));
     if (error) return next(BoxError.toHttpError(error));
 
     req.clearTimeout();
@@ -737,14 +762,9 @@ async function exec(req, res, next) {
     }
 }
 
-async function execWebSocket(req, res, next) {
+async function startExecWebSocket(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
-
-    let cmd = null;
-    if (req.query.cmd) {
-        cmd = safe.JSON.parse(req.query.cmd);
-        if (!Array.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
-    }
+    assert.strictEqual(typeof req.params.execId, 'string');
 
     const columns = req.query.columns ? parseInt(req.query.columns, 10) : null;
     if (isNaN(columns)) return next(new HttpError(400, 'columns must be a number'));
@@ -757,7 +777,7 @@ async function execWebSocket(req, res, next) {
     // in a badly configured reverse proxy, we might be here without an upgrade
     if (req.headers['upgrade'] !== 'websocket') return next(new HttpError(404, 'exec requires websocket'));
 
-    const [error, duplexStream] = await safe(apps.exec(req.app, { cmd: cmd, rows: rows, columns: columns, tty: tty }));
+    const [error, duplexStream] = await safe(apps.startExec(req.app, req.params.execId, { rows, columns, tty }));
     if (error) return next(BoxError.toHttpError(error));
 
     req.clearTimeout();
@@ -785,6 +805,15 @@ async function execWebSocket(req, res, next) {
     });
 }
 
+async function getExec(req, res, next) {
+    assert.strictEqual(typeof req.app, 'object');
+    assert.strictEqual(typeof req.params.execId, 'string');
+
+    const [error, result] = await safe(apps.getExec(req.app, req.params.execId));
+    if (error) return next(BoxError.toHttpError(error));
+    next(new HttpSuccess(200, result)); // { exitCode, running }
+}
+
 async function listBackups(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
 

src/routes/profile.js

@@ -6,6 +6,8 @@ exports = module.exports = {
     update,
     getAvatar,
     setAvatar,
+    getBackgroundImage,
+    setBackgroundImage,
     setPassword,
     setTwoFactorAuthenticationSecret,
     enableTwoFactorAuthentication,
@@ -37,10 +39,14 @@ async function authorize(req, res, next) {
 async function get(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    const [error, avatarUrl] = await safe(users.getAvatarUrl(req.user));
+    let [error, avatarUrl] = await safe(users.getAvatarUrl(req.user));
     if (error) return next(BoxError.toHttpError(error));
     if (!avatarUrl) return next(new HttpError(404, 'User not found'));
 
+    let backgroundImage;
+    [error, backgroundImage] = await safe(users.getBackgroundImage(req.user.id));
+    if (error) return next(BoxError.toHttpError(error));
+
     next(new HttpSuccess(200, {
         id: req.user.id,
         username: req.user.username,
@@ -50,6 +56,7 @@ async function get(req, res, next) {
         twoFactorAuthenticationEnabled: req.user.twoFactorAuthenticationEnabled,
         role: req.user.role,
         source: req.user.source,
+        hasBackgroundImage: !!backgroundImage,
         avatarUrl
     }));
 }
@@ -107,6 +114,31 @@ async function getAvatar(req, res, next) {
     res.send(avatar);
 }
 
+async function setBackgroundImage(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    let backgroundImage = null;
+
+    if (req.files && req.files.backgroundImage) {
+        backgroundImage = safe.fs.readFileSync(req.files.backgroundImage.path);
+        if (!backgroundImage) return next(BoxError.toHttpError(new BoxError(BoxError.FS_ERROR, safe.error.message)));
+    }
+
+    const [error] = await safe(users.setBackgroundImage(req.user.id, backgroundImage));
+    if (error) return next(BoxError.toHttpError(error));
+
+    next(new HttpSuccess(202, {}));
+}
+
+async function getBackgroundImage(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    const [error, backgroundImage] = await safe(users.getBackgroundImage(req.user.id));
+    if (error) return next(BoxError.toHttpError(error));
+
+    res.send(backgroundImage);
+}
+
 async function setPassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.user, 'object');

src/routes/test/appstore-test.js

@@ -85,7 +85,7 @@ describe('Appstore Cloudron Registration API - existing user', function () {
 
     it('can setup subscription', async function () {
         const scope1 = nock(settings.apiServerOrigin())
-            .post('/api/v1/register_user', (body) => body.email && body.password)
+            .post('/api/v1/register_user', (body) => body.email && body.password && body.utmSource)
             .reply(201, {});
 
         const scope2 = nock(settings.apiServerOrigin())
@@ -142,7 +142,7 @@ describe('Appstore Cloudron Registration API - new user signup', function () {
 
     it('can setup subscription', async function () {
         const scope1 = nock(settings.apiServerOrigin())
-            .post('/api/v1/register_user', (body) => body.email && body.password)
+            .post('/api/v1/register_user', (body) => body.email && body.password && body.utmSource)
             .reply(201, {});
 
         const scope2 = nock(settings.apiServerOrigin())

src/routes/test/users-test.js

@@ -640,15 +640,5 @@ describe('Users API', function () {
             expect(response.statusCode).to.equal(409);
         });
     });
-
-    describe('transfer ownership', function () {
-        it('succeeds', async function () {
-            const response = await superagent.post(`${serverUrl}/api/v1/users/${user.id}/make_owner`)
-                .query({ access_token: owner.token })
-                .send({});
-
-            expect(response.statusCode).to.equal(204);
-        });
-    });
 });
 

src/routes/users.js

@@ -10,7 +10,6 @@ exports = module.exports = {
     verifyPassword,
     setGroups,
     setGhost,
-    makeOwner,
     makeLocal,
 
     getPasswordResetLink,
@@ -202,20 +201,6 @@ async function setPassword(req, res, next) {
     next(new HttpSuccess(204));
 }
 
-// This route transfers ownership from token user to user specified in path param
-async function makeOwner(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
-
-    // first make new one owner, then demote current one
-    let [error] = await safe(users.update(req.resource, { role: users.ROLE_OWNER }, AuditSource.fromRequest(req)));
-    if (error) return next(BoxError.toHttpError(error));
-
-    [error] = await safe(users.update(req.user, { role: users.ROLE_USER }, AuditSource.fromRequest(req)));
-    if (error) return next(BoxError.toHttpError(error));
-
-    next(new HttpSuccess(204));
-}
-
 async function makeLocal(req, res, next) {
     assert.strictEqual(typeof req.resource, 'object');
 

src/scheduler.js

@@ -16,9 +16,8 @@ const apps = require('./apps.js'),
     safe = require('safetydance'),
     _ = require('underscore');
 
-// appId -> { containerId, schedulerConfig (manifest), cronjobs }
-let gState = { };
-let gSuspendedAppIds = new Set(); // suspended because some apptask is running
+const gState = {}; // appId -> { containerId, schedulerConfig (manifest+crontab), cronjobs }
+const gSuspendedAppIds = new Set(); // suspended because some apptask is running
 
 // TODO: this should probably also stop existing jobs to completely prevent race but the code is not re-entrant
 function suspendJobs(appId) {
@@ -59,26 +58,30 @@ async function createJobs(app, schedulerConfig) {
     assert(schedulerConfig && typeof schedulerConfig === 'object');
 
     const appId = app.id;
-    const jobs = { };
+    const jobs = {};
 
     for (const taskName of Object.keys(schedulerConfig)) {
-        const task = schedulerConfig[taskName];
-        const randomSecond = Math.floor(60*Math.random()); // don't start all crons to decrease memory pressure
-        const cronTime = (constants.TEST ? '*/5 ' : `${randomSecond} `) + task.schedule; // time ticks faster in tests
-
+        const { schedule, command } = schedulerConfig[taskName];
         const containerName = `${app.id}-${taskName}`;
-        const cmd = schedulerConfig[taskName].command;
 
-        // stopJobs only deletes jobs since previous run. This means that when box code restarts, none of the containers
+        // stopJobs only deletes jobs since previous sync. This means that when box code restarts, none of the containers
         // are removed. The deleteContainer here ensures we re-create the cron containers with the latest config
         await safe(docker.deleteContainer(containerName)); // ignore error
-        const [error] = await safe(docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], { } /* options */));
+        const [error] = await safe(docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', command ], {} /* options */), { debug });
         if (error && error.reason !== BoxError.ALREADY_EXISTS) continue;
 
         debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`);
 
+        let cronTime;
+        if (schedule === '@service') {
+            cronTime = new Date(Date.now() + 2*1000); // 2 seconds from now
+        } else {
+            // random is so that all crons start at once to decrease memory pressure
+            cronTime = (constants.TEST ? '*/5 ' : `${Math.floor(60*Math.random())} `) + schedule; // time ticks faster in tests
+        }
+
         const cronJob = new CronJob({
-            cronTime: cronTime, // at this point, the pattern has been validated
+            cronTime,
             onTick: async () => {
                 const [error] = await safe(runTask(appId, taskName)); //  put the app id in closure, so we don't use the outdated app object by mistake
                 if (error) debug(`could not run task ${taskName} : ${error.message}`);
@@ -120,10 +123,9 @@ async function sync() {
         debug(`sync: removing jobs of ${appId}`);
         const [error] = await safe(stopJobs(appId, gState[appId]));
         if (error) debug(`sync: error stopping jobs of removed app ${appId}: ${error.message}`);
+        delete gState[appId];
     }
 
-    gState = _.omit(gState, removedAppIds);
-
     for (const app of allApps) {
         const appState = gState[app.id] || null;
         const schedulerConfig = apps.getSchedulerConfig(app);

src/scripts/backupupload.js

@@ -7,23 +7,13 @@ if (process.argv[2] === '--check') {
     process.exit(0);
 }
 
-const assert = require('assert'),
-    async = require('async'),
-    backuptask = require('../backuptask.js'),
+const backuptask = require('../backuptask.js'),
     database = require('../database.js'),
     debug = require('debug')('box:backupupload'),
+    safe = require('safetydance'),
     settings = require('../settings.js'),
     v8 = require('v8');
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        database.initialize,
-        settings.initCache
-    ], callback);
-}
-
 // Main process starts here
 const remotePath = process.argv[2];
 const format = process.argv[3];
@@ -70,20 +60,20 @@ function dumpMemoryInfo() {
     debug(`v8 heap: used ${h(hs.used_heap_size)} total: ${h(hs.total_heap_size)} max: ${h(hs.heap_size_limit)}`);
 }
 
-initialize(function (error) {
-    if (error) throw error;
+(async function main() {
+    await database.initialize();
+    await settings.initCache();
 
     dumpMemoryInfo();
     const timerId = setInterval(dumpMemoryInfo, 30000);
 
-    backuptask.upload(remotePath, format, dataLayoutString, throttledProgressCallback(5000), function resultHandler(error) {
-        debug('upload completed. error: ', error);
+    const [uploadError] = await safe(backuptask.upload(remotePath, format, dataLayoutString, throttledProgressCallback(5000)));
+    debug('upload completed. error: ', uploadError);
 
-        process.send({ result: error ? error.message : '' });
-        clearInterval(timerId);
+    process.send({ result: uploadError ? uploadError.message : '' });
+    clearInterval(timerId);
 
-        // https://nodejs.org/api/process.html are exit codes used by node. apps.js uses the value below
-        // to check apptask crashes
-        process.exit(error ? 50 : 0);
-    });
-});
+    // https://nodejs.org/api/process.html are exit codes used by node. apps.js uses the value below
+    // to check apptask crashes
+    process.exit(uploadError ? 50 : 0);
+})();

src/scripts/checkvolume.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
+if [[ $# -eq 0 ]]; then
+    echo "No arguments supplied"
+    exit 1
+fi
+
+if [[ "$1" == "--check" ]]; then
+    echo "OK"
+    exit 0
+fi
+
+target_dir="$1"
+source_dir="$2"
+
+source_stat=$(stat --format='%d,%i' "${source_dir}")
+target_stat=$(stat --format='%d,%i' "${target_dir}")
+
+# test sameness across bind mounts. if it's same, we can skip the emptiness check
+if [[ "${source_stat}" == "${target_stat}" ]]; then
+    echo "Source dir and target dir are the same"
+    exit 0
+fi
+
+readonly test_file="${target_dir}/.chown-test"
+
+mkdir -p "${target_dir}"
+rm -f "${test_file}" # clean up any from previous run
+
+if [[ -n $(ls -A "${target_dir}") ]]; then
+    echo "volume dir is not empty"
+    exit 2
+fi
+
+touch "${test_file}"
+if ! chown yellowtent:yellowtent "${test_file}"; then
+    echo "chown does not work"
+    exit 3
+fi
+rm -f "${test_file}"
+rm -r "${target_dir}" # will get recreated by the local storage addon
+

src/scripts/mkdirvolume.sh

@@ -20,4 +20,3 @@ fi
 volume_dir="$1"
 
 mkdir -p "${volume_dir}"
-

src/scripts/mvvolume.sh

@@ -26,8 +26,17 @@ if [[ "${BOX_ENV}" == "test" ]]; then
     [[ "${target_dir}" != *"/.cloudron_test/"* ]] && exit 1
 fi
 
+source_stat=$(stat --format='%d,%i' "${source_dir}")
+target_stat=$(stat --format='%d,%i' "${target_dir}")
+
+# test sameness across bind mounts
+if [[ "${source_stat}" == "${target_stat}" ]]; then
+    echo "Source dir and target dir are the same"
+    exit 0
+fi
+
 # copy and remove - this way if the copy fails, the original is intact
-# the find logic is so that move to a subdir works (and we also move hidden files)
+# the find logic is so that move to a one level subdir works (and we also move hidden files)
 find "${source_dir}" -maxdepth 1 -mindepth 1 -not -wholename "${target_dir}" -exec cp -ar '{}' "${target_dir}" \;
 find "${source_dir}" -maxdepth 1 -mindepth 1 -not -wholename "${target_dir}" -exec rm -rf '{}' \;
 # this will fail if target is a subdir or if source is a mountpoint

src/scripts/remotesupport.sh

@@ -17,7 +17,7 @@ if [[ "$1" == "--check" ]]; then
     exit 0
 fi
 
-CLOUDRON_SUPPORT_PUBLIC_KEY='ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQVilclYAIu+ioDp/sgzzFz6YU0hPcRYY7ze/LiF/lC7uQqK062O54BFXTvQ3ehtFZCx3bNckjlT2e6gB8Qq07OM66De4/S/g+HJW4TReY2ppSPMVNag0TNGxDzVH8pPHOysAm33LqT2b6L/wEXwC6zWFXhOhHjcMqXvi8Ejaj20H1HVVcf/j8qs5Thkp9nAaFTgQTPu8pgwD8wDeYX1hc9d0PYGesTADvo6HF4hLEoEnefLw7PaStEbzk2fD3j7/g5r5HcgQQXBe74xYZ/1gWOX2pFNuRYOBSEIrNfJEjFJsqk3NR1+ZoMGK7j+AZBR4k0xbrmncQLcQzl6MMDzkp support@cloudron.io'
+CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGWS+930b8QdzbchGljt3KSljH9wRhYvht8srrtQHdzg support@cloudron.io"
 
 cmd="$1"
 keys_file="$2"

src/server.js

@@ -151,6 +151,8 @@ function initializeExpressSync() {
     router.post('/api/v1/profile',                                 json, token, routes.profile.authorize, routes.profile.update);
     router.get ('/api/v1/profile/avatar/:identifier',                           routes.profile.getAvatar); // this is not scoped so it can used directly in img tag
     router.post('/api/v1/profile/avatar',                          json, token, (req, res, next) => { return typeof req.body.avatar === 'string' ? next() : multipart(req, res, next); }, routes.profile.setAvatar); // avatar is not exposed in LDAP. so it's personal and not locked
+    router.get ('/api/v1/profile/backgroundImage',                       token, routes.profile.getBackgroundImage);
+    router.post('/api/v1/profile/backgroundImage',                       token, multipart, routes.profile.setBackgroundImage); // backgroundImage is not exposed in LDAP. so it's personal and not locked
     router.post('/api/v1/profile/password',                        json, token, routes.users.verifyPassword, routes.profile.setPassword);
     router.post('/api/v1/profile/twofactorauthentication_secret',  json, token, routes.profile.setTwoFactorAuthenticationSecret);
     router.post('/api/v1/profile/twofactorauthentication_enable',  json, token, routes.profile.enableTwoFactorAuthentication);
@@ -177,7 +179,6 @@ function initializeExpressSync() {
     router.post('/api/v1/users/:userId/password',      json, token, authorizeUserManager, routes.users.load, routes.users.setPassword);
     router.post('/api/v1/users/:userId/ghost',         json, token, authorizeAdmin,       routes.users.load, routes.users.setGhost);
     router.put ('/api/v1/users/:userId/groups',        json, token, authorizeUserManager, routes.users.load, routes.users.setGroups);
-    router.post('/api/v1/users/:userId/make_owner',    json, token, authorizeOwner,       routes.users.load, routes.users.makeOwner);
     router.post('/api/v1/users/:userId/twofactorauthentication_disable', json, token, authorizeUserManager, routes.users.load, routes.users.disableTwoFactorAuthentication);
     router.get ('/api/v1/users/:userId/password_reset_link',   json, token, authorizeUserManager, routes.users.load, routes.users.getPasswordResetLink);
     router.post('/api/v1/users/:userId/send_password_reset_email',   json, token, authorizeUserManager, routes.users.load, routes.users.sendPasswordResetEmail);
@@ -222,7 +223,7 @@ function initializeExpressSync() {
     router.post('/api/v1/apps/:id/configure/mailbox',            json, token, routes.apps.load, authorizeAdmin, routes.apps.setMailbox);
     router.post('/api/v1/apps/:id/configure/inbox',              json, token, routes.apps.load, authorizeAdmin, routes.apps.setInbox);
     router.post('/api/v1/apps/:id/configure/env',                json, token, routes.apps.load, authorizeOperator, routes.apps.setEnvironment);
-    router.post('/api/v1/apps/:id/configure/data_dir',           json, token, routes.apps.load, authorizeAdmin, routes.apps.setDataDir);
+    router.post('/api/v1/apps/:id/configure/storage',            json, token, routes.apps.load, authorizeAdmin, routes.apps.setStorage);
     router.post('/api/v1/apps/:id/configure/location',           json, token, routes.apps.load, authorizeAdmin, routes.apps.setLocation);
     router.post('/api/v1/apps/:id/configure/mounts',             json, token, routes.apps.load, authorizeAdmin, routes.apps.setMounts);
     router.post('/api/v1/apps/:id/configure/crontab',            json, token, routes.apps.load, authorizeOperator, routes.apps.setCrontab);
@@ -248,10 +249,12 @@ function initializeExpressSync() {
     router.get ('/api/v1/apps/:id/download',                           token, routes.apps.load, authorizeOperator, routes.apps.downloadFile);
     router.post('/api/v1/apps/:id/upload',                       json, token, multipart, routes.apps.load, authorizeOperator, routes.apps.uploadFile);
     router.use ('/api/v1/apps/:id/files/*',                            token, routes.apps.load, authorizeOperator, routes.filemanager.proxy('app'));
-    router.get ('/api/v1/apps/:id/exec',                               token, routes.apps.load, authorizeOperator, routes.apps.exec);
+    router.post('/api/v1/apps/:id/exec',                         json, token, routes.apps.load, authorizeOperator, routes.apps.createExec);
+    router.get ('/api/v1/apps/:id/exec/:execId/start',                 token, routes.apps.load, authorizeOperator, routes.apps.startExec);
+    router.get ('/api/v1/apps/:id/exec/:execId',                       token, routes.apps.load, authorizeOperator, routes.apps.getExec);
 
     // websocket cannot do bearer authentication
-    router.get ('/api/v1/apps/:id/execws',    token, routes.apps.load, routes.accesscontrol.authorizeOperator, routes.apps.execWebSocket);
+    router.get ('/api/v1/apps/:id/exec/:execId/startws',    token, routes.apps.load, routes.accesscontrol.authorizeOperator, routes.apps.startExecWebSocket);
 
     // branding routes
     router.get ('/api/v1/branding/:setting',       token, authorizeOwner, routes.branding.get);

src/services.js

@@ -63,6 +63,8 @@ const addonConfigs = require('./addonconfigs.js'),
 const NOOP = async function (/*app, options*/) {};
 const RMADDONDIR_CMD = path.join(__dirname, 'scripts/rmaddondir.sh');
 const RESTART_SERVICE_CMD = path.join(__dirname, 'scripts/restartservice.sh');
+const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh');
+const MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');
 
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
@@ -856,11 +858,10 @@ async function setupLocalStorage(app, options) {
 
     debug('setupLocalStorage');
 
-    const volumeDataDir = apps.getDataDir(app, app.dataDir);
+    const volumeDataDir = await apps.getStorageDir(app);
 
-    // reomve any existing volume in case it's bound with an old dataDir
-    await docker.removeVolume(`${app.id}-localstorage`);
-    await docker.createVolume(`${app.id}-localstorage`, volumeDataDir, { fqdn: app.fqdn, appId: app.id });
+    const [error] = await safe(shell.promises.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}));
+    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating app storage data dir: ${error.message}`);
 }
 
 async function clearLocalStorage(app, options) {
@@ -869,7 +870,9 @@ async function clearLocalStorage(app, options) {
 
     debug('clearLocalStorage');
 
-    await docker.clearVolume(`${app.id}-localstorage`, { removeDirectory: false });
+    const volumeDataDir = await apps.getStorageDir(app);
+    const [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, 'clear', volumeDataDir ], {});
+    if (error) throw new BoxError(BoxError.FS_ERROR, error);
 }
 
 async function teardownLocalStorage(app, options) {
@@ -878,8 +881,9 @@ async function teardownLocalStorage(app, options) {
 
     debug('teardownLocalStorage');
 
-    await docker.clearVolume(`${app.id}-localstorage`, { removeDirectory: true });
-    await docker.removeVolume(`${app.id}-localstorage`);
+    const volumeDataDir = await apps.getStorageDir(app);
+    const [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, 'rmdir', volumeDataDir ], {});
+    if (error) throw new BoxError(BoxError.FS_ERROR, error);
 }
 
 async function setupTurn(app, options) {
@@ -1048,6 +1052,9 @@ async function setupSendMail(app, options) {
         { name: 'CLOUDRON_MAIL_FROM', value: app.mailboxName + '@' + app.mailboxDomain },
         { name: 'CLOUDRON_MAIL_DOMAIN', value: app.mailboxDomain }
     ];
+
+    if (app.manifest.addons.sendmail.supportsDisplayName) env.push({ name: 'CLOUDRON_MAIL_FROM_DISPLAY_NAME', value: app.mailboxDisplayName });
+
     debug('Setting sendmail addon config to %j', env);
     await addonConfigs.set(app.id, 'sendmail', env);
 }
@@ -1240,10 +1247,15 @@ async function pipeRequestToFile(url, filename) {
     return new Promise((resolve, reject) => {
         const writeStream = fs.createWriteStream(filename);
         const request = http.request(url, { method: 'POST' }); // ClientRequest
-        request.setTimeout(600000, () => request.destroy(new Error('Request timedout'))); // connect OR post-connect idle timeout
+        request.setTimeout(600000, () => {
+            debug('pipeRequestToFile: timeout - connect or post-connect idle timeout');
+            request.destroy(); // connect OR post-connect idle timeout
+            reject(new Error('Request timedout'));
+        });
 
         request.on('error', (error) => reject(new BoxError(BoxError.NETWORK_ERROR, `Could not pipe ${url} to ${filename}: ${error.message}`))); // network error, dns error
         request.on('response', (response) => {
+            debug(`pipeRequestToFile: connected with status code ${response.statusCode}`);
             if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code or HTTP error when piping ${url} to ${filename}: status ${response.statusCode}`));
 
             pipeline(response, writeStream, (error) => {
@@ -1264,16 +1276,23 @@ async function pipeFileToRequest(filename, url) {
     return new Promise((resolve, reject) => {
         const readStream = fs.createReadStream(filename);
         const request = http.request(url, { method: 'POST' }); // ClientRequest
-        request.setTimeout(60000, () => request.destroy(new Error('Request timedout'))); // connect OR post-connect idle timeout
+        request.setTimeout(600000, () => {
+            debug('pipeFileToRequest: timeout - connect or post-connect idle timeout');
+            request.destroy();
+            reject(new Error('Request timedout'));
+        });
         request.on('response', (response) => {
+            debug(`pipeFileToRequest: request completed with status code ${response.statusCode}`);
             response.resume(); // drain the response
             if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code or HTTP error when piping ${filename} to ${url}: status ${response.statusCode} complete ${response.complete}`));
 
             resolve();
         });
 
+        debug(`pipeFileToRequest: piping ${filename} to ${url}`);
         pipeline(readStream, request, function (error) {
             if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error piping file ${filename} to request ${url}`));
+            debug(`pipeFileToRequest: piped ${filename} to ${url}`); // now we have to wait for 'response' above
         });
     });
 }

src/settings.js

@@ -78,6 +78,7 @@ exports = module.exports = {
     // these values come from the cache
     apiServerOrigin,
     webServerOrigin,
+    consoleServerOrigin,
     dashboardDomain,
     setDashboardLocation,
     setMailLocation,
@@ -121,6 +122,7 @@ exports = module.exports = {
 
     API_SERVER_ORIGIN_KEY: 'api_server_origin',
     WEB_SERVER_ORIGIN_KEY: 'web_server_origin',
+    CONSOLE_SERVER_ORIGIN_KEY: 'console_server_origin',
     DASHBOARD_DOMAIN_KEY: 'admin_domain',
     DASHBOARD_FQDN_KEY: 'admin_fqdn',
     MAIL_DOMAIN_KEY: 'mail_domain',
@@ -217,6 +219,7 @@ const gDefaults = (function () {
 
     result[exports.API_SERVER_ORIGIN_KEY] = 'https://api.cloudron.io';
     result[exports.WEB_SERVER_ORIGIN_KEY] = 'https://cloudron.io';
+    result[exports.CONSOLE_SERVER_ORIGIN_KEY] = 'https://console.cloudron.io';
     result[exports.DEMO_KEY] = false;
 
     result[exports.APPSTORE_LISTING_CONFIG_KEY] = {
@@ -754,6 +757,7 @@ async function initCache() {
     gCache = {
         apiServerOrigin: allSettings[exports.API_SERVER_ORIGIN_KEY],
         webServerOrigin: allSettings[exports.WEB_SERVER_ORIGIN_KEY],
+        consoleServerOrigin: allSettings[exports.CONSOLE_SERVER_ORIGIN_KEY],
         dashboardDomain: allSettings[exports.DASHBOARD_DOMAIN_KEY],
         dashboardFqdn: allSettings[exports.DASHBOARD_FQDN_KEY],
         mailDomain: allSettings[exports.MAIL_DOMAIN_KEY],
@@ -811,6 +815,7 @@ async function setFooter(footer) {
 function provider() { return gCache.provider; }
 function apiServerOrigin() { return gCache.apiServerOrigin; }
 function webServerOrigin() { return gCache.webServerOrigin; }
+function consoleServerOrigin() { return gCache.consoleServerOrigin; }
 function dashboardDomain() { return gCache.dashboardDomain; }
 function dashboardFqdn() { return gCache.dashboardFqdn; }
 function isDemo() { return gCache.isDemo; }

src/sftp.js

@@ -62,9 +62,9 @@ async function start(existingInfra) {
     // custom app data directories
     const allApps = await apps.list();
     for (const app of allApps) {
-        if (!app.manifest.addons['localstorage'] || !app.dataDir) continue;
+        if (!app.manifest.addons['localstorage'] || !app.storageVolumeId) continue;
 
-        const hostDir = apps.getDataDir(app, app.dataDir), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp
+        const hostDir = await apps.getStorageDir(app), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp
         if (!safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user
             // do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail
             debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`);

src/shell.js

@@ -102,5 +102,6 @@ function sudo(tag, args, options, callback) {
 
     const cp = spawn(tag, SUDO, sudoArgs.concat(args), options, callback);
     cp.stdin.end();
+    if (options.onMessage) cp.on('message', options.onMessage);
     return cp;
 }

src/storage.js

@@ -2,13 +2,8 @@
 
 exports = module.exports = {
     api,
-
-    getBackupFilePath,
 };
 
-const assert = require('assert'),
-    path = require('path');
-
 // choose which storage backend we use for test purpose we use s3
 function api(provider) {
     switch (provider) {
@@ -36,19 +31,3 @@ function api(provider) {
     default: return null;
     }
 }
-
-// This is not part of the storage api, since we don't want to pull the "format" logistics into that
-function getBackupFilePath(backupConfig, remotePath, format) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-    assert.strictEqual(typeof format, 'string');
-
-    const rootPath = api(backupConfig.provider).getRootPath(backupConfig);
-
-    if (format === 'tgz') {
-        const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz';
-        return path.join(rootPath, remotePath + fileType);
-    } else {
-        return path.join(rootPath, remotePath);
-    }
-}

src/storage/filesystem.js

@@ -26,6 +26,7 @@ const PROVIDER_FILESYSTEM = 'filesystem';
 const PROVIDER_MOUNTPOINT = 'mountpoint';
 const PROVIDER_SSHFS = 'sshfs';
 const PROVIDER_CIFS = 'cifs';
+const PROVIDER_XFS = 'xfs';
 const PROVIDER_NFS = 'nfs';
 const PROVIDER_EXT4 = 'ext4';
 
@@ -35,7 +36,6 @@ const assert = require('assert'),
     DataLayout = require('../datalayout.js'),
     debug = require('debug')('box:storage/filesystem'),
     df = require('@sindresorhus/df'),
-    EventEmitter = require('events'),
     fs = require('fs'),
     mounts = require('../mounts.js'),
     path = require('path'),
@@ -53,6 +53,7 @@ function getRootPath(apiConfig) {
     case PROVIDER_NFS:
     case PROVIDER_CIFS:
     case PROVIDER_EXT4:
+    case PROVIDER_XFS:
         return path.join(paths.MANAGED_BACKUP_MOUNT_DIR, apiConfig.prefix);
     case PROVIDER_MOUNTPOINT:
         return path.join(apiConfig.mountPoint, apiConfig.prefix);
@@ -90,7 +91,7 @@ async function checkPreconditions(apiConfig, dataLayout) {
     if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`);
 
     // Check filesystem is mounted so we don't write into the actual folder on disk
-    if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS || apiConfig.provider === PROVIDER_EXT4) {
+    if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS || apiConfig.provider === PROVIDER_EXT4 || apiConfig.provider === PROVIDER_XFS) {
         if (result.mountpoint !== paths.MANAGED_BACKUP_MOUNT_DIR) throw new BoxError(BoxError.FS_ERROR, 'Backup target is not mounted');
     } else if (apiConfig.provider === PROVIDER_MOUNTPOINT) {
         if (result.mountpoint === '/') throw new BoxError(BoxError.FS_ERROR, `${apiConfig.backupFolder} is not mounted`);
@@ -104,6 +105,7 @@ function hasChownSupportSync(apiConfig) {
     switch (apiConfig.provider) {
     case PROVIDER_NFS:
     case PROVIDER_EXT4:
+    case PROVIDER_XFS:
     case PROVIDER_FILESYSTEM:
         return true;
     case PROVIDER_SSHFS:
@@ -211,29 +213,22 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
     });
 }
 
-function copy(apiConfig, oldFilePath, newFilePath) {
+async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof oldFilePath, 'string');
     assert.strictEqual(typeof newFilePath, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
 
-    var events = new EventEmitter();
+    const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(newFilePath), { recursive: true }));
+    if (mkdirError) throw new BoxError(BoxError.EXTERNAL_ERROR, mkdirError.message);
 
-    fs.mkdir(path.dirname(newFilePath), { recursive: true }, function (error) {
-        if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    progressCallback({ message: `Copying ${oldFilePath} to ${newFilePath}` });
 
-        events.emit('progress', `Copying ${oldFilePath} to ${newFilePath}`);
+    let cpOptions = ((apiConfig.provider !== PROVIDER_MOUNTPOINT && apiConfig.provider !== PROVIDER_CIFS) || apiConfig.preserveAttributes) ? '-a' : '-dR';
+    cpOptions += apiConfig.noHardlinks ? '' : 'l';  // this will hardlink backups saving space
 
-        let cpOptions = ((apiConfig.provider !== PROVIDER_MOUNTPOINT && apiConfig.provider !== PROVIDER_CIFS) || apiConfig.preserveAttributes) ? '-a' : '-dR';
-        cpOptions += apiConfig.noHardlinks ? '' : 'l';  // this will hardlink backups saving space
-
-        shell.spawn('copy', '/bin/cp', [ cpOptions, oldFilePath, newFilePath ], { }, function (error) {
-            if (error) return events.emit('done', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-
-            events.emit('done', null);
-        });
-    });
-
-    return events;
+    const [copyError] = await safe(shell.promises.spawn('copy', '/bin/cp', [ cpOptions, oldFilePath, newFilePath ], { }));
+    if (copyError) throw new BoxError(BoxError.EXTERNAL_ERROR, copyError.message);
 }
 
 async function remove(apiConfig, filename) {
@@ -295,7 +290,7 @@ async function testConfig(apiConfig) {
         if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
         const error = validateBackupTarget(apiConfig.backupFolder);
         if (error) throw error;
-    } else { // cifs/ext4/nfs/mountpoint/sshfs
+    } else { // xfs/cifs/ext4/nfs/mountpoint/sshfs
         if (apiConfig.provider === PROVIDER_MOUNTPOINT) {
             if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
             const error = validateBackupTarget(apiConfig.mountPoint);

src/storage/gcs.js

@@ -31,7 +31,6 @@ const assert = require('assert'),
     constants = require('../constants.js'),
     DataLayout = require('../datalayout.js'),
     debug = require('debug')('box:storage/gcs'),
-    EventEmitter = require('events'),
     PassThrough = require('stream').PassThrough,
     path = require('path'),
     safe = require('safetydance'),
@@ -184,12 +183,11 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac
     }, callback);
 }
 
-function copy(apiConfig, oldFilePath, newFilePath) {
+async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof oldFilePath, 'string');
     assert.strictEqual(typeof newFilePath, 'string');
-
-    var events = new EventEmitter();
+    assert.strictEqual(typeof progressCallback, 'function');
 
     function copyFile(entry, iteratorCallback) {
         var relativePath = path.relative(oldFilePath, entry.fullPath);
@@ -206,20 +204,19 @@ function copy(apiConfig, oldFilePath, newFilePath) {
 
     const batchSize = 1000;
     const concurrency = apiConfig.copyConcurrency || 10;
-    var total = 0;
+    let total = 0;
 
-    listDir(apiConfig, oldFilePath, batchSize, function (entries, done) {
+    const listDirAsync = util.promisify(listDir);
+
+    const [copyError] = await safe(listDirAsync(apiConfig, oldFilePath, batchSize, function (entries, done) {
         total += entries.length;
 
-        events.emit('progress', `Copying ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}`);
+        progressCallback({ message: `Copying ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` });
 
         async.eachLimit(entries, concurrency, copyFile, done);
-    }, function (error) {
-        events.emit('progress', `Copied ${total} files`);
-        process.nextTick(() => events.emit('done', error));
-    });
+    }));
 
-    return events;
+    progressCallback({ message: `Copied ${total} files with error: ${copyError}` });
 }
 
 async function remove(apiConfig, filename) {

src/storage/interface.js

@@ -19,7 +19,6 @@ exports = module.exports = {
     exists,
 
     download,
-    downloadDir,
     copy,
 
     listDir,
@@ -36,8 +35,7 @@ exports = module.exports = {
 
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    DataLayout = require('../datalayout.js'),
-    EventEmitter = require('events');
+    DataLayout = require('../datalayout.js');
 
 function removePrivateFields(apiConfig) {
     // in-place removal of tokens and api keys with constants.SECRET_PLACEHOLDER
@@ -89,24 +87,13 @@ function download(apiConfig, backupFilePath, callback) {
     callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'download is not implemented'));
 }
 
-function downloadDir(apiConfig, backupFilePath, destDir) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupFilePath, 'string');
-    assert.strictEqual(typeof destDir, 'string');
-
-    var events = new EventEmitter();
-    process.nextTick(function () { events.emit('done', null); });
-    return events;
-}
-
-function copy(apiConfig, oldFilePath, newFilePath) {
+async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof oldFilePath, 'string');
     assert.strictEqual(typeof newFilePath, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
 
-    var events = new EventEmitter();
-    process.nextTick(function () { events.emit('done', null); });
-    return events;
+    throw new BoxError(BoxError.NOT_IMPLEMENTED, 'copy is not implemented');
 }
 
 function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {

src/storage/noop.js

@@ -7,7 +7,6 @@ exports = module.exports = {
     upload,
     exists,
     download,
-    downloadDir,
     copy,
 
     listDir,
@@ -25,8 +24,7 @@ exports = module.exports = {
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     DataLayout = require('../datalayout.js'),
-    debug = require('debug')('box:storage/noop'),
-    EventEmitter = require('events');
+    debug = require('debug')('box:storage/noop');
 
 function getRootPath(apiConfig) {
     assert.strictEqual(typeof apiConfig, 'object');
@@ -78,30 +76,13 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
     callback();
 }
 
-function downloadDir(apiConfig, backupFilePath, destDir) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof backupFilePath, 'string');
-    assert.strictEqual(typeof destDir, 'string');
-
-    var events = new EventEmitter();
-    process.nextTick(function () {
-        debug('downloadDir: %s -> %s', backupFilePath, destDir);
-
-        events.emit('done', new BoxError(BoxError.NOT_IMPLEMENTED, 'Cannot download from noop backend'));
-    });
-    return events;
-}
-
-function copy(apiConfig, oldFilePath, newFilePath) {
+async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof oldFilePath, 'string');
     assert.strictEqual(typeof newFilePath, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
 
-    debug('copy: %s -> %s', oldFilePath, newFilePath);
-
-    var events = new EventEmitter();
-    process.nextTick(function () { events.emit('done', null); });
-    return events;
+    debug(`copy: ${oldFilePath} -> ${newFilePath}`);
 }
 
 async function remove(apiConfig, filename) {
@@ -111,7 +92,7 @@ async function remove(apiConfig, filename) {
     debug(`remove: ${filename}`);
 }
 
-function removeDir(apiConfig, pathPrefix, progressCallback) {
+async function removeDir(apiConfig, pathPrefix, progressCallback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof pathPrefix, 'string');
     assert.strictEqual(typeof progressCallback, 'function');

src/storage/s3-block-read-stream.js

@@ -1,110 +0,0 @@
-'use strict';
-
-// initial code from https://github.com/tilfin/s3-block-read-stream/blob/master/LICENSE (MIT)
-
-const Readable = require('stream').Readable;
-const util = require('util');
-
-function S3ReadStream(s3, params, options) {
-    if (!(this instanceof S3ReadStream))
-        return new S3ReadStream(s3, params, options);
-
-    const opts = options || {};
-    this._s3 = s3;
-    this._params = params;
-    this._readSize = 0;
-    this._fileSize = -1;
-    this._path = params.Bucket + '/' + params.Key;
-
-    this._interval = opts.interval || 0; // msec
-    delete opts.interval;
-    this._blockSize = opts.blockSize || 64 * 1048576; //MB
-    delete opts.blockSize;
-    this._log = opts.logCallback || (opts.debug ? function(msg) { console.warn(msg); } : function(){});
-    delete opts.logCallback;
-
-    Readable.call(this, opts);
-}
-util.inherits(S3ReadStream, Readable);
-S3ReadStream.prototype._read = function() {
-    if (this._readSize === this._fileSize) {
-        this._done();
-    } else if (this._readSize) {
-        setTimeout(() => {
-            this._nextDownload();
-        }, this._interval);
-    } else {
-        this._log(`${this._path} - Start`);
-        this._fetchSize();
-    }
-};
-
-S3ReadStream.prototype._fetchSize = function() {
-    const params = {};
-    for (var key in this._params) {
-        if (!key.match(/^Response/)) {
-            params[key] = this._params[key];
-        }
-    }
-
-    this._s3.headObject(params, (err, data) => {
-        if (err) {
-            process.nextTick(() => this.emit('error', err));
-            return;
-        }
-
-        const reslen = parseInt(data.ContentLength, 10);
-        this._log(`${this._path} - File Size: ${reslen}`);
-
-        if (reslen > 0) {
-            this._fileSize = reslen;
-            this._nextDownload();
-        } else {
-            this._done();
-        }
-    });
-};
-
-S3ReadStream.prototype._downloadRange = function(offset, length) {
-    const params = Object.assign({}, this._params);
-    const lastPos = offset + length - 1;
-    const range = 'bytes=' + offset + '-' + lastPos;
-    params['Range'] = range;
-
-    this._log(`${this._path} - Download Range: ${range}`);
-
-    this._s3.getObject(params, (err, data) => {
-        if (err) {
-            process.nextTick(() => this.emit('error', err));
-            return;
-        }
-
-        const reslen = parseInt(data.ContentLength, 10);
-        this._log(`${this._path} - Received Size: ${reslen}`);
-
-        if (reslen > 0) {
-            this._readSize += reslen;
-            this.push(data.Body);
-        } else {
-            this._done();
-        }
-    });
-};
-
-S3ReadStream.prototype._nextDownload = function() {
-    let len = 0;
-    if (this._readSize + this._blockSize < this._fileSize) {
-        len = this._blockSize;
-    } else {
-        len = this._fileSize - this._readSize;
-    }
-    this._downloadRange(this._readSize, len);
-};
-
-S3ReadStream.prototype._done = function() {
-    this._readSize = 0;
-    this.push(null);
-    this._log(`${this._path} - Done`);
-};
-
-module.exports = S3ReadStream;

src/storage/s3.js

@@ -33,11 +33,9 @@ const assert = require('assert'),
     constants = require('../constants.js'),
     DataLayout = require('../datalayout.js'),
     debug = require('debug')('box:storage/s3'),
-    EventEmitter = require('events'),
     https = require('https'),
-    PassThrough = require('stream').PassThrough,
     path = require('path'),
-    S3BlockReadStream = require('./s3-block-read-stream.js'),
+    Readable = require('stream').Readable,
     safe = require('safetydance'),
     util = require('util'),
     _ = require('underscore');
@@ -173,6 +171,87 @@ async function exists(apiConfig, backupFilePath) {
     }
 }
 
+// Download the object in small parts. By downloading small parts, we reduce the chance of sporadic network errors when downloading large objects
+// We can retry each part individually, but we haven't had the need for this yet
+class S3MultipartDownloadStream extends Readable {
+    constructor (s3, params, options) {
+        super(options);
+        this._s3 = s3;
+        this._params = params;
+        this._readSize = 0;
+        this._fileSize = -1;
+        this._path = params.Bucket + '/' + params.Key;
+
+        this._blockSize = options.blockSize || 64 * 1048576; // MB
+    }
+
+    _done() {
+        this._readSize = 0;
+        this.push(null); // EOF
+    }
+
+    _handleError(error) {
+        if (S3_NOT_FOUND(error)) {
+            this.destroy(new BoxError(BoxError.NOT_FOUND, `Backup not found: ${this._path}`));
+        } else {
+            debug(`download: ${this._path} s3 stream error.`, error);
+            this.destroy(new BoxError(BoxError.EXTERNAL_ERROR, `Error multipartDownload ${this._path}. Message: ${error.message} HTTP Code: ${error.code}`));
+        }
+    }
+
+    _downloadRange(offset, length) {
+        const params = Object.assign({}, this._params);
+        const lastPos = offset + length - 1;
+        const range = `bytes=${offset}-${lastPos}`;
+        params['Range'] = range;
+
+        this._s3.getObject(params, (error, data) => {
+            if (error) return this._handleError(error);
+
+            const length = parseInt(data.ContentLength, 10);
+
+            if (length > 0) {
+                this._readSize += length;
+                this.push(data.Body);
+            } else {
+                this._done();
+            }
+        });
+    }
+
+    _nextDownload() {
+        let len = 0;
+        if (this._readSize + this._blockSize < this._fileSize) {
+            len = this._blockSize;
+        } else {
+            len = this._fileSize - this._readSize;
+        }
+        this._downloadRange(this._readSize, len);
+    }
+
+    _fetchSize() {
+        this._s3.headObject(this._params, (error, data) => {
+            if (error) return this._handleError(error);
+
+            const length = parseInt(data.ContentLength, 10);
+
+            if (length > 0) {
+                this._fileSize = length;
+                this._nextDownload();
+            } else {
+                this._done();
+            }
+        });
+    }
+
+    _read() {
+        if (this._readSize === this._fileSize) return this._done();
+        if (this._readSize === 0) return this._fetchSize();
+
+        this._nextDownload();
+    }
+}
+
 function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');
@@ -187,21 +266,8 @@ function download(apiConfig, backupFilePath, callback) {
 
     const s3 = new aws.S3(credentials);
 
-    const ps = new PassThrough();
-    const multipartDownload = new S3BlockReadStream(s3, params, { blockSize: 64 * 1024 * 1024 /*, logCallback: debug */ });
-
-    multipartDownload.on('error', function (error) {
-        if (S3_NOT_FOUND(error)) {
-            ps.emit('error', new BoxError(BoxError.NOT_FOUND, `Backup not found: ${backupFilePath}`));
-        } else {
-            debug(`download: ${apiConfig.bucket}:${backupFilePath} s3 stream error.`, error);
-            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, `Error multipartDownload ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`));
-        }
-    });
-
-    multipartDownload.pipe(ps);
-
-    callback(null, ps);
+    const multipartDownloadStream = new S3MultipartDownloadStream(s3, params, { blockSize: 64 * 1024 * 1024 });
+    return callback(null, multipartDownloadStream);
 }
 
 function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
@@ -256,12 +322,11 @@ function encodeCopySource(bucket, path) {
     return `/${bucket}/${output}`;
 }
 
-function copy(apiConfig, oldFilePath, newFilePath) {
+async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof oldFilePath, 'string');
     assert.strictEqual(typeof newFilePath, 'string');
-
-    const events = new EventEmitter();
+    assert.strictEqual(typeof progressCallback, 'function');
 
     function copyFile(entry, iteratorCallback) {
         const credentials = getS3Config(apiConfig);
@@ -288,11 +353,11 @@ function copy(apiConfig, oldFilePath, newFilePath) {
         const largeFileLimit = (apiConfig.provider === 'exoscale-sos' || apiConfig.provider === 'backblaze-b2' || apiConfig.provider === 'digitalocean-spaces') ? 1024 * 1024 * 1024 : 5 * 1024 * 1024 * 1024;
 
         if (entry.size < largeFileLimit) {
-            events.emit('progress', `Copying ${relativePath || oldFilePath}`);
+            progressCallback({ message: `Copying ${relativePath || oldFilePath}` });
 
             copyParams.CopySource = encodeCopySource(apiConfig.bucket, entry.fullPath);
             s3.copyObject(copyParams, done).on('retry', function (response) {
-                events.emit('progress', `Retrying (${response.retryCount+1}) copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}`);
+                progressCallback({ message: `Retrying (${response.retryCount+1}) copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}` });
                 // on DO, we get a random 408. these are not retried by the SDK
                 if (response.error) response.error.retryable = true; // https://github.com/aws/aws-sdk-js/issues/412
             });
@@ -300,7 +365,7 @@ function copy(apiConfig, oldFilePath, newFilePath) {
             return;
         }
 
-        events.emit('progress', `Copying (multipart) ${relativePath || oldFilePath}`);
+        progressCallback({ message: `Copying (multipart) ${relativePath || oldFilePath}` });
 
         s3.createMultipartUpload(copyParams, function (error, multipart) {
             if (error) return done(error);
@@ -327,12 +392,12 @@ function copy(apiConfig, oldFilePath, newFilePath) {
                     UploadId: uploadId
                 };
 
-                events.emit('progress', `Copying part ${partCopyParams.PartNumber} - ${partCopyParams.CopySource} ${partCopyParams.CopySourceRange}`);
+                progressCallback({ message: `Copying part ${partCopyParams.PartNumber} - ${partCopyParams.CopySource} ${partCopyParams.CopySourceRange}` });
 
                 s3.uploadPartCopy(partCopyParams, function (error, part) {
                     if (error) return iteratorDone(error);
 
-                    events.emit('progress', `Copying part ${partCopyParams.PartNumber} - Etag: ${part.CopyPartResult.ETag}`);
+                    progressCallback({ message: `Copying part ${partCopyParams.PartNumber} - Etag: ${part.CopyPartResult.ETag}` });
 
                     if (!part.CopyPartResult.ETag) return iteratorDone(new Error('Multi-part copy is broken or not implemented by the S3 storage provider'));
 
@@ -340,7 +405,7 @@ function copy(apiConfig, oldFilePath, newFilePath) {
 
                     iteratorDone();
                 }).on('retry', function (response) {
-                    events.emit('progress', `Retrying (${response.retryCount+1}) multipart copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}`);
+                    progressCallback({ message: `Retrying (${response.retryCount+1}) multipart copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}` });
                 });
             }, function chunksCopied(error) {
                 if (error) { // we must still recommend the user to set a AbortIncompleteMultipartUpload lifecycle rule
@@ -349,7 +414,7 @@ function copy(apiConfig, oldFilePath, newFilePath) {
                         Key: path.join(newFilePath, relativePath),
                         UploadId: uploadId
                     };
-                    events.emit('progress', `Aborting multipart copy of ${relativePath || oldFilePath}`);
+                    progressCallback({ message: `Aborting multipart copy of ${relativePath || oldFilePath}` });
                     return s3.abortMultipartUpload(abortParams, () => done(error)); // ignore any abort errors
                 }
 
@@ -360,7 +425,7 @@ function copy(apiConfig, oldFilePath, newFilePath) {
                     UploadId: uploadId
                 };
 
-                events.emit('progress', `Finishing multipart copy - ${completeMultipartParams.Key}`);
+                progressCallback({ message: `Finishing multipart copy - ${completeMultipartParams.Key}` });
 
                 s3.completeMultipartUpload(completeMultipartParams, done);
             });
@@ -369,21 +434,19 @@ function copy(apiConfig, oldFilePath, newFilePath) {
 
     let total = 0;
     const concurrency = apiConfig.copyConcurrency || (apiConfig.provider === 's3' ? 500 : 10);
-    events.emit('progress', `Copying with concurrency of ${concurrency}`);
+    progressCallback({ message: `Copying with concurrency of ${concurrency}` });
 
-    listDir(apiConfig, oldFilePath, 1000, function listDirIterator(entries, done) {
+    const listDirAsync = util.promisify(listDir);
+
+    const [copyError] = await safe(listDirAsync(apiConfig, oldFilePath, 1000, function listDirIterator(entries, done) {
         total += entries.length;
 
-        events.emit('progress', `Copying files from ${total-entries.length}-${total}`);
+        progressCallback({ message: `Copying files from ${total-entries.length}-${total}` });
 
         async.eachLimit(entries, concurrency, copyFile, done);
-    }, function (error) {
-        events.emit('progress', `Copied ${total} files with error: ${error}`);
+    }));
 
-        process.nextTick(() => events.emit('done', error));
-    });
-
-    return events;
+    progressCallback({ message: `Copied ${total} files with error: ${copyError}` });
 }
 
 async function remove(apiConfig, filename) {

src/system.js

@@ -41,10 +41,11 @@ async function getAppDisks(appsDataDisk) {
 
     const allApps = await apps.list();
     for (const app of allApps) {
-        if (!app.dataDir) {
+        if (!app.storageVolumeId) {
             appDisks[app.id] = appsDataDisk;
         } else {
-            const [error, result] = await safe(df.file(app.dataDir));
+            const dataDir = await apps.getStorageDir(app);
+            const [error, result] = await safe(df.file(dataDir));
             appDisks[app.id] = error ? appsDataDisk : result.filesystem; // ignore any errors
         }
     }
@@ -68,7 +69,7 @@ async function getDisks() {
     if (error) throw new BoxError(BoxError.FS_ERROR, error);
 
     // filter by ext4 and then sort to make sure root disk is first
-    const ext4Disks = allDisks.filter((r) => r.type === 'ext4').sort((a, b) => a.mountpoint.localeCompare(b.mountpoint));
+    const ext4Disks = allDisks.filter((r) => r.type === 'ext4' || r.type === 'xfs').sort((a, b) => a.mountpoint.localeCompare(b.mountpoint));
 
     const diskInfos = [];
     for (const p of [ paths.BOX_DATA_DIR, paths.MAIL_DATA_DIR, paths.PLATFORM_DATA_DIR, paths.APPS_DATA_DIR, info.DockerRootDir ]) {
@@ -80,7 +81,7 @@ async function getDisks() {
     const backupsFilesystem = await getBackupsFilesystem();
 
     const result = {
-        disks: ext4Disks, // root disk is first. { filesystem, type, size, used, avialable, capacity, mountpoint }
+        disks: ext4Disks, // root disk is first. { filesystem, type, size, used, available, capacity, mountpoint }
         boxDataDisk: diskInfos[0].filesystem,
         mailDataDisk: diskInfos[1].filesystem,
         platformDataDisk: diskInfos[2].filesystem,

src/taskworker.js

@@ -31,7 +31,7 @@ const TASKS = { // indexed by task type
 
     _identity: async (arg, progressCallback) => { progressCallback(); return arg; },
     _error: async (arg, progressCallback) => { progressCallback(); throw new Error(`Failed for arg: ${arg}`); },
-    _crash: async (arg) => { throw new Error(`Crashing for arg: ${arg}`); }, // the test looks for this debug string in the log file
+    _crash: (arg) => { throw new Error(`Crashing for arg: ${arg}`); }, // the test looks for this debug string in the log file
     _sleep: async (arg) => setTimeout(process.exit, arg)
 };
 

src/test/apps-test.js

@@ -337,6 +337,9 @@ describe('Apps', function () {
 
             safe(() => apps._parseCrontab('*/1  *  *  *\t* ')); // no command
             expect(safe.error.message).to.be('Invalid cron configuration at line 1');
+
+            safe(() => apps._parseCrontab('@whatever /bin/false')); // invalid extension
+            expect(safe.error.message).to.be('Unknown extension pattern at line 1');
         });
 
         it('succeeds for crontab', function () {
@@ -353,5 +356,16 @@ describe('Apps', function () {
             expect(result[1].command).to.be('echo "==> This is a custom cron task running every hour"');
 
         });
+
+        it('succeeds for crontab (extensions)', function () {
+            const result = apps._parseCrontab('@service /bin/service\n\n@weekly /bin/weekly');
+            expect(result.length).to.be(2);
+
+            expect(result[0].schedule).to.be('@service');
+            expect(result[0].command).to.be('/bin/service');
+
+            expect(result[1].schedule).to.be('0 0 * * 0');
+            expect(result[1].command).to.be('/bin/weekly');
+        });
     });
 });

src/test/backupformat-test.js

@@ -0,0 +1,61 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+const common = require('./common.js'),
+    DataLayout = require('../datalayout.js'),
+    expect = require('expect.js'),
+    fs = require('fs'),
+    os = require('os'),
+    path = require('path'),
+    rsync = require('../backupformat/rsync.js');
+
+describe('backuptask', function () {
+    const { setup, cleanup, createTree } = common;
+
+    before(setup);
+    after(cleanup);
+
+    describe('fs meta data', function () {
+        let tmpdir;
+        before(function () {
+            tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'backups-test'));
+        });
+        after(function () {
+            fs.rmSync(tmpdir, { recursive: true, force: true });
+        });
+
+        it('saves special files', async function () {
+            createTree(tmpdir, { 'data': { 'subdir': { 'emptydir': { } } }, 'dir2': { 'file': 'stuff' } });
+            fs.chmodSync(path.join(tmpdir, 'dir2/file'), parseInt('0755', 8));
+
+            let dataLayout = new DataLayout(tmpdir, []);
+
+            await rsync._saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
+
+            const emptyDirs = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).emptyDirs;
+            expect(emptyDirs).to.eql(['./data/subdir/emptydir']);
+
+            const execFiles = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).execFiles;
+            expect(execFiles).to.eql(['./dir2/file']);
+        });
+
+        it('restores special files', async function () {
+            fs.rmSync(path.join(tmpdir, 'data'), { recursive: true, force: true });
+
+            expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(false); // just make sure rimraf worked
+
+            let dataLayout = new DataLayout(tmpdir, []);
+
+            await rsync._restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
+
+            expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(true);
+            const mode = fs.statSync(path.join(tmpdir, 'dir2/file')).mode;
+            expect(mode & ~fs.constants.S_IFREG).to.be(parseInt('0755', 8));
+        });
+    });
+});

src/test/backuptask-test.js

@@ -7,9 +7,7 @@
 'use strict';
 
 const backups = require('../backups.js'),
-    backuptask = require('../backuptask.js'),
     common = require('./common.js'),
-    DataLayout = require('../datalayout.js'),
     delay = require('../delay.js'),
     expect = require('expect.js'),
     fs = require('fs'),
@@ -19,51 +17,11 @@ const backups = require('../backups.js'),
     tasks = require('../tasks.js');
 
 describe('backuptask', function () {
-    const { setup, cleanup, createTree } = common;
+    const { setup, cleanup } = common;
 
     before(setup);
     after(cleanup);
 
-    describe('fs meta data', function () {
-        let tmpdir;
-        before(function () {
-            tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'backups-test'));
-        });
-        after(function () {
-            fs.rmSync(tmpdir, { recursive: true, force: true });
-        });
-
-        it('saves special files', async function () {
-            createTree(tmpdir, { 'data': { 'subdir': { 'emptydir': { } } }, 'dir2': { 'file': 'stuff' } });
-            fs.chmodSync(path.join(tmpdir, 'dir2/file'), parseInt('0755', 8));
-
-            let dataLayout = new DataLayout(tmpdir, []);
-
-            await backuptask._saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
-
-            const emptyDirs = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).emptyDirs;
-            expect(emptyDirs).to.eql(['./data/subdir/emptydir']);
-
-            const execFiles = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).execFiles;
-            expect(execFiles).to.eql(['./dir2/file']);
-        });
-
-        it('restores special files', async function () {
-            fs.rmSync(path.join(tmpdir, 'data'), { recursive: true, force: true });
-
-            expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(false); // just make sure rimraf worked
-
-            let dataLayout = new DataLayout(tmpdir, []);
-
-            await backuptask._restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
-
-            expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(true);
-            const mode = fs.statSync(path.join(tmpdir, 'dir2/file')).mode;
-            expect(mode & ~fs.constants.S_IFREG).to.be(parseInt('0755', 8));
-        });
-    });
-
-
     describe('fullBackup', function () {
         let backupInfo1;
 

src/test/dns-providers-test.js

@@ -338,18 +338,13 @@ describe('dns provider', function () {
                 data: '1.2.3.4'
             }];
 
-            const DOMAIN_RECORD_1 = [{ // replaced
-                ttl: 600,
-                data: '0.0.0.0'
-            }];
-
             const req1 = nock(GODADDY_API)
                 .get('/' + domainCopy.zoneName + '/records/A/test')
                 .reply(200, DOMAIN_RECORD_0);
 
             const req2 = nock(GODADDY_API)
-                .put('/' + domainCopy.zoneName + '/records/A/test', DOMAIN_RECORD_1)
-                .reply(200, {});
+                .delete('/' + domainCopy.zoneName + '/records/A/test')
+                .reply(204, {});
 
             await dns.removeDnsRecords('test', domainCopy.domain, 'A', ['1.2.3.4']);
             expect(req1.isDone()).to.be.ok();

src/test/ldap-test.js

@@ -221,117 +221,78 @@ describe('Ldap', function () {
             mockApp.accessRestriction = null;
 
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: 'objectclass=group' });
-            expect(entries.length).to.equal(4);
-
-            // ensure order for testability
-            entries.sort(function (a, b) { return a.cn < b.cn; });
-
-            expect(entries[0].cn).to.equal('users');
-            expect(entries[0].memberuid.length).to.equal(2);
-            expect(entries[0].memberuid).to.contain(admin.id);
-            expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
-        });
-
-        it ('succeeds with cn wildcard filter', async function () {
-            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' });
-            expect(entries.length).to.equal(4);
-
-            // ensure order for testability
-            entries.sort(function (a, b) { return a.cn < b.cn; });
-
-            expect(entries[0].cn).to.equal('users');
-            expect(entries[0].memberuid.length).to.equal(2);
-            expect(entries[0].memberuid).to.contain(admin.id);
-            expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
-        });
-
-        it('succeeds with memberuid filter', async function () {
-            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(memberuid=' + user.id + ')' });
             expect(entries.length).to.equal(2);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
 
-            expect(entries[1].cn).to.equal('ldap-test-1');
-            expect(entries[1].memberuid.length).to.equal(2);
-            expect(entries[1].memberuid).to.contain(admin.id);
-            expect(entries[1].memberuid).to.contain(user.id);
+            expect(entries[1].cn).to.equal('ldap-test-2');
+            expect(entries[1].memberuid).to.equal(admin.id);
+        });
+
+        it ('succeeds with cn wildcard filter', async function () {
+            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' });
+            expect(entries.length).to.equal(2);
+
+            // ensure order for testability
+            entries.sort(function (a, b) { return a.cn < b.cn; });
+
+            expect(entries[0].cn).to.equal('ldap-test-1');
+            expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
+
+            expect(entries[1].cn).to.equal('ldap-test-2');
+            expect(entries[1].memberuid).to.equal(admin.id);
+        });
+
+        it('succeeds with memberuid filter', async function () {
+            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(memberuid=' + user.id + ')' });
+            expect(entries.length).to.equal(1);
+
+            // ensure order for testability
+            entries.sort(function (a, b) { return a.cn < b.cn; });
+
+            expect(entries[0].cn).to.equal('ldap-test-1');
+            expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
         });
 
         it ('does only list groups who have access', async function () {
             mockApp.accessRestriction = { users: [], groups: [ group.id ] };
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' });
+            expect(entries.length).to.equal(1);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries.length).to.equal(3);
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
             expect(entries[0].memberuid).to.contain(admin.id);
             expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
         });
 
         it ('succeeds with pagination', async function () {
             mockApp.accessRestriction = null;
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: 'objectclass=group', paged: true });
-            expect(entries.length).to.equal(4);
+            expect(entries.length).to.equal(2);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
             expect(entries[0].memberuid).to.contain(admin.id);
             expect(entries[0].memberuid).to.contain(user.id);
 
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
+            expect(entries[1].cn).to.equal('ldap-test-2');
             expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
         });
     });
 

src/test/mail-test.js

@@ -75,23 +75,43 @@ describe('Mail', function () {
 
     describe('mailbox name', function () {
         it('allows valid names', function () {
-            expect(mail._validateName('1')).to.be(null); // single char
-            expect(mail._validateName('ap')).to.be(null); // alpha
-            expect(mail._validateName('aP')).to.be(null); // caps
-            expect(mail._validateName('0P')).to.be(null); // number
-            expect(mail._validateName('a.p.x')).to.be(null); // dot
-            expect(mail._validateName('a-p-x')).to.be(null); // hyphen
-            expect(mail._validateName('a-p_x')).to.be(null); // underscore
+            expect(mail.validateName('1')).to.be(null); // single char
+            expect(mail.validateName('ap')).to.be(null); // alpha
+            expect(mail.validateName('aP')).to.be(null); // caps
+            expect(mail.validateName('0P')).to.be(null); // number
+            expect(mail.validateName('a.p.x')).to.be(null); // dot
+            expect(mail.validateName('a-p-x')).to.be(null); // hyphen
+            expect(mail.validateName('a-p_x')).to.be(null); // underscore
         });
 
         it('disallows invalid names', function () {
-            expect(mail._validateName('@')).to.be.an(Error);
-            expect(mail._validateName('a+p')).to.be.an(Error);
-            expect(mail._validateName('a#p')).to.be.an(Error);
-            expect(mail._validateName('a!')).to.be.an(Error);
+            expect(mail.validateName('@')).to.be.an(Error);
+            expect(mail.validateName('a+p')).to.be.an(Error);
+            expect(mail.validateName('a#p')).to.be.an(Error);
+            expect(mail.validateName('a!')).to.be.an(Error);
         });
     });
 
+    describe('mailbox display name', function () {
+        it('allows valid names', function () {
+            expect(mail.validateDisplayName('1')).to.be(null); // single char
+            expect(mail.validateDisplayName('ap')).to.be(null); // alpha
+            expect(mail.validateDisplayName('aP')).to.be(null); // caps
+            expect(mail.validateDisplayName('0P')).to.be(null); // number
+            expect(mail.validateDisplayName('a p.x')).to.be(null); // space
+            expect(mail.validateDisplayName('a-p-x')).to.be(null); // hyphen
+            expect(mail.validateDisplayName('a-p_x')).to.be(null); // underscore
+        });
+
+        it('disallows invalid names', function () {
+            expect(mail.validateDisplayName('@')).to.be.an(Error);
+            expect(mail.validateDisplayName('a<p')).to.be.an(Error);
+            expect(mail.validateDisplayName('a>p')).to.be.an(Error);
+            expect(mail.validateDisplayName('a"x"')).to.be.an(Error);
+        });
+    });
+
+
     describe('mailboxes', function () {
         it('add user mailbox succeeds', async function () {
             await mail.addMailbox('girish', domain.domain, { ownerId: 'uid-0', ownerType: mail.OWNERTYPE_USER, active: true }, auditSource);

src/test/storage-test.js

@@ -139,16 +139,12 @@ describe('Storage', function () {
             });
         });
 
-        it('can copy', function (done) {
+        it('can copy', async function () {
             const sourceFile = gTmpFolder + '/uploadtest/test.txt'; // keep the test within save device
             const destFile = gTmpFolder + '/uploadtest/test-hardlink.txt';
 
-            const events = filesystem.copy(gBackupConfig, sourceFile, destFile);
-            events.on('done', function (error) {
-                expect(error).to.be(null);
-                expect(fs.statSync(destFile).nlink).to.be(2); // created a hardlink
-                done();
-            });
+            await filesystem.copy(gBackupConfig, sourceFile, destFile, () => {});
+            expect(fs.statSync(destFile).nlink).to.be(2); // created a hardlink
         });
 
         it('can remove file', async function () {
@@ -193,12 +189,8 @@ describe('Storage', function () {
             }, done);
         });
 
-        it('can copy', function (done) {
-            const events = noop.copy(gBackupConfig, 'sourceFile', 'destFile');
-            events.on('done', function (error) {
-                expect(error).to.be(null);
-                done();
-            });
+        it('can copy', async function () {
+            await noop.copy(gBackupConfig, 'sourceFile', 'destFile', () => {});
         });
 
         it('can remove file', async function () {
@@ -269,21 +261,15 @@ describe('Storage', function () {
             });
         });
 
-        it('can copy', function (done) {
+        it('can copy', async function () {
             fs.writeFileSync(path.join(gS3Folder, 'uploadtest/C++.gitignore'), 'special', 'utf8');
 
             const sourceKey = 'uploadtest';
 
-            const events = s3.copy(gBackupConfig, sourceKey, 'uploadtest-copy');
-            events.on('done', function (error) {
-                const sourceFile = path.join(__dirname, 'storage/data/test.txt');
-                expect(error).to.be(null);
-                expect(fs.statSync(path.join(gS3Folder, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size);
-
-                expect(fs.statSync(path.join(gS3Folder, 'uploadtest-copy/C++.gitignore')).size).to.be(7);
-
-                done();
-            });
+            await s3.copy(gBackupConfig, sourceKey, 'uploadtest-copy', () => {});
+            const sourceFile = path.join(__dirname, 'storage/data/test.txt');
+            expect(fs.statSync(path.join(gS3Folder, 'uploadtest-copy/test.txt')).size).to.be(fs.statSync(sourceFile).size);
+            expect(fs.statSync(path.join(gS3Folder, 'uploadtest-copy/C++.gitignore')).size).to.be(7);
         });
 
         xit('can remove file', async function () {

src/test/userdirectory-test.js

@@ -210,94 +210,64 @@ describe('User Directory Ldap', function () {
             mockApp.accessRestriction = null;
 
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: 'objectclass=group' }, auth);
-            expect(entries.length).to.equal(4);
-
-            // ensure order for testability
-            entries.sort(function (a, b) { return a.cn < b.cn; });
-
-            expect(entries[0].cn).to.equal('users');
-            expect(entries[0].memberuid.length).to.equal(2);
-            expect(entries[0].memberuid).to.contain(admin.id);
-            expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
-        });
-
-        it ('succeeds with cn wildcard filter', async function () {
-            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' }, auth);
-            expect(entries.length).to.equal(4);
-
-            // ensure order for testability
-            entries.sort(function (a, b) { return a.cn < b.cn; });
-
-            expect(entries[0].cn).to.equal('users');
-            expect(entries[0].memberuid.length).to.equal(2);
-            expect(entries[0].memberuid).to.contain(admin.id);
-            expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
-        });
-
-        it('succeeds with memberuid filter', async function () {
-            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(memberuid=' + user.id + ')' }, auth);
             expect(entries.length).to.equal(2);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
 
-            expect(entries[1].cn).to.equal('ldap-test-1');
-            expect(entries[1].memberuid.length).to.equal(2);
-            expect(entries[1].memberuid).to.contain(admin.id);
-            expect(entries[1].memberuid).to.contain(user.id);
+            expect(entries[1].cn).to.equal('ldap-test-2');
+            expect(entries[1].memberuid).to.equal(admin.id);
+        });
+
+        it ('succeeds with cn wildcard filter', async function () {
+            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' }, auth);
+            expect(entries.length).to.equal(2);
+
+            // ensure order for testability
+            entries.sort(function (a, b) { return a.cn < b.cn; });
+
+            expect(entries[0].cn).to.equal('ldap-test-1');
+            expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
+
+            expect(entries[1].cn).to.equal('ldap-test-2');
+            expect(entries[1].memberuid).to.equal(admin.id);
+        });
+
+        it('succeeds with memberuid filter', async function () {
+            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(memberuid=' + user.id + ')' }, auth);
+            expect(entries.length).to.equal(1);
+
+            // ensure order for testability
+            entries.sort(function (a, b) { return a.cn < b.cn; });
+
+            expect(entries[0].cn).to.equal('ldap-test-1');
+            expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
         });
 
         it ('succeeds with pagination', async function () {
             mockApp.accessRestriction = null;
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: 'objectclass=group', paged: true }, auth);
-            expect(entries.length).to.equal(4);
+            expect(entries.length).to.equal(2);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
             expect(entries[0].memberuid).to.contain(admin.id);
             expect(entries[0].memberuid).to.contain(user.id);
 
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
+            expect(entries[1].cn).to.equal('ldap-test-2');
             expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
         });
     });
 });

src/userdirectory.js

@@ -207,37 +207,6 @@ async function groupSearch(req, res, next) {
 
     const results = [];
 
-    // those are the old virtual groups for backwards compat
-    const virtualGroups = [{
-        name: 'users',
-        admin: false
-    }, {
-        name: 'admins',
-        admin: true
-    }];
-
-    virtualGroups.forEach(function (group) {
-        const dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-        const members = group.admin ? result.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : result;
-
-        const obj = {
-            dn: dn.toString(),
-            attributes: {
-                objectclass: ['group'],
-                cn: group.name,
-                memberuid: members.map(function(entry) { return entry.id; }).sort()
-            }
-        };
-
-        // ensure all filter values are also lowercase
-        const lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
-        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
-        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-            results.push(obj);
-        }
-    });
-
     let [errorGroups, resultGroups] = await safe(groups.listWithMembers());
     if (errorGroups) return next(new ldap.OperationsError(errorGroups.toString()));
 

src/users.js

@@ -48,6 +48,9 @@ exports = module.exports = {
     setAvatar,
     getAvatar,
 
+    getBackgroundImage,
+    setBackgroundImage,
+
     AP_MAIL: 'mail',
     AP_WEBADMIN: 'webadmin',
 
@@ -61,7 +64,7 @@ exports = module.exports = {
 
 const ORDERED_ROLES = [ exports.ROLE_USER, exports.ROLE_USER_MANAGER, exports.ROLE_MAIL_MANAGER, exports.ROLE_ADMIN, exports.ROLE_OWNER ];
 
-// the avatar field is special and not added here to reduce response sizes
+// the avatar and backgroundImage fields are special and not added here to reduce response sizes
 const USERS_FIELDS = [ 'id', 'username', 'email', 'fallbackEmail', 'password', 'salt', 'creationTime', 'inviteToken', 'resetToken', 'displayName',
     'twoFactorAuthenticationEnabled', 'twoFactorAuthenticationSecret', 'active', 'source', 'role', 'resetTokenCreationTime', 'loginLocationsJson' ].join(',');
 
@@ -903,3 +906,19 @@ async function setAvatar(id, avatar) {
     const result = await database.query('UPDATE users SET avatar=? WHERE id = ?', [ avatar, id ]);
     if (result.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'User not found');
 }
+
+async function getBackgroundImage(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query('SELECT backgroundImage FROM users WHERE id = ?', [ id ]);
+    if (result.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'User not found');
+    return result[0].backgroundImage;
+}
+
+async function setBackgroundImage(id, backgroundImage) {
+    assert.strictEqual(typeof id, 'string');
+    assert(Buffer.isBuffer(backgroundImage) || backgroundImage === null);
+
+    const result = await database.query('UPDATE users SET backgroundImage=? WHERE id = ?', [ backgroundImage, id ]);
+    if (result.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'User not found');
+}