7.2.5 changes

This is because GROUP_CONCAT defaults to 1024. uuid is 40 chars.
1024/40 = ~25

(cherry picked from commit 94a196bfa0)

CHANGES

@@ -2472,6 +2472,10 @@
 * nginx: use ubuntu packages for ubuntu 20.04 and 22.04
 * Ubuntu 22.04 LTS support
 * Add Hetzner DNS
+* cron: add support for extensions (@reboot, @weekly etc)
+* Add profile backgroundImage api
+* exec: rework API to get exit code
+* Add update available filter
 
 [7.2.1]
 * Refactor backup code to use async/await
@@ -2481,3 +2485,26 @@
 * Add profile backgroundImage api
 * cron: add support for extensions (@reboot, @weekly etc)
 
+[7.2.2]
+* Update cloudron-manifestformat for new scheduler patterns
+* collectd: FQDNLookup causes collectd install to fail
+
+[7.2.3]
+* appstore: allow re-registration on server side delete
+* transfer ownership route is not used anymore
+* graphite: fix issue where disk names with '.' do not render
+* dark mode fixes
+* sendmail: mail from display name
+* Use volumes for app data instead of raw path
+* initial xfs support
+
+[7.2.4]
+* volumes: Ensure long volume names do not overflow the table
+* Move all appstore filter to the left
+* app data: allow sameness of old and new dir
+
+[7.2.5]
+* Fix storage volume migration
+* Fix issue where only 25 group members were returned
+* Fix eventlog display
+

migrations/20220601004757-apps-add-mailboxDisplayName.js

@@ -0,0 +1,12 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN mailboxDisplayName VARCHAR(128) DEFAULT "" NOT NULL', [], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN mailboxDisplayName', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20220602050517-apps-add-storageVolumeId.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const path = require('path'),
+    safe = require('safetydance'),
+    uuid = require('uuid');
+
+function getMountPoint(dataDir) {
+    const output = safe.child_process.execSync(`df --output=target "${dataDir}" | tail -1`, { encoding: 'utf8' });
+    if (!output) return dataDir;
+    const mountPoint = output.trim();
+    if (mountPoint === '/') return dataDir;
+    return mountPoint;
+}
+
+exports.up = async function(db) {
+    // use safe() here because this migration failed midway in 7.2.4
+    await safe(db.runSql('ALTER TABLE apps ADD storageVolumeId VARCHAR(128), ADD FOREIGN KEY(storageVolumeId) REFERENCES volumes(id)'));
+    await safe(db.runSql('ALTER TABLE apps ADD storageVolumePrefix VARCHAR(128)'));
+    await safe(db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_storageVolume UNIQUE (storageVolumeId, storageVolumePrefix)'));
+
+    const apps = await db.runSql('SELECT * FROM apps WHERE dataDir IS NOT NULL');
+
+    for (const app of apps) {
+        if (app.storageVolumeId) {
+            console.log(`data-dir (${app.id}): app was migrated in 7.2.4`);
+            continue;
+        }
+
+        const allVolumes = await db.runSql('SELECT * FROM volumes');
+
+        console.log(`data-dir (${app.id}): migrating data dir ${app.dataDir}`);
+
+        const mountPoint = getMountPoint(app.dataDir);
+        const prefix = path.relative(mountPoint, app.dataDir);
+
+        console.log(`data-dir (${app.id}): migrating to mountpoint ${mountPoint} and prefix ${prefix}`);
+
+        const volume = allVolumes.find(v => v.hostPath === mountPoint);
+        if (volume) {
+            console.log(`data-dir (${app.id}): using existing volume ${volume.id}`);
+            await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ volume.id, prefix, app.id ]);
+            continue;
+        }
+
+        const id = uuid.v4().replace(/-/g, ''); // to make systemd mount file names more readable
+        const name = `appdata-${id}`;
+        const type = app.dataDir === mountPoint ? 'filesystem' : 'mountpoint';
+
+        console.log(`data-dir (${app.id}): creating new volume ${id}`);
+        await db.runSql('INSERT INTO volumes (id, name, hostPath, mountType, mountOptionsJson) VALUES (?, ?, ?, ?, ?)', [ id, name, mountPoint, type, JSON.stringify({}) ]);
+        await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ id, prefix, app.id ]);
+    }
+
+    await db.runSql('ALTER TABLE apps DROP COLUMN dataDir');
+};
+
+exports.down = async function(/*db*/) {
+};

migrations/schema.sql

@@ -86,13 +86,15 @@ CREATE TABLE IF NOT EXISTS apps(
     enableAutomaticUpdate BOOLEAN DEFAULT 1,
     enableMailbox BOOLEAN DEFAULT 1, // whether sendmail addon is enabled
     mailboxName VARCHAR(128), // mailbox of this app
-    mailboxDomain VARCHAR(128), // mailbox domain of this apps
+    mailboxDomain VARCHAR(128), // mailbox domain of this app
+    mailboxDisplayName VARCHAR(128), // mailbox display name
     enableInbox BOOLEAN DEFAULT 0, // whether recvmail addon is enabled
     inboxName VARCHAR(128), // mailbox of this app
-    inboxDomain VARCHAR(128), // mailbox domain of this apps
+    inboxDomain VARCHAR(128), // mailbox domain of this app
     label VARCHAR(128),     // display name
     tagsJson VARCHAR(2048), // array of tags
-    dataDir VARCHAR(256) UNIQUE,
+    storageVolumeId VARCHAR(128),
+    storageVolumePrefix VARCHAR(128),
     taskId INTEGER, // current task
     errorJson TEXT,
     servicesConfigJson TEXT, // app services configuration
@@ -103,6 +105,8 @@ CREATE TABLE IF NOT EXISTS apps(
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
+    FOREIGN KEY(storageVolumeId) REFERENCES volumes(id),
+    UNIQUE (storageVolumeId, storageVolumePrefix),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(

package-lock.json

@@ -15,7 +15,7 @@
         "aws-sdk": "^2.1115.0",
         "basic-auth": "^2.0.1",
         "body-parser": "^1.20.0",
-        "cloudron-manifestformat": "^5.15.2",
+        "cloudron-manifestformat": "^5.16.0",
         "connect": "^3.7.0",
         "connect-lastmile": "^2.1.1",
         "connect-timeout": "^1.9.0",
@@ -1413,9 +1413,9 @@
       }
     },
     "node_modules/cloudron-manifestformat": {
-      "version": "5.15.2",
-      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.15.2.tgz",
-      "integrity": "sha512-FD7s8IG0SztQjqMn0N0ko15Z7um+4+zGfok8xP21TCdOnuOZRdKvXsGHGU6VQNMCm8vLubhOSIRn2L2fRKzgQw==",
+      "version": "5.16.0",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.16.0.tgz",
+      "integrity": "sha512-YNlHrCMmz/L/wtkGOnB+lBa+aBGDDKZd9+pzxL5S4jZTYOcLgQwaD1+fX1Zqts0GnBk14hRsvJocO76qivYn/A==",
       "dependencies": {
         "cron": "^1.8.2",
         "java-packagename-regex": "^1.0.0",
@@ -9012,9 +9012,9 @@
       }
     },
     "cloudron-manifestformat": {
-      "version": "5.15.2",
-      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.15.2.tgz",
-      "integrity": "sha512-FD7s8IG0SztQjqMn0N0ko15Z7um+4+zGfok8xP21TCdOnuOZRdKvXsGHGU6VQNMCm8vLubhOSIRn2L2fRKzgQw==",
+      "version": "5.16.0",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.16.0.tgz",
+      "integrity": "sha512-YNlHrCMmz/L/wtkGOnB+lBa+aBGDDKZd9+pzxL5S4jZTYOcLgQwaD1+fX1Zqts0GnBk14hRsvJocO76qivYn/A==",
       "requires": {
         "cron": "^1.8.2",
         "java-packagename-regex": "^1.0.0",

package.json

@@ -18,7 +18,7 @@
     "aws-sdk": "^2.1115.0",
     "basic-auth": "^2.0.1",
     "body-parser": "^1.20.0",
-    "cloudron-manifestformat": "^5.15.2",
+    "cloudron-manifestformat": "^5.16.0",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.1.1",
     "connect-timeout": "^1.9.0",

scripts/cloudron-setup

@@ -26,8 +26,8 @@ readonly GREEN='\033[32m'
 readonly DONE='\033[m'
 
 # verify the system has minimum requirements met
-if [[ "${rootfs_type}" != "ext4" ]]; then
-    echo "Error: Cloudron requires '/' to be ext4" # see #364
+if [[ "${rootfs_type}" != "ext4" && "${rootfs_type}" != "xfs" ]]; then
+    echo "Error: Cloudron requires '/' to be ext4 or xfs" # see #364
     exit 1
 fi
 

scripts/init-ubuntu.sh

@@ -117,7 +117,12 @@ if [[ "${ubuntu_version}" == "22.04" ]]; then
     cd /tmp && apt install -y --no-install-recommends ./libcollectdclient1_5.12.0-9_amd64.deb ./collectd-core_5.12.0-9_amd64.deb ./collectd_5.12.0-9_amd64.deb ./collectd-utils_5.12.0-9_amd64.deb && rm -f /tmp/collectd_*.deb
     echo -e "\nLD_PRELOAD=/usr/lib/python3.10/config-3.10-x86_64-linux-gnu/libpython3.10.so" >> /etc/default/collectd
 else
-    apt-get install -y --no-install-recommends collectd collectd-utils
+    if ! apt-get install -y --no-install-recommends collectd collectd-utils; then
+        # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
+        echo "Failed to install collectd, continuing anyway. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
+        sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
+    fi
+
     if [[ "${ubuntu_version}" == "20.04" ]]; then
         echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
     fi
@@ -139,7 +144,7 @@ sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See h
 
 # https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
 echo "==> Disabling motd news"
-if [ -f "/etc/default/motd-news" ]; then
+if [[ -f "/etc/default/motd-news" ]]; then
     sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
 fi
 

setup/start/cloudron-firewall.sh

@@ -77,8 +77,6 @@ if [[ -f "${ldap_allowlist_json}" ]]; then
     done < "${ldap_allowlist_json}"
 
     # ldap server we expose 3004 and also redirect from standard ldaps port 636
-    $iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
-
     $iptables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
     $iptables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist src -p tcp --dport 3004 -j ACCEPT
 
@@ -149,6 +147,7 @@ for port in 3306 5432 6379 27017; do
     $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
+# Add the rate limit chain to input chain
 $iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
 $ip6tables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $ip6tables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
 

setup/start/collectd/collectd.conf

@@ -13,7 +13,7 @@
 ##############################################################################
 
 Hostname "localhost"
-#FQDNLookup true
+FQDNLookup false
 #BaseDir "/var/lib/collectd"
 #PluginDir "/usr/lib/collectd"
 #TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db"
@@ -232,7 +232,7 @@ LoadPlugin swap
 
 <Plugin write_graphite>
    <Node "graphing">
-       Host "localhost"
+       Host "127.0.0.1"
        Port "2003"
        Protocol "tcp"
        LogSendErrors true

setup/start/collectd/df.py

@@ -14,7 +14,7 @@ def read():
     for d in disks:
         device = d[0]
         if 'devicemapper' in d[1] or not device.startswith('/dev/'): continue
-        instance = device[len('/dev/'):].replace('/', '_') # see #348
+        instance = device[len('/dev/'):].replace('/', '_').replace('.', '_') # see #348
 
         try:
             st = os.statvfs(d[1]) # handle disk removal

setup/start/sudoers

@@ -1,6 +1,9 @@
 # sudo logging breaks journalctl output with very long urls (systemd bug)
 Defaults !syslog
 
+Defaults!/home/yellowtent/box/src/scripts/checkvolume.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/checkvolume.sh
+
 Defaults!/home/yellowtent/box/src/scripts/clearvolume.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/clearvolume.sh
 

src/apps.js

@@ -42,7 +42,7 @@ exports = module.exports = {
     setMailbox,
     setInbox,
     setLocation,
-    setDataDir,
+    setStorage,
     repair,
 
     restore,
@@ -81,7 +81,7 @@ exports = module.exports = {
     schedulePendingTasks,
     restartAppsUsingAddons,
 
-    getDataDir,
+    getStorageDir,
     getIcon,
     getMemoryLimit,
     getLimits,
@@ -168,6 +168,7 @@ const appstore = require('./appstore.js'),
     semver = require('semver'),
     services = require('./services.js'),
     settings = require('./settings.js'),
+    shell = require('./shell.js'),
     spawn = require('child_process').spawn,
     split = require('split'),
     superagent = require('superagent'),
@@ -178,6 +179,7 @@ const appstore = require('./appstore.js'),
     util = require('util'),
     uuid = require('uuid'),
     validator = require('validator'),
+    volumes = require('./volumes.js'),
     _ = require('underscore');
 
 const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
@@ -185,11 +187,13 @@ const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationS
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson', 'apps.operatorsJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', 'apps.crontab',
     'apps.creationTime', 'apps.updateTime', 'apps.enableAutomaticUpdate',
-    'apps.enableMailbox', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain',
-    'apps.dataDir', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
+    'apps.enableMailbox', 'apps.mailboxDisplayName', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain',
+    'apps.storageVolumeId', 'apps.storageVolumePrefix', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
 
 // const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
 
+const CHECKVOLUME_CMD = path.join(__dirname, 'scripts/checkvolume.sh');
+
 function validatePortBindings(portBindings, manifest) {
     assert.strictEqual(typeof portBindings, 'object');
     assert.strictEqual(typeof manifest, 'object');
@@ -477,28 +481,29 @@ function validateEnv(env) {
     return null;
 }
 
-function validateDataDir(dataDir) {
-    if (dataDir === null) return null;
+async function checkStorage(app, volumeId, prefix) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof volumeId, 'string');
+    assert.strictEqual(typeof prefix, 'string');
 
-    if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`);
-    if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`);
-    if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`);
+    const volume = await volumes.get(volumeId);
+    if (volume === null) throw new BoxError(BoxError.BAD_FIELD, 'Storage volume not found');
 
-    // nfs shares will have the directory mounted already
-    let stat = safe.fs.lstatSync(dataDir);
-    if (stat) {
-        if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`);
-        let entries = safe.fs.readdirSync(dataDir);
-        if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`);
-        if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`);
-    }
+    const status = await volumes.getStatus(volume);
+    if (status.state !== 'active') throw new BoxError(BoxError.BAD_FIELD, 'Volume is not active');
 
-    // backup logic relies on paths not overlapping (because it recurses)
-    if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`);
+    if (path.isAbsolute(prefix)) throw new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" must be a relative path`);
+    if (prefix.endsWith('/')) throw new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" contains trailing slash`);
+    if (prefix !== '' && path.normalize(prefix) !== prefix) throw new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" is not a normalized path`);
 
-    // if we made it this far, it cannot start with any of these realistically
-    const fhs = [ '/bin', '/boot', '/etc', '/lib', '/lib32', '/lib64', '/proc', '/run', '/sbin', '/tmp', '/usr' ];
-    if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`);
+    const sourceDir = await getStorageDir(app);
+    const targetDir = path.join(volume.hostPath, prefix);
+    const rel = path.relative(sourceDir, targetDir);
+    if (!rel.startsWith('../') && rel.split('/').length > 1) throw new BoxError(BoxError.BAD_FIELD, 'Only one level subdirectory moves are supported');
+
+    const [error] = await safe(shell.promises.sudo('checkStorage', [ CHECKVOLUME_CMD, targetDir, sourceDir ], {}));
+    if (error && error.code === 2) throw new BoxError(BoxError.BAD_FIELD, `Target directory ${targetDir} is not empty`);
+    if (error && error.code === 3) throw new BoxError(BoxError.BAD_FIELD, `Target directory ${targetDir} does not support chown`);
 
     return null;
 }
@@ -530,17 +535,20 @@ function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, port
         if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is in use`);
     }
 
-    if (match[2] === 'dataDir') {
-        return new BoxError(BoxError.BAD_FIELD, `Data directory ${match[1]} is in use`);
+    if (match[2] === 'apps_storageVolume') {
+        return new BoxError(BoxError.BAD_FIELD, `Storage directory ${match[1]} is in use`);
     }
 
     return new BoxError(BoxError.ALREADY_EXISTS, `${match[2]} '${match[1]}' is in use`);
 }
 
-function getDataDir(app, dataDir) {
-    assert(dataDir === null || typeof dataDir === 'string');
+async function getStorageDir(app) {
+    assert.strictEqual(typeof app, 'object');
 
-    return dataDir || path.join(paths.APPS_DATA_DIR, app.id, 'data');
+    if (!app.storageVolumeId) return path.join(paths.APPS_DATA_DIR, app.id, 'data');
+    const volume = await volumes.get(app.storageVolumeId);
+    if (!volume) throw new BoxError(BoxError.NOT_FOUND, 'Volume not found'); // not possible
+    return path.join(volume.hostPath, app.storageVolumePrefix);
 }
 
 function removeInternalFields(app) {
@@ -549,8 +557,8 @@ function removeInternalFields(app) {
         'subdomain', 'domain', 'fqdn', 'crontab',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'operators',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts',
-        'enableMailbox', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
+        'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'storageVolumeId', 'storageVolumePrefix', 'mounts',
+        'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
 }
 
 // non-admins can only see these
@@ -779,6 +787,7 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
         tagsJson = data.tags ? JSON.stringify(data.tags) : null,
         mailboxName = data.mailboxName || null,
         mailboxDomain = data.mailboxDomain || null,
+        mailboxDisplayName = data.mailboxDisplayName || '',
         reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null,
         servicesConfigJson = data.servicesConfig ? JSON.stringify(data.servicesConfig) : null,
         enableMailbox = data.enableMailbox || false,
@@ -789,10 +798,11 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
     queries.push({
         query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
             + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, '
-            + 'enableMailbox) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+            + 'enableMailbox, mailboxDisplayName) '
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
         args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
-            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox ]
+            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon,
+            enableMailbox, mailboxDisplayName ]
     });
 
     queries.push({
@@ -1593,6 +1603,7 @@ async function setMailbox(app, data, auditSource) {
     const optional = 'optional' in app.manifest.addons.sendmail ? app.manifest.addons.sendmail.optional : false;
     if (!optional && !enableMailbox) throw new BoxError(BoxError.BAD_FIELD, 'App requires sendmail to be enabled');
 
+    const mailboxDisplayName = data.mailboxDisplayName || '';
     let mailboxName = data.mailboxName || null;
     const mailboxDomain = data.mailboxDomain || null;
 
@@ -1605,15 +1616,20 @@ async function setMailbox(app, data, auditSource) {
         } else {
             mailboxName = mailboxNameForSubdomain(app.subdomain, app.domain, app.manifest);
         }
+
+        if (mailboxDisplayName) {
+            error = mail.validateDisplayName(mailboxDisplayName);
+            if (error) throw new BoxError(BoxError.BAD_FIELD, error.message);
+        }
     }
 
     const task = {
         args: {},
-        values: { enableMailbox, mailboxName, mailboxDomain }
+        values: { enableMailbox, mailboxName, mailboxDomain, mailboxDisplayName }
     };
     const taskId = await addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task, auditSource);
 
-    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mailboxName, mailboxDomain, taskId });
+    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, mailboxName, mailboxDomain, mailboxDisplayName, taskId });
 
     return { taskId };
 }
@@ -1786,25 +1802,29 @@ async function setLocation(app, data, auditSource) {
     return { taskId };
 }
 
-async function setDataDir(app, dataDir, auditSource) {
+async function setStorage(app, volumeId, volumePrefix, auditSource) {
     assert.strictEqual(typeof app, 'object');
-    assert(dataDir === null || typeof dataDir === 'string');
+    assert(volumeId === null || typeof volumeId === 'string');
+    assert(volumePrefix === null || typeof volumePrefix === 'string');
     assert.strictEqual(typeof auditSource, 'object');
 
     const appId = app.id;
     let error = checkAppState(app, exports.ISTATE_PENDING_DATA_DIR_MIGRATION);
     if (error) throw error;
 
-    error = validateDataDir(dataDir);
-    if (error) throw error;
+    if (volumeId) {
+        await checkStorage(app, volumeId, volumePrefix);
+    } else {
+        volumeId = volumePrefix = null;
+    }
 
     const task = {
-        args: { newDataDir: dataDir },
+        args: { newStorageVolumeId: volumeId, newStorageVolumePrefix: volumePrefix },
         values: {}
     };
     const taskId = await addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, auditSource);
 
-    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, dataDir, taskId });
+    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, volumeId, volumePrefix, taskId });
 
     return { taskId };
 }
@@ -2235,7 +2255,8 @@ async function clone(app, data, user, auditSource) {
         tags: app.tags,
         enableAutomaticUpdate: app.enableAutomaticUpdate,
         icon: icons.icon,
-        enableMailbox: app.enableMailbox
+        enableMailbox: app.enableMailbox,
+        mailboxDisplayName: app.mailboxDisplayName
     };
 
     const [addError] = await safe(add(newAppId, appStoreId, manifest, subdomain, domain, translatePortBindings(portBindings, manifest), obj));

src/appstore.js

@@ -89,8 +89,8 @@ async function login(email, password, totpToken) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${response.status}`);
-    if (!response.body.accessToken) throw new BoxError(BoxError.EXTERNAL_ERROR, `login invalid response: ${response.text}`);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. status code: ${response.status}`);
+    if (!response.body.accessToken) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. invalid response: ${response.text}`);
 
     return response.body; // { userId, accessToken }
 }
@@ -105,8 +105,8 @@ async function registerUser(email, password) {
         .ok(() => true));
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, 'account already exists');
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${response.status}`);
+    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, 'Registration error: account already exists');
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Registration error. invalid response: ${response.status}`);
 }
 
 async function getWebToken() {
@@ -129,7 +129,6 @@ async function getSubscription() {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR);
     if (response.status === 502) throw new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`);
 
@@ -185,8 +184,7 @@ async function unpurchaseApp(appId, data) {
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 404) return;   // was never purchased
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed to get app. status:${response.status}`);
 
     [error, response] = await safe(superagent.del(url)
         .send(data)
@@ -196,7 +194,7 @@ async function unpurchaseApp(appId, data) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
+    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed. status:${response.status}`);
 }
 
 async function getBoxUpdate(options) {
@@ -218,7 +216,6 @@ async function getBoxUpdate(options) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status === 204) return; // no update
     if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
@@ -261,7 +258,6 @@ async function getAppUpdate(app, options) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status === 204) return; // no update
     if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
@@ -285,10 +281,10 @@ async function getAppUpdate(app, options) {
 async function registerCloudron(data) {
     assert.strictEqual(typeof data, 'object');
 
-    const { domain, accessToken, version } = data;
+    const { domain, accessToken, version, existingApps } = data;
 
     const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/register_cloudron`)
-        .send({ domain, accessToken, version })
+        .send({ domain, accessToken, version, existingApps })
         .timeout(30 * 1000)
         .ok(() => true));
 
@@ -326,7 +322,6 @@ async function updateCloudron(data) {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
     debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
@@ -335,13 +330,14 @@ async function updateCloudron(data) {
 async function registerWithLoginCredentials(options) {
     assert.strictEqual(typeof options, 'object');
 
-    const token = await settings.getAppstoreApiToken();
-    if (token) throw new BoxError(BoxError.CONFLICT, 'Cloudron is already registered');
-
     if (options.signup) await registerUser(options.email, options.password);
 
     const result = await login(options.email, options.password, options.totpToken || '');
     await registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION });
+
+    for (const app of await apps.list()) {
+        await purchaseApp({ appId: app.id, appstoreId: app.appStoreId, manifestId: app.manifest.id || 'customapp' });
+    }
 }
 
 async function unregister() {
@@ -391,7 +387,6 @@ async function createTicket(info, auditSource) {
     const [error, response] = await safe(request);
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
     await eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
@@ -412,7 +407,6 @@ async function getApps() {
 
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 403 || response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', response.status, response.body));
     if (!response.body.apps) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
@@ -443,7 +437,6 @@ async function getAppVersion(appId, version) {
     if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
     if (response.status === 403 || response.statusCode === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
     if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
     if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', response.status, response.body));
 
     return response.body;

src/apptask.js

@@ -160,7 +160,8 @@ async function deleteAppDir(app, options) {
 async function addCollectdProfile(app) {
     assert.strictEqual(typeof app, 'object');
 
-    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir: apps.getDataDir(app, app.dataDir) });
+    const appDataDir = await apps.getStorageDir(app);
+    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir });
     await collectd.addProfile(app.id, collectdConf);
 }
 
@@ -268,19 +269,22 @@ async function waitForDnsPropagation(app) {
     }
 }
 
-async function moveDataDir(app, targetDir) {
+async function moveDataDir(app, targetVolumeId, targetVolumePrefix) {
     assert.strictEqual(typeof app, 'object');
-    assert(targetDir === null || typeof targetDir === 'string');
+    assert(targetVolumeId === null || typeof targetVolumeId === 'string');
+    assert(targetVolumePrefix === null || typeof targetVolumePrefix === 'string');
 
-    const resolvedSourceDir = apps.getDataDir(app, app.dataDir);
-    const resolvedTargetDir = apps.getDataDir(app, targetDir);
+    const resolvedSourceDir = await apps.getStorageDir(app);
+    const resolvedTargetDir = await apps.getStorageDir(_.extend({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix }));
 
     debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
 
-    if (resolvedSourceDir === resolvedTargetDir) return;
+    if (resolvedSourceDir !== resolvedTargetDir) {
+        const [error] = await safe(shell.promises.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
+        if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`);
+    }
 
-    const [error] = await safe(shell.promises.sudo('moveDataDir', [ MV_VOLUME_CMD, resolvedSourceDir, resolvedTargetDir ], {}));
-    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `Error migrating data directory: ${error.message}`);
+    await updateApp(app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix });
 }
 
 async function downloadImage(manifest) {
@@ -520,8 +524,9 @@ async function migrateDataDir(app, args, progressCallback) {
     assert.strictEqual(typeof args, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    const newDataDir = args.newDataDir;
-    assert(newDataDir === null || typeof newDataDir === 'string');
+    const { newStorageVolumeId, newStorageVolumePrefix } = args;
+    assert(newStorageVolumeId === null || typeof newStorageVolumeId === 'string');
+    assert(newStorageVolumePrefix === null || typeof newStorageVolumePrefix === 'string');
 
     await progressCallback({ percent: 10, message: 'Cleaning up old install' });
     await deleteContainers(app, { managedOnly: true });
@@ -529,12 +534,12 @@ async function migrateDataDir(app, args, progressCallback) {
     await progressCallback({ percent: 45, message: 'Ensuring app data directory' });
     await createAppDir(app);
 
-    // re-setup addons since this creates the localStorage volume
+    // re-setup addons since this creates the localStorage destination
     await progressCallback({ percent: 50, message: 'Setting up addons' });
-    await services.setupAddons(_.extend({}, app, { dataDir: newDataDir }), app.manifest.addons);
+    await services.setupAddons(_.extend({}, app, { storageVolumeId: newStorageVolumeId, storageVolumePrefix: newStorageVolumePrefix }), app.manifest.addons);
 
     await progressCallback({ percent: 60, message: 'Moving data dir' });
-    await moveDataDir(app, newDataDir);
+    await moveDataDir(app, newStorageVolumeId, newStorageVolumePrefix);
 
     await progressCallback({ percent: 90, message: 'Creating container' });
     await createContainer(app);
@@ -542,7 +547,7 @@ async function migrateDataDir(app, args, progressCallback) {
     await startApp(app);
 
     await progressCallback({ percent: 100, message: 'Done' });
-    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir });
+    await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
 }
 
 // configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local"

src/backuptask.js

@@ -102,7 +102,7 @@ async function downloadApp(app, restoreConfig, progressCallback) {
 
     const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
     if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
-    const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
+    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);
 
     const startTime = new Date();
     const backupConfig = restoreConfig.backupConfig || await settings.getBackupConfig();
@@ -318,7 +318,7 @@ async function uploadAppSnapshot(backupConfig, app, progressCallback) {
     const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
     if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`);
 
-    const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
+    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);
 
     progressCallback({ message: `Uploading app snapshot ${app.fqdn}`});
 

src/constants.js

@@ -72,6 +72,6 @@ exports = module.exports = {
 
     FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
 
-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '7.0.0-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '7.2.0-test'
 };
 

src/database.js

@@ -61,8 +61,9 @@ async function initialize() {
         // note the pool also has an 'acquire' event but that is called whenever we do a getConnection()
         connection.on('error', (error) => debug(`Connection ${connection.threadId} error: ${error.message} ${error.code}`));
 
-        connection.query('USE ' + gDatabase.name);
+        connection.query(`USE ${gDatabase.name}`);
         connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
+        connection.query('SET SESSION group_concat_max_len = 65536'); // GROUP_CONCAT has only 1024 default
     });
 }
 

src/docker.js

@@ -23,10 +23,6 @@ exports = module.exports = {
     getEvents,
     memoryUsage,
 
-    createVolume,
-    removeVolume,
-    clearVolume,
-
     update,
 
     createExec,
@@ -42,7 +38,6 @@ const apps = require('./apps.js'),
     debug = require('debug')('box:docker'),
     delay = require('./delay.js'),
     Docker = require('dockerode'),
-    path = require('path'),
     reverseProxy = require('./reverseproxy.js'),
     services = require('./services.js'),
     settings = require('./settings.js'),
@@ -52,9 +47,6 @@ const apps = require('./apps.js'),
     volumes = require('./volumes.js'),
     _ = require('underscore');
 
-const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
-    MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');
-
 const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
 const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
 
@@ -200,15 +192,17 @@ async function getAddonMounts(app) {
 
     for (const addon of Object.keys(addons)) {
         switch (addon) {
-        case 'localstorage':
+        case 'localstorage': {
+            const storageDir = await apps.getStorageDir(app);
             mounts.push({
                 Target: '/app/data',
-                Source: `${app.id}-localstorage`,
-                Type: 'volume',
+                Source: storageDir,
+                Type: 'bind',
                 ReadOnly: false
             });
 
             break;
+        }
         case 'tls': {
             const bundle = await reverseProxy.getCertificatePath(app.fqdn, app.domain);
 
@@ -630,53 +624,6 @@ async function memoryUsage(containerId) {
     return result;
 }
 
-async function createVolume(name, volumeDataDir, labels) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof volumeDataDir, 'string');
-    assert.strictEqual(typeof labels, 'object');
-
-    const volumeOptions = {
-        Name: name,
-        Driver: 'local',
-        DriverOpts: { // https://github.com/moby/moby/issues/19990#issuecomment-248955005
-            type: 'none',
-            device: volumeDataDir,
-            o: 'bind'
-        },
-        Labels: labels
-    };
-
-    // requires sudo because the path can be outside appsdata
-    let [error] = await safe(shell.promises.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}));
-    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`);
-
-    [error] = await safe(gConnection.createVolume(volumeOptions));
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
-}
-
-async function clearVolume(name, options) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof options, 'object');
-
-    let volume = gConnection.getVolume(name);
-    let [error, v] = await safe(volume.inspect());
-    if (error && error.statusCode === 404) return;
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, error);
-
-    const volumeDataDir = v.Options.device;
-    [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, options.removeDirectory ? 'rmdir' : 'clear', volumeDataDir ], {});
-    if (error) throw new BoxError(BoxError.FS_ERROR, error);
-}
-
-// this only removes the volume and not the data
-async function removeVolume(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    let volume = gConnection.getVolume(name);
-    const [error] = await safe(volume.remove());
-    if (error && error.statusCode !== 404) throw new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume: ${error.message}`);
-}
-
 async function info() {
     const [error, result] = await safe(gConnection.info());
     if (error) throw new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker');

src/ldap.js

@@ -209,37 +209,6 @@ async function groupSearch(req, res, next) {
 
     const results = [];
 
-    // those are the old virtual groups for backwards compat
-    const virtualGroups = [{
-        name: 'users',
-        admin: false
-    }, {
-        name: 'admins',
-        admin: true
-    }];
-
-    virtualGroups.forEach(function (group) {
-        const dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-        const members = group.admin ? usersWithAccess.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : usersWithAccess;
-
-        const obj = {
-            dn: dn.toString(),
-            attributes: {
-                objectclass: ['group'],
-                cn: group.name,
-                memberuid: members.map(function(entry) { return entry.id; }).sort()
-            }
-        };
-
-        // ensure all filter values are also lowercase
-        const lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
-        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
-        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-            results.push(obj);
-        }
-    });
-
     let [errorGroups, resultGroups] = await safe(groups.listWithMembers());
     if (errorGroups) return next(new ldap.OperationsError(errorGroups.toString()));
 
@@ -558,7 +527,7 @@ async function userSearchSftp(req, res, next) {
     const obj = {
         dn: ldap.parseDN(`cn=${username}@${appFqdn},ou=sftp,dc=cloudron`).toString(),
         attributes: {
-            homeDirectory: app.dataDir ? `/mnt/app-${app.id}` : `/mnt/appsdata/${app.id}/data`, // see also sftp.js
+            homeDirectory: app.storageVolumeId ? `/mnt/app-${app.id}` : `/mnt/appsdata/${app.id}/data`, // see also sftp.js
             objectclass: ['user'],
             objectcategory: 'person',
             cn: user.id,

src/mail.js

@@ -22,6 +22,7 @@ exports = module.exports = {
     setDnsRecords,
 
     validateName,
+    validateDisplayName,
 
     setMailFromValidation,
     setCatchAllAddress,
@@ -65,7 +66,6 @@ exports = module.exports = {
     TYPE_LIST: 'list',
     TYPE_ALIAS: 'alias',
 
-    _validateName: validateName,
     _delByDomain: delByDomain,
     _updateDomain: updateDomain
 };
@@ -169,6 +169,16 @@ function validateName(name) {
     return null;
 }
 
+function validateDisplayName(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name must be atleast 1 char');
+    if (name.length >= 100) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name too long');
+    if (/["<>@]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name is not valid');
+
+    return null;
+}
+
 async function checkOutboundPort25() {
     const relay = {
         value: 'OK',

src/mounts.js

@@ -54,6 +54,7 @@ function validateMountOptions(type, options) {
         if (typeof options.remoteDir !== 'string') return new BoxError(BoxError.BAD_FIELD, 'remoteDir is not a string');
         return null;
     case 'ext4':
+    case 'xfs':
         if (typeof options.diskPath !== 'string') return new BoxError(BoxError.BAD_FIELD, 'diskPath is not a string');
         return null;
     default:
@@ -62,7 +63,7 @@ function validateMountOptions(type, options) {
 }
 
 function isManagedProvider(provider) {
-    return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4';
+    return provider === 'sshfs' || provider === 'cifs' || provider === 'nfs' || provider === 'ext4' || provider === 'xfs';
 }
 
 function mountObjectFromBackupConfig(backupConfig) {
@@ -108,6 +109,11 @@ function renderMountFile(mount) {
         what = mountOptions.diskPath; // like /dev/disk/by-uuid/uuid or /dev/disk/by-id/scsi-id
         options = 'discard,defaults,noatime';
         break;
+    case 'xfs':
+        type = 'xfs';
+        what = mountOptions.diskPath; // like /dev/disk/by-uuid/uuid or /dev/disk/by-id/scsi-id
+        options = 'discard,defaults,noatime';
+        break;
     case 'sshfs': {
         const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${mountOptions.host}`);
         if (!safe.fs.writeFileSync(keyFilePath, `${mount.mountOptions.privateKey}\n`, { mode: 0o600 })) throw new BoxError(BoxError.FS_ERROR, `Could not write private key: ${safe.error.message}`);

src/routes/apps.js

@@ -35,7 +35,7 @@ exports = module.exports = {
     setMailbox,
     setInbox,
     setLocation,
-    setDataDir,
+    setStorage,
     setMounts,
 
     stop,
@@ -373,6 +373,7 @@ async function setMailbox(req, res, next) {
     if (req.body.enable) {
         if (req.body.mailboxName !== null && typeof req.body.mailboxName !== 'string') return next(new HttpError(400, 'mailboxName must be a string'));
         if (typeof req.body.mailboxDomain !== 'string') return next(new HttpError(400, 'mailboxDomain must be a string'));
+        if ('mailboxDisplayName' in req.body && typeof req.body.mailboxDisplayName !== 'string') return next(new HttpError(400, 'mailboxDisplayName must be a string'));
     }
 
     const [error, result] = await safe(apps.setMailbox(req.app, req.body, AuditSource.fromRequest(req)));
@@ -431,13 +432,18 @@ async function setLocation(req, res, next) {
     next(new HttpSuccess(202, { taskId: result.taskId }));
 }
 
-async function setDataDir(req, res, next) {
+async function setStorage(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.app, 'object');
 
-    if (req.body.dataDir !== null && typeof req.body.dataDir !== 'string') return next(new HttpError(400, 'dataDir must be a string'));
+    const { storageVolumeId, storageVolumePrefix } = req.body;
 
-    const [error, result] = await safe(apps.setDataDir(req.app, req.body.dataDir, AuditSource.fromRequest(req)));
+    if (storageVolumeId !== null) {
+        if (typeof storageVolumeId !== 'string') return next(new HttpError(400, 'storageVolumeId must be a string'));
+        if (typeof storageVolumePrefix !== 'string') return next(new HttpError(400, 'storageVolumePrefix must be a string'));
+    }
+
+    const [error, result] = await safe(apps.setStorage(req.app, storageVolumeId, storageVolumePrefix, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(202, { taskId: result.taskId }));

src/routes/test/users-test.js

@@ -640,15 +640,5 @@ describe('Users API', function () {
             expect(response.statusCode).to.equal(409);
         });
     });
-
-    describe('transfer ownership', function () {
-        it('succeeds', async function () {
-            const response = await superagent.post(`${serverUrl}/api/v1/users/${user.id}/make_owner`)
-                .query({ access_token: owner.token })
-                .send({});
-
-            expect(response.statusCode).to.equal(204);
-        });
-    });
 });
 

src/routes/users.js

@@ -10,7 +10,6 @@ exports = module.exports = {
     verifyPassword,
     setGroups,
     setGhost,
-    makeOwner,
     makeLocal,
 
     getPasswordResetLink,
@@ -202,20 +201,6 @@ async function setPassword(req, res, next) {
     next(new HttpSuccess(204));
 }
 
-// This route transfers ownership from token user to user specified in path param
-async function makeOwner(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
-
-    // first make new one owner, then demote current one
-    let [error] = await safe(users.update(req.resource, { role: users.ROLE_OWNER }, AuditSource.fromRequest(req)));
-    if (error) return next(BoxError.toHttpError(error));
-
-    [error] = await safe(users.update(req.user, { role: users.ROLE_USER }, AuditSource.fromRequest(req)));
-    if (error) return next(BoxError.toHttpError(error));
-
-    next(new HttpSuccess(204));
-}
-
 async function makeLocal(req, res, next) {
     assert.strictEqual(typeof req.resource, 'object');
 

src/scripts/checkvolume.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
+if [[ $# -eq 0 ]]; then
+    echo "No arguments supplied"
+    exit 1
+fi
+
+if [[ "$1" == "--check" ]]; then
+    echo "OK"
+    exit 0
+fi
+
+target_dir="$1"
+source_dir="$2"
+
+source_stat=$(stat --format='%d,%i' "${source_dir}")
+target_stat=$(stat --format='%d,%i' "${target_dir}")
+
+# test sameness across bind mounts. if it's same, we can skip the emptiness check
+if [[ "${source_stat}" == "${target_stat}" ]]; then
+    echo "Source dir and target dir are the same"
+    exit 0
+fi
+
+readonly test_file="${target_dir}/.chown-test"
+
+mkdir -p "${target_dir}"
+rm -f "${test_file}" # clean up any from previous run
+
+if [[ -n $(ls -A "${target_dir}") ]]; then
+    echo "volume dir is not empty"
+    exit 2
+fi
+
+touch "${test_file}"
+if ! chown yellowtent:yellowtent "${test_file}"; then
+    echo "chown does not work"
+    exit 3
+fi
+rm -f "${test_file}"
+rm -r "${target_dir}" # will get recreated by the local storage addon
+

src/scripts/mkdirvolume.sh

@@ -20,4 +20,3 @@ fi
 volume_dir="$1"
 
 mkdir -p "${volume_dir}"
-

src/scripts/mvvolume.sh

@@ -26,8 +26,17 @@ if [[ "${BOX_ENV}" == "test" ]]; then
     [[ "${target_dir}" != *"/.cloudron_test/"* ]] && exit 1
 fi
 
+source_stat=$(stat --format='%d,%i' "${source_dir}")
+target_stat=$(stat --format='%d,%i' "${target_dir}")
+
+# test sameness across bind mounts
+if [[ "${source_stat}" == "${target_stat}" ]]; then
+    echo "Source dir and target dir are the same"
+    exit 0
+fi
+
 # copy and remove - this way if the copy fails, the original is intact
-# the find logic is so that move to a subdir works (and we also move hidden files)
+# the find logic is so that move to a one level subdir works (and we also move hidden files)
 find "${source_dir}" -maxdepth 1 -mindepth 1 -not -wholename "${target_dir}" -exec cp -ar '{}' "${target_dir}" \;
 find "${source_dir}" -maxdepth 1 -mindepth 1 -not -wholename "${target_dir}" -exec rm -rf '{}' \;
 # this will fail if target is a subdir or if source is a mountpoint

src/server.js

@@ -179,7 +179,6 @@ function initializeExpressSync() {
     router.post('/api/v1/users/:userId/password',      json, token, authorizeUserManager, routes.users.load, routes.users.setPassword);
     router.post('/api/v1/users/:userId/ghost',         json, token, authorizeAdmin,       routes.users.load, routes.users.setGhost);
     router.put ('/api/v1/users/:userId/groups',        json, token, authorizeUserManager, routes.users.load, routes.users.setGroups);
-    router.post('/api/v1/users/:userId/make_owner',    json, token, authorizeOwner,       routes.users.load, routes.users.makeOwner);
     router.post('/api/v1/users/:userId/twofactorauthentication_disable', json, token, authorizeUserManager, routes.users.load, routes.users.disableTwoFactorAuthentication);
     router.get ('/api/v1/users/:userId/password_reset_link',   json, token, authorizeUserManager, routes.users.load, routes.users.getPasswordResetLink);
     router.post('/api/v1/users/:userId/send_password_reset_email',   json, token, authorizeUserManager, routes.users.load, routes.users.sendPasswordResetEmail);
@@ -224,7 +223,7 @@ function initializeExpressSync() {
     router.post('/api/v1/apps/:id/configure/mailbox',            json, token, routes.apps.load, authorizeAdmin, routes.apps.setMailbox);
     router.post('/api/v1/apps/:id/configure/inbox',              json, token, routes.apps.load, authorizeAdmin, routes.apps.setInbox);
     router.post('/api/v1/apps/:id/configure/env',                json, token, routes.apps.load, authorizeOperator, routes.apps.setEnvironment);
-    router.post('/api/v1/apps/:id/configure/data_dir',           json, token, routes.apps.load, authorizeAdmin, routes.apps.setDataDir);
+    router.post('/api/v1/apps/:id/configure/storage',            json, token, routes.apps.load, authorizeAdmin, routes.apps.setStorage);
     router.post('/api/v1/apps/:id/configure/location',           json, token, routes.apps.load, authorizeAdmin, routes.apps.setLocation);
     router.post('/api/v1/apps/:id/configure/mounts',             json, token, routes.apps.load, authorizeAdmin, routes.apps.setMounts);
     router.post('/api/v1/apps/:id/configure/crontab',            json, token, routes.apps.load, authorizeOperator, routes.apps.setCrontab);

src/services.js

@@ -63,6 +63,8 @@ const addonConfigs = require('./addonconfigs.js'),
 const NOOP = async function (/*app, options*/) {};
 const RMADDONDIR_CMD = path.join(__dirname, 'scripts/rmaddondir.sh');
 const RESTART_SERVICE_CMD = path.join(__dirname, 'scripts/restartservice.sh');
+const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh');
+const MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh');
 
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
@@ -856,11 +858,10 @@ async function setupLocalStorage(app, options) {
 
     debug('setupLocalStorage');
 
-    const volumeDataDir = apps.getDataDir(app, app.dataDir);
+    const volumeDataDir = await apps.getStorageDir(app);
 
-    // reomve any existing volume in case it's bound with an old dataDir
-    await docker.removeVolume(`${app.id}-localstorage`);
-    await docker.createVolume(`${app.id}-localstorage`, volumeDataDir, { fqdn: app.fqdn, appId: app.id });
+    const [error] = await safe(shell.promises.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}));
+    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating app storage data dir: ${error.message}`);
 }
 
 async function clearLocalStorage(app, options) {
@@ -869,7 +870,9 @@ async function clearLocalStorage(app, options) {
 
     debug('clearLocalStorage');
 
-    await docker.clearVolume(`${app.id}-localstorage`, { removeDirectory: false });
+    const volumeDataDir = await apps.getStorageDir(app);
+    const [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, 'clear', volumeDataDir ], {});
+    if (error) throw new BoxError(BoxError.FS_ERROR, error);
 }
 
 async function teardownLocalStorage(app, options) {
@@ -878,8 +881,9 @@ async function teardownLocalStorage(app, options) {
 
     debug('teardownLocalStorage');
 
-    await docker.clearVolume(`${app.id}-localstorage`, { removeDirectory: true });
-    await docker.removeVolume(`${app.id}-localstorage`);
+    const volumeDataDir = await apps.getStorageDir(app);
+    const [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, 'rmdir', volumeDataDir ], {});
+    if (error) throw new BoxError(BoxError.FS_ERROR, error);
 }
 
 async function setupTurn(app, options) {
@@ -1048,6 +1052,9 @@ async function setupSendMail(app, options) {
         { name: 'CLOUDRON_MAIL_FROM', value: app.mailboxName + '@' + app.mailboxDomain },
         { name: 'CLOUDRON_MAIL_DOMAIN', value: app.mailboxDomain }
     ];
+
+    if (app.manifest.addons.sendmail.supportsDisplayName) env.push({ name: 'CLOUDRON_MAIL_FROM_DISPLAY_NAME', value: app.mailboxDisplayName });
+
     debug('Setting sendmail addon config to %j', env);
     await addonConfigs.set(app.id, 'sendmail', env);
 }

src/sftp.js

@@ -62,9 +62,9 @@ async function start(existingInfra) {
     // custom app data directories
     const allApps = await apps.list();
     for (const app of allApps) {
-        if (!app.manifest.addons['localstorage'] || !app.dataDir) continue;
+        if (!app.manifest.addons['localstorage'] || !app.storageVolumeId) continue;
 
-        const hostDir = apps.getDataDir(app, app.dataDir), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp
+        const hostDir = await apps.getStorageDir(app), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp
         if (!safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user
             // do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail
             debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`);

src/storage/filesystem.js

@@ -26,6 +26,7 @@ const PROVIDER_FILESYSTEM = 'filesystem';
 const PROVIDER_MOUNTPOINT = 'mountpoint';
 const PROVIDER_SSHFS = 'sshfs';
 const PROVIDER_CIFS = 'cifs';
+const PROVIDER_XFS = 'xfs';
 const PROVIDER_NFS = 'nfs';
 const PROVIDER_EXT4 = 'ext4';
 
@@ -52,6 +53,7 @@ function getRootPath(apiConfig) {
     case PROVIDER_NFS:
     case PROVIDER_CIFS:
     case PROVIDER_EXT4:
+    case PROVIDER_XFS:
         return path.join(paths.MANAGED_BACKUP_MOUNT_DIR, apiConfig.prefix);
     case PROVIDER_MOUNTPOINT:
         return path.join(apiConfig.mountPoint, apiConfig.prefix);
@@ -89,7 +91,7 @@ async function checkPreconditions(apiConfig, dataLayout) {
     if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`);
 
     // Check filesystem is mounted so we don't write into the actual folder on disk
-    if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS || apiConfig.provider === PROVIDER_EXT4) {
+    if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS || apiConfig.provider === PROVIDER_EXT4 || apiConfig.provider === PROVIDER_XFS) {
         if (result.mountpoint !== paths.MANAGED_BACKUP_MOUNT_DIR) throw new BoxError(BoxError.FS_ERROR, 'Backup target is not mounted');
     } else if (apiConfig.provider === PROVIDER_MOUNTPOINT) {
         if (result.mountpoint === '/') throw new BoxError(BoxError.FS_ERROR, `${apiConfig.backupFolder} is not mounted`);
@@ -103,6 +105,7 @@ function hasChownSupportSync(apiConfig) {
     switch (apiConfig.provider) {
     case PROVIDER_NFS:
     case PROVIDER_EXT4:
+    case PROVIDER_XFS:
     case PROVIDER_FILESYSTEM:
         return true;
     case PROVIDER_SSHFS:
@@ -287,7 +290,7 @@ async function testConfig(apiConfig) {
         if (!apiConfig.backupFolder || typeof apiConfig.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
         const error = validateBackupTarget(apiConfig.backupFolder);
         if (error) throw error;
-    } else { // cifs/ext4/nfs/mountpoint/sshfs
+    } else { // xfs/cifs/ext4/nfs/mountpoint/sshfs
         if (apiConfig.provider === PROVIDER_MOUNTPOINT) {
             if (!apiConfig.mountPoint || typeof apiConfig.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
             const error = validateBackupTarget(apiConfig.mountPoint);

src/system.js

@@ -41,10 +41,11 @@ async function getAppDisks(appsDataDisk) {
 
     const allApps = await apps.list();
     for (const app of allApps) {
-        if (!app.dataDir) {
+        if (!app.storageVolumeId) {
             appDisks[app.id] = appsDataDisk;
         } else {
-            const [error, result] = await safe(df.file(app.dataDir));
+            const dataDir = await apps.getStorageDir(app);
+            const [error, result] = await safe(df.file(dataDir));
             appDisks[app.id] = error ? appsDataDisk : result.filesystem; // ignore any errors
         }
     }
@@ -68,7 +69,7 @@ async function getDisks() {
     if (error) throw new BoxError(BoxError.FS_ERROR, error);
 
     // filter by ext4 and then sort to make sure root disk is first
-    const ext4Disks = allDisks.filter((r) => r.type === 'ext4').sort((a, b) => a.mountpoint.localeCompare(b.mountpoint));
+    const ext4Disks = allDisks.filter((r) => r.type === 'ext4' || r.type === 'xfs').sort((a, b) => a.mountpoint.localeCompare(b.mountpoint));
 
     const diskInfos = [];
     for (const p of [ paths.BOX_DATA_DIR, paths.MAIL_DATA_DIR, paths.PLATFORM_DATA_DIR, paths.APPS_DATA_DIR, info.DockerRootDir ]) {
@@ -80,7 +81,7 @@ async function getDisks() {
     const backupsFilesystem = await getBackupsFilesystem();
 
     const result = {
-        disks: ext4Disks, // root disk is first. { filesystem, type, size, used, avialable, capacity, mountpoint }
+        disks: ext4Disks, // root disk is first. { filesystem, type, size, used, available, capacity, mountpoint }
         boxDataDisk: diskInfos[0].filesystem,
         mailDataDisk: diskInfos[1].filesystem,
         platformDataDisk: diskInfos[2].filesystem,

src/test/ldap-test.js

@@ -221,117 +221,78 @@ describe('Ldap', function () {
             mockApp.accessRestriction = null;
 
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: 'objectclass=group' });
-            expect(entries.length).to.equal(4);
-
-            // ensure order for testability
-            entries.sort(function (a, b) { return a.cn < b.cn; });
-
-            expect(entries[0].cn).to.equal('users');
-            expect(entries[0].memberuid.length).to.equal(2);
-            expect(entries[0].memberuid).to.contain(admin.id);
-            expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
-        });
-
-        it ('succeeds with cn wildcard filter', async function () {
-            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' });
-            expect(entries.length).to.equal(4);
-
-            // ensure order for testability
-            entries.sort(function (a, b) { return a.cn < b.cn; });
-
-            expect(entries[0].cn).to.equal('users');
-            expect(entries[0].memberuid.length).to.equal(2);
-            expect(entries[0].memberuid).to.contain(admin.id);
-            expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
-        });
-
-        it('succeeds with memberuid filter', async function () {
-            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(memberuid=' + user.id + ')' });
             expect(entries.length).to.equal(2);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
 
-            expect(entries[1].cn).to.equal('ldap-test-1');
-            expect(entries[1].memberuid.length).to.equal(2);
-            expect(entries[1].memberuid).to.contain(admin.id);
-            expect(entries[1].memberuid).to.contain(user.id);
+            expect(entries[1].cn).to.equal('ldap-test-2');
+            expect(entries[1].memberuid).to.equal(admin.id);
+        });
+
+        it ('succeeds with cn wildcard filter', async function () {
+            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' });
+            expect(entries.length).to.equal(2);
+
+            // ensure order for testability
+            entries.sort(function (a, b) { return a.cn < b.cn; });
+
+            expect(entries[0].cn).to.equal('ldap-test-1');
+            expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
+
+            expect(entries[1].cn).to.equal('ldap-test-2');
+            expect(entries[1].memberuid).to.equal(admin.id);
+        });
+
+        it('succeeds with memberuid filter', async function () {
+            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(memberuid=' + user.id + ')' });
+            expect(entries.length).to.equal(1);
+
+            // ensure order for testability
+            entries.sort(function (a, b) { return a.cn < b.cn; });
+
+            expect(entries[0].cn).to.equal('ldap-test-1');
+            expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
         });
 
         it ('does only list groups who have access', async function () {
             mockApp.accessRestriction = { users: [], groups: [ group.id ] };
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' });
+            expect(entries.length).to.equal(1);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries.length).to.equal(3);
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
             expect(entries[0].memberuid).to.contain(admin.id);
             expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
         });
 
         it ('succeeds with pagination', async function () {
             mockApp.accessRestriction = null;
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: 'objectclass=group', paged: true });
-            expect(entries.length).to.equal(4);
+            expect(entries.length).to.equal(2);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
             expect(entries[0].memberuid).to.contain(admin.id);
             expect(entries[0].memberuid).to.contain(user.id);
 
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
+            expect(entries[1].cn).to.equal('ldap-test-2');
             expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
         });
     });
 

src/test/mail-test.js

@@ -75,23 +75,43 @@ describe('Mail', function () {
 
     describe('mailbox name', function () {
         it('allows valid names', function () {
-            expect(mail._validateName('1')).to.be(null); // single char
-            expect(mail._validateName('ap')).to.be(null); // alpha
-            expect(mail._validateName('aP')).to.be(null); // caps
-            expect(mail._validateName('0P')).to.be(null); // number
-            expect(mail._validateName('a.p.x')).to.be(null); // dot
-            expect(mail._validateName('a-p-x')).to.be(null); // hyphen
-            expect(mail._validateName('a-p_x')).to.be(null); // underscore
+            expect(mail.validateName('1')).to.be(null); // single char
+            expect(mail.validateName('ap')).to.be(null); // alpha
+            expect(mail.validateName('aP')).to.be(null); // caps
+            expect(mail.validateName('0P')).to.be(null); // number
+            expect(mail.validateName('a.p.x')).to.be(null); // dot
+            expect(mail.validateName('a-p-x')).to.be(null); // hyphen
+            expect(mail.validateName('a-p_x')).to.be(null); // underscore
         });
 
         it('disallows invalid names', function () {
-            expect(mail._validateName('@')).to.be.an(Error);
-            expect(mail._validateName('a+p')).to.be.an(Error);
-            expect(mail._validateName('a#p')).to.be.an(Error);
-            expect(mail._validateName('a!')).to.be.an(Error);
+            expect(mail.validateName('@')).to.be.an(Error);
+            expect(mail.validateName('a+p')).to.be.an(Error);
+            expect(mail.validateName('a#p')).to.be.an(Error);
+            expect(mail.validateName('a!')).to.be.an(Error);
         });
     });
 
+    describe('mailbox display name', function () {
+        it('allows valid names', function () {
+            expect(mail.validateDisplayName('1')).to.be(null); // single char
+            expect(mail.validateDisplayName('ap')).to.be(null); // alpha
+            expect(mail.validateDisplayName('aP')).to.be(null); // caps
+            expect(mail.validateDisplayName('0P')).to.be(null); // number
+            expect(mail.validateDisplayName('a p.x')).to.be(null); // space
+            expect(mail.validateDisplayName('a-p-x')).to.be(null); // hyphen
+            expect(mail.validateDisplayName('a-p_x')).to.be(null); // underscore
+        });
+
+        it('disallows invalid names', function () {
+            expect(mail.validateDisplayName('@')).to.be.an(Error);
+            expect(mail.validateDisplayName('a<p')).to.be.an(Error);
+            expect(mail.validateDisplayName('a>p')).to.be.an(Error);
+            expect(mail.validateDisplayName('a"x"')).to.be.an(Error);
+        });
+    });
+
+
     describe('mailboxes', function () {
         it('add user mailbox succeeds', async function () {
             await mail.addMailbox('girish', domain.domain, { ownerId: 'uid-0', ownerType: mail.OWNERTYPE_USER, active: true }, auditSource);

src/test/userdirectory-test.js

@@ -210,94 +210,64 @@ describe('User Directory Ldap', function () {
             mockApp.accessRestriction = null;
 
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: 'objectclass=group' }, auth);
-            expect(entries.length).to.equal(4);
-
-            // ensure order for testability
-            entries.sort(function (a, b) { return a.cn < b.cn; });
-
-            expect(entries[0].cn).to.equal('users');
-            expect(entries[0].memberuid.length).to.equal(2);
-            expect(entries[0].memberuid).to.contain(admin.id);
-            expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
-        });
-
-        it ('succeeds with cn wildcard filter', async function () {
-            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' }, auth);
-            expect(entries.length).to.equal(4);
-
-            // ensure order for testability
-            entries.sort(function (a, b) { return a.cn < b.cn; });
-
-            expect(entries[0].cn).to.equal('users');
-            expect(entries[0].memberuid.length).to.equal(2);
-            expect(entries[0].memberuid).to.contain(admin.id);
-            expect(entries[0].memberuid).to.contain(user.id);
-
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
-            expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
-        });
-
-        it('succeeds with memberuid filter', async function () {
-            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(memberuid=' + user.id + ')' }, auth);
             expect(entries.length).to.equal(2);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
 
-            expect(entries[1].cn).to.equal('ldap-test-1');
-            expect(entries[1].memberuid.length).to.equal(2);
-            expect(entries[1].memberuid).to.contain(admin.id);
-            expect(entries[1].memberuid).to.contain(user.id);
+            expect(entries[1].cn).to.equal('ldap-test-2');
+            expect(entries[1].memberuid).to.equal(admin.id);
+        });
+
+        it ('succeeds with cn wildcard filter', async function () {
+            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(cn=*)' }, auth);
+            expect(entries.length).to.equal(2);
+
+            // ensure order for testability
+            entries.sort(function (a, b) { return a.cn < b.cn; });
+
+            expect(entries[0].cn).to.equal('ldap-test-1');
+            expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
+
+            expect(entries[1].cn).to.equal('ldap-test-2');
+            expect(entries[1].memberuid).to.equal(admin.id);
+        });
+
+        it('succeeds with memberuid filter', async function () {
+            const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: '&(objectclass=group)(memberuid=' + user.id + ')' }, auth);
+            expect(entries.length).to.equal(1);
+
+            // ensure order for testability
+            entries.sort(function (a, b) { return a.cn < b.cn; });
+
+            expect(entries[0].cn).to.equal('ldap-test-1');
+            expect(entries[0].memberuid.length).to.equal(2);
+            expect(entries[0].memberuid).to.contain(admin.id);
+            expect(entries[0].memberuid).to.contain(user.id);
         });
 
         it ('succeeds with pagination', async function () {
             mockApp.accessRestriction = null;
             const entries = await ldapSearch('ou=groups,dc=cloudron', { filter: 'objectclass=group', paged: true }, auth);
-            expect(entries.length).to.equal(4);
+            expect(entries.length).to.equal(2);
 
             // ensure order for testability
             entries.sort(function (a, b) { return a.cn < b.cn; });
 
-            expect(entries[0].cn).to.equal('users');
+            expect(entries[0].cn).to.equal('ldap-test-1');
             expect(entries[0].memberuid.length).to.equal(2);
             expect(entries[0].memberuid).to.contain(admin.id);
             expect(entries[0].memberuid).to.contain(user.id);
 
-            expect(entries[1].cn).to.equal('admins');
-            // if only one entry, the array becomes a string :-/
+            expect(entries[1].cn).to.equal('ldap-test-2');
             expect(entries[1].memberuid).to.equal(admin.id);
-
-            expect(entries[2].cn).to.equal('ldap-test-1');
-            expect(entries[2].memberuid.length).to.equal(2);
-            expect(entries[2].memberuid).to.contain(admin.id);
-            expect(entries[2].memberuid).to.contain(user.id);
-
-            expect(entries[3].cn).to.equal('ldap-test-2');
-            expect(entries[3].memberuid).to.equal(admin.id);
         });
     });
 });

src/userdirectory.js

@@ -207,37 +207,6 @@ async function groupSearch(req, res, next) {
 
     const results = [];
 
-    // those are the old virtual groups for backwards compat
-    const virtualGroups = [{
-        name: 'users',
-        admin: false
-    }, {
-        name: 'admins',
-        admin: true
-    }];
-
-    virtualGroups.forEach(function (group) {
-        const dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-        const members = group.admin ? result.filter(function (user) { return users.compareRoles(user.role, users.ROLE_ADMIN) >= 0; }) : result;
-
-        const obj = {
-            dn: dn.toString(),
-            attributes: {
-                objectclass: ['group'],
-                cn: group.name,
-                memberuid: members.map(function(entry) { return entry.id; }).sort()
-            }
-        };
-
-        // ensure all filter values are also lowercase
-        const lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
-        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
-        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-            results.push(obj);
-        }
-    });
-
     let [errorGroups, resultGroups] = await safe(groups.listWithMembers());
     if (errorGroups) return next(new ldap.OperationsError(errorGroups.toString()));