proxyauth: make auth error handler return 401 for docker client

restart service does not rebuild automatically, we should add a route
for that. we need to figure where to scale services etc if we randomly
create containers like that.

CHANGES

@@ -2132,4 +2132,54 @@
 * Add volume management
 * backups: adjust node's heap size based on memory limit
 * s3: diasble per-chunk timeout
+* logs: more descriptive log file names on download
+* collectd: remove collectd config when app stopped (and add it back when started)
+* Apps can optionally request an authwall to be installed in front of them
+* mailbox can now owned by a group
+* linode: enable dns provider in setup view
+* dns: apps can now use the dns port
+* httpPaths: allow apps to specify forwarding from custom paths to container ports (for OLS)
+* add elasticemail smtp relay option
+* mail: add option to fts using solr
+* mail: change the namespace separator of new installations to /
+* mail: enable acl
+* Disable THP
+* filemanager: allow download dirs as zip files
+* aws: add china region
+* security: fix issue where apps could send with any username (but valid password)
+* i18n support
+
+[6.0.1]
+* app: add export route
+* mail: on location change, fix lock up when one or more domains have invalid credentials
+* mail: fix crash because of write after timeout closure
+* scaleway: fix installation issue where THP is not enabled in kernel
+
+[6.1.0]
+* mail: update haraka to 2.8.27. this fixes zero-length queue file crash
+* update: set/unset appStoreId from the update route
+* proxyauth: Do not follow redirects
+* proxyauth: add 2FA
+* appstore: add category translations
+* appstore: add media category
+* prepend the version to assets when sourcing to avoid cache hits on update
+* filemanger: list volumes of the app
+* Display upload size and size progress
+* nfs: chown the backups for hardlinks to work
+* remove user add/remove/role change email notifications
+* persist update indicator across restarts
+* cloudron-setup: add --generate-setup-token
+* dashboard: pass accessToken query param to automatically login
+* wellknown: add a way to set well known docs
+* oom: notification mails have links to dashboard
+* collectd: do not install xorg* packages
+* apptask: backup/restore tasks now use the backup memory limit configuration
+* eventlog: add logout event
+* mailbox: include alias in mailbox search
+* proxyAuth: add path exclusion
+* turn: fix for CVE-2020-26262
+* app password: fix regression where apps are not listed anymore in the UI
+* Support for multiDomain apps (domain aliases)
+* netcup: add dns provider
+* Container swap size is now dynamically determined based on system RAM/swap ratio
 

baseimage/initializeBaseUbuntuImage.sh

@@ -16,9 +16,6 @@ export DEBIAN_FRONTEND=noninteractive
 readonly ubuntu_codename=$(lsb_release -cs)
 readonly ubuntu_version=$(lsb_release -rs)
 
-# readonly arch="amd64"
-readonly arch="arm64"
-
 # hold grub since updating it breaks on some VPS providers. also, dist-upgrade will trigger it
 apt-mark hold grub* >/dev/null
 apt-get -o Dpkg::Options::="--force-confdef" update -y
@@ -32,10 +29,12 @@ debconf-set-selections <<< 'mysql-server mysql-server/root_password_again passwo
 
 # this enables automatic security upgrades (https://help.ubuntu.com/community/AutomaticSecurityUpdates)
 # resolvconf is needed for unbound to work property after disabling systemd-resolved in 18.04
+
 gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
 mysql_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "mysql-server-8.0" || echo "mysql-server-5.7")
-apt-get -y install \
+apt-get -y install --no-install-recommends \
     acl \
+    apparmor \
     build-essential \
     cifs-utils \
     cron \
@@ -46,6 +45,7 @@ apt-get -y install \
     ipset \
     iptables \
     libpython2.7 \
+    linux-generic \
     logrotate \
     $mysql_package \
     openssh-server \
@@ -55,23 +55,17 @@ apt-get -y install \
     tzdata \
     unattended-upgrades \
     unbound \
+    unzip \
     xfsprogs
 
-# TODO make it more generic for arm
-if [[ "${arch}" == "arm64" ]]; then
-    apt-get install -y linux-raspi
-else
-    apt-get isntall -y linux-generic
-fi
-
-echo "==> installing nginx for ${ubuntu_codename} for TLSv3 support"
-curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-1~${ubuntu_codename}_${arch}.deb -o /tmp/nginx.deb
+echo "==> installing nginx for xenial for TLSv3 support"
+curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
 # apt install with install deps (as opposed to dpkg -i)
 apt install -y /tmp/nginx.deb
 rm /tmp/nginx.deb
 
 # on some providers like scaleway the sudo file is changed and we want to keep the old one
-apt-get -o Dpkg::Options::="--force-confold" install -y sudo
+apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends sudo
 
 # this ensures that unattended upgades are enabled, if it was disabled during ubuntu install time (see #346)
 # debconf-set-selection of unattended-upgrades/enable_auto_updates + dpkg-reconfigure does not work
@@ -79,14 +73,10 @@ cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upg
 
 echo "==> Installing node.js"
 mkdir -p /usr/local/node-10.18.1
-if [[ "${arch}" == "arm64" ]]; then
-    curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-arm64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-10.18.1
-else
-    curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-10.18.1
-fi
+curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-10.18.1
 ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
 ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
-apt-get install -y python   # Install python which is required for npm rebuild
+apt-get install -y --no-install-recommends python   # Install python which is required for npm rebuild
 [[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
 # https://docs.docker.com/engine/installation/linux/ubuntulinux/
@@ -97,9 +87,9 @@ mkdir -p /etc/systemd/system/docker.service.d
 echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
 
 # there are 3 packages for docker - containerd, CLI and the daemon
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/${arch}/containerd.io_1.2.13-2_${arch}.deb" -o /tmp/containerd.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/${arch}/docker-ce-cli_19.03.12~3-0~ubuntu-${ubuntu_codename}_${arch}.deb" -o /tmp/docker-ce-cli.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/${arch}/docker-ce_19.03.12~3-0~ubuntu-${ubuntu_codename}_${arch}.deb" -o /tmp/docker.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.13-2_amd64.deb" -o /tmp/containerd.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 # apt install with install deps (as opposed to dpkg -i)
 apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
 rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
@@ -110,15 +100,11 @@ if [[ "${storage_driver}" != "overlay2" ]]; then
     exit 1
 fi
 
-if [[ "${arch}" == "arm64" ]]; then
-    echo -n " cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"  >> /boot/firmware/cmdline.txt
-else
-    # do not upgrade grub because it might prompt user and break this script
-    echo "==> Enable memory accounting"
-    apt-get -y --no-upgrade install grub2-common
-    sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
-    update-grub
-fi
+# do not upgrade grub because it might prompt user and break this script
+echo "==> Enable memory accounting"
+apt-get -y --no-upgrade --no-install-recommends install grub2-common
+sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
+update-grub
 
 echo "==> Downloading docker images"
 if [ ! -f "${arg_infraversionpath}/infra_version.js" ]; then
@@ -135,18 +121,23 @@ for image in ${images}; do
 done
 
 echo "==> Install collectd"
-if ! apt-get install -y libcurl3-gnutls collectd collectd-utils; then
+# without this, libnotify4 will install gnome-shell
+apt-get install -y libnotify4 --no-install-recommends
+if ! apt-get install -y --no-install-recommends libcurl3-gnutls collectd collectd-utils; then
     # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
     echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
     sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
 fi
 # https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
+[[ "${ubuntu_version}" == "20.04" ]] && echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
 
-[[ "${ubuntu_version}" == "20.04" && "${arch}" == "amd64" ]] && echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
-[[ "${ubuntu_version}" == "20.04" && "${arch}" == "arm64" ]] && echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-aarch64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
-
+# some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
 echo "==> Configuring host"
 sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
+if systemctl is-active ntp; then
+    systemctl stop ntp
+    apt purge -y ntp
+fi
 timedatectl set-ntp 1
 # mysql follows the system timezone
 timedatectl set-timezone UTC
@@ -160,7 +151,7 @@ if [ -f "/etc/default/motd-news" ]; then
     sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
 fi
 
-# Disable bind for good measure (on online.net, kimsufi servers these are pre-installed and conflicts with unbound)
+# Disable bind for good measure (on online.net, kimsufi servers these are pre-installed)
 systemctl stop bind9 || true
 systemctl disable bind9 || true
 
@@ -172,7 +163,7 @@ systemctl disable dnsmasq || true
 systemctl stop postfix || true
 systemctl disable postfix || true
 
-# on ubuntu 18.04, this is the default. this requires resolvconf for DNS to work further after the disable
+# on ubuntu 18.04 and 20.04, this is the default. this requires resolvconf for DNS to work further after the disable
 systemctl stop systemd-resolved || true
 systemctl disable systemd-resolved || true
 
@@ -181,4 +172,3 @@ systemctl disable systemd-resolved || true
 ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
 echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: ${ip6}" > /etc/unbound/unbound.conf.d/cloudron-network.conf
 systemctl restart unbound
-

box.js

@@ -7,6 +7,7 @@ let async = require('async'),
     fs = require('fs'),
     ldap = require('./src/ldap.js'),
     paths = require('./src/paths.js'),
+    proxyAuth = require('./src/proxyauth.js'),
     server = require('./src/server.js');
 
 const NOOP_CALLBACK = function () { };
@@ -22,7 +23,8 @@ function setupLogging(callback) {
 
 async.series([
     setupLogging,
-    server.start,
+    server.start, // do this first since it also inits the database
+    proxyAuth.start,
     ldap.start,
     dockerProxy.start
 ], function (error) {
@@ -38,6 +40,7 @@ async.series([
     process.on('SIGINT', function () {
         debug('Received SIGINT. Shutting down.');
 
+        proxyAuth.stop(NOOP_CALLBACK);
         server.stop(NOOP_CALLBACK);
         ldap.stop(NOOP_CALLBACK);
         dockerProxy.stop(NOOP_CALLBACK);
@@ -47,6 +50,7 @@ async.series([
     process.on('SIGTERM', function () {
         debug('Received SIGTERM. Shutting down.');
 
+        proxyAuth.stop(NOOP_CALLBACK);
         server.stop(NOOP_CALLBACK);
         ldap.stop(NOOP_CALLBACK);
         dockerProxy.stop(NOOP_CALLBACK);

migrations/20201110174504-apps-add-proxyAuth.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN proxyAuth BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN proxyAuth', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20201113071728-mailboxes-add-ownerType.js

@@ -0,0 +1,18 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN ownerType VARCHAR(16)'),
+        db.runSql.bind(db, 'UPDATE mailboxes SET ownerType=?', [ 'user' ]),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes MODIFY ownerType VARCHAR(16) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN ownerType', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20201119072309-apps-remove-httpPort.js

@@ -0,0 +1,13 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN httpPort')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20201120212726-apps-add-containerIp.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const async = require('async'),
+    iputils = require('../src/iputils.js');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN containerIp VARCHAR(16) UNIQUE', function (error) {
+        if (error) console.error(error);
+
+        let baseIp = iputils.intFromIp('172.18.16.0');
+
+        db.all('SELECT * FROM apps', function (error, apps) {
+            if (error) return callback(error);
+
+            async.eachSeries(apps, function (app, iteratorDone) {
+                const nextIp = iputils.ipFromInt(++baseIp);
+                db.runSql('UPDATE apps SET containerIp=? WHERE id=?', [ nextIp, app.id ], iteratorDone);
+            }, callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN containerIp', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20201204081504-settings-sftp-requireAdmin.js

@@ -0,0 +1,21 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM settings WHERE name=?', ['platform_config'], function (error, results) {
+        let value;
+        if (error || results.length === 0) {
+            value = { sftp: { requireAdmin: true } };
+        } else {
+            value = JSON.parse(results[0].value);
+            if (!value.sftp) value.sftp = {};
+            value.sftp.requireAdmin = true;
+        }
+
+        // existing installations may not even have the key. so use REPLACE instead of UPDATE
+        db.runSql('REPLACE INTO settings (name, value) VALUES (?, ?)', [ 'platform_config', JSON.stringify(value) ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20201222182945-groupMembers-unique-members.js

@@ -0,0 +1,18 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'CREATE TABLE groupMembers_copy(groupId VARCHAR(128) NOT NULL, userId VARCHAR(128) NOT NULL, FOREIGN KEY(groupId) REFERENCES userGroups(id), FOREIGN KEY(userId) REFERENCES users(id), UNIQUE (groupId, userId)) CHARACTER SET utf8 COLLATE utf8_bin'),    // In mysql CREATE TABLE.. LIKE does not copy indexes
+        db.runSql.bind(db, 'INSERT INTO groupMembers_copy SELECT * FROM groupMembers GROUP BY groupId, userId'),
+        db.runSql.bind(db, 'DROP TABLE groupMembers'),
+        db.runSql.bind(db, 'ALTER TABLE groupMembers_copy RENAME TO groupMembers')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE groupMembers DROP INDEX groupMembers_member'),
+    ], callback);
+};

migrations/20201223014453-domains-add-wellKnownJson.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE domains ADD COLUMN wellKnownJson TEXT', function (error) {
+        if (error) return callback(error);
+
+        // keep the paths around, so that we don't need to trigger a re-configure. the old nginx config will use the paths
+        // the new one will proxy calls to the box code
+        const WELLKNOWN_DIR = '/home/yellowtent/boxdata/well-known';
+        const output = safe.child_process.execSync('find . -type f -printf "%P\n"', { cwd: WELLKNOWN_DIR, encoding: 'utf8' });
+        if (!output) return callback();
+        const paths = output.trim().split('\n');
+        if (paths.length === 0) return callback(); // user didn't configure any well-known
+
+        let wellKnown = {};
+        for (let path of paths) {
+            const fqdn = path.split('/', 1)[0];
+            const loc = path.slice(fqdn.length+1);
+            const doc = safe.fs.readFileSync(`${WELLKNOWN_DIR}/${path}`, { encoding: 'utf8' });
+            if (!doc) continue;
+
+            wellKnown[fqdn] = {};
+            wellKnown[fqdn][loc] = doc;
+        }
+
+        console.log('Migrating well-known', JSON.stringify(wellKnown, null, 4));
+
+        async.eachSeries(Object.keys(wellKnown), function (fqdn, iteratorDone) {
+            db.runSql('UPDATE domains SET wellKnownJson=? WHERE domain=?', [ JSON.stringify(wellKnown[fqdn]), fqdn ], function (error, result) {
+                if (error) {
+                    console.error(error); // maybe the domain does not exist anymore
+                } else if (result.affectedRows === 0) {
+                    console.log(`Could not migrate wellknown as domain ${fqdn} is missing`);
+                }
+                iteratorDone();
+            });
+        }, function (error) {
+            callback(error);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE domains DROP COLUMN wellKnownJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210120025243-settings-platform-config-memoryLimit.js

@@ -0,0 +1,23 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM settings WHERE name=?', ['platform_config'], function (error, results) {
+        if (error || results.length === 0) return callback(null);
+
+        let value = JSON.parse(results[0].value);
+
+        for (const serviceName of Object.keys(value)) {
+            const service = value[serviceName];
+            if (!service.memorySwap) continue;
+            service.memoryLimit = service.memorySwap;
+            delete service.memorySwap;
+            delete service.memory;
+        }
+
+        db.runSql('UPDATE settings SET value=? WHERE name=?', [ JSON.stringify(value), 'platform_config' ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210120025750-apps-servicesConfig-memoryLimit.js

@@ -0,0 +1,28 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM apps', function (error, apps) {
+        if (error) return callback(error);
+
+        async.eachSeries(apps, function (app, iteratorDone) {
+            if (!app.servicesConfigJson) return iteratorDone();
+
+            let servicesConfig = JSON.parse(app.servicesConfigJson);
+            for (const serviceName of Object.keys(servicesConfig)) {
+                const service = servicesConfig[serviceName];
+                if (!service.memorySwap) continue;
+                service.memoryLimit = service.memorySwap;
+                delete service.memorySwap;
+                delete service.memory;
+            }
+
+            db.runSql('UPDATE apps SET servicesConfigJson=? WHERE id=?', [ JSON.stringify(servicesConfig), app.id ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210121201822-settings-rename-platform-config-to-services-config.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'services_config', 'platform_config' ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'platform_config', 'services_config' ], callback);
+};

migrations/schema.sql

@@ -44,7 +44,8 @@ CREATE TABLE IF NOT EXISTS groupMembers(
     groupId VARCHAR(128) NOT NULL,
     userId VARCHAR(128) NOT NULL,
     FOREIGN KEY(groupId) REFERENCES userGroups(id),
-    FOREIGN KEY(userId) REFERENCES users(id));
+    FOREIGN KEY(userId) REFERENCES users(id),
+    UNIQUE (groupId, userId));
 
 CREATE TABLE IF NOT EXISTS tokens(
     id VARCHAR(128) NOT NULL UNIQUE,
@@ -65,7 +66,6 @@ CREATE TABLE IF NOT EXISTS apps(
     healthTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, // when the app last responded
     containerId VARCHAR(128),
     manifestJson TEXT,
-    httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
     accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, // when the app was installed
     updateTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,   // when the last app update was done
@@ -86,6 +86,7 @@ CREATE TABLE IF NOT EXISTS apps(
     taskId INTEGER, // current task
     errorJson TEXT,
     servicesConfigJson TEXT, // app services configuration
+    containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair'
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
@@ -147,6 +148,7 @@ CREATE TABLE IF NOT EXISTS domains(
     provider VARCHAR(16) NOT NULL,
     configJson TEXT, /* JSON containing the dns backend provider config */
     tlsConfigJson TEXT, /* JSON containing the tls provider config */
+    wellKnownJson TEXT, /* JSON containing well known docs for this domain */
 
     PRIMARY KEY (domain))
 
@@ -180,6 +182,7 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     name VARCHAR(128) NOT NULL,
     type VARCHAR(16) NOT NULL, /* 'mailbox', 'alias', 'list' */
     ownerId VARCHAR(128) NOT NULL, /* user id */
+    ownerType VARCHAR(16) NOT NULL,
     aliasName VARCHAR(128), /* the target name type is an alias */
     aliasDomain VARCHAR(128), /* the target domain */
     membersJson TEXT, /* members of a group. fully qualified */

package-lock.json

@@ -272,11 +272,6 @@
       "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz",
       "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w=="
     },
-    "@types/color-name": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
-      "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ=="
-    },
     "@types/node": {
       "version": "13.5.3",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-13.5.3.tgz",
@@ -322,9 +317,9 @@
       }
     },
     "abstract-logging": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.0.tgz",
-      "integrity": "sha512-/oA9z7JszpIioo6J6dB79LVUgJ3eD3cxkAmdCkvWWS+Y9tPtALs1rLqOekLUXUbYqM2fB9TTK0ibAyZJJOP/CA=="
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz",
+      "integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA=="
     },
     "accepts": {
       "version": "1.3.7",
@@ -458,9 +453,9 @@
       "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
     },
     "aws-sdk": {
-      "version": "2.759.0",
-      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.759.0.tgz",
-      "integrity": "sha512-XTmt6b8NfluqmixO18Bu9ZttZMW9rwEDVO+XITsIQ5dZvLectwrjlbEn2refudJI1Y2pxLguw+tABvXi1wtbbQ==",
+      "version": "2.828.0",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.828.0.tgz",
+      "integrity": "sha512-JoDujGdncSIF9ka+XFZjop/7G+fNGucwPwYj7OHYMmFIOV5p7YmqomdbVmH/vIzd988YZz8oLOinWc4jM6vvhg==",
       "requires": {
         "buffer": "4.9.2",
         "events": "1.1.1",
@@ -501,7 +496,7 @@
     },
     "backoff": {
       "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/backoff/-/backoff-2.5.0.tgz",
+      "resolved": false,
       "integrity": "sha1-9hbtqdPktmuMp/ynn2lXIsX44m8=",
       "requires": {
         "precond": "0.2"
@@ -743,9 +738,9 @@
       }
     },
     "cloudron-manifestformat": {
-      "version": "5.6.0",
-      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.6.0.tgz",
-      "integrity": "sha512-BqM2vw/OWUHmPmrQo3xwAME0ncX3JPmPtxrhOYy0ZRpNcRDLrwXz02WVM9hAvIoawJNJjVb+x22RQoa1y5DdMw==",
+      "version": "5.10.1",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.10.1.tgz",
+      "integrity": "sha512-NI4wkf3rioeoJZQJKtpBHH3gGvyIBpw7sOvU2lFPMlsWWb6nXvGYJWCJkZ/s7Sdm937LGE6ZkBcNRNfcmxVMIg==",
       "requires": {
         "cron": "^1.8.2",
         "java-packagename-regex": "^1.0.0",
@@ -755,20 +750,36 @@
         "validator": "^12.2.0"
       },
       "dependencies": {
+        "lru-cache": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+          "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+          "requires": {
+            "yallist": "^4.0.0"
+          }
+        },
         "safetydance": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-1.0.0.tgz",
           "integrity": "sha512-ji9T/p5poiGgx4N3OFORGChAS9L8YL8S0/RpYvEPmQucCPrzmQMfdzbjFG/4+0BhJUvNA19KZUVj3YE8gkLpjA=="
         },
         "semver": {
-          "version": "7.3.2",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
-          "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ=="
+          "version": "7.3.4",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
+          "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
+          "requires": {
+            "lru-cache": "^6.0.0"
+          }
         },
         "validator": {
           "version": "12.2.0",
           "resolved": "https://registry.npmjs.org/validator/-/validator-12.2.0.tgz",
           "integrity": "sha512-jJfE/DW6tIK1Ek8nCfNFqt8Wb3nzMoAbocBF6/Icgg1ZFSBpObdnwVY2jQj6qUqzhx5jc71fpvBWyLGO7Xl+nQ=="
+        },
+        "yallist": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
         }
       }
     },
@@ -950,6 +961,20 @@
       "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
       "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
     },
+    "cookie": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
+      "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg=="
+    },
+    "cookie-parser": {
+      "version": "1.4.5",
+      "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.5.tgz",
+      "integrity": "sha512-f13bPUj/gG/5mDr+xLmSxxDsB9DQiTIfhJS/sqjrmfAWiAN+x2O4i/XguTL9yDZ+/IFDanJ+5x7hC4CXT9Tdzw==",
+      "requires": {
+        "cookie": "0.4.0",
+        "cookie-signature": "1.0.6"
+      }
+    },
     "cookie-session": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/cookie-session/-/cookie-session-1.4.0.tgz",
@@ -1057,9 +1082,9 @@
       "integrity": "sha512-lcWy3AXDRJOD7MplwZMmNSRM//kZtJaLz4n6D1P5z9wEmZGBKhJRBIr1Xs9KNQJmdXPblvgffynYji4iylUTcA=="
     },
     "db-migrate": {
-      "version": "0.11.11",
-      "resolved": "https://registry.npmjs.org/db-migrate/-/db-migrate-0.11.11.tgz",
-      "integrity": "sha512-GHZodjB5hXRy+76ZIb9z0OrUn0qSeGfvS0cCfyzPeFCBZ1YU9o9HUBQ8pUT+v/fJ9+a29eRz2xQsLfccXZtf8g==",
+      "version": "0.11.12",
+      "resolved": "https://registry.npmjs.org/db-migrate/-/db-migrate-0.11.12.tgz",
+      "integrity": "sha512-HUS8T5A3sKGCi+hz9XMKMwAKfU9sqhpDufW9nbVSRc5wxDO695uxA5lDe+If0OdvwwQOVxOnEZqkzAAxgyeFWg==",
       "requires": {
         "balanced-match": "^1.0.0",
         "bluebird": "^3.1.1",
@@ -1084,11 +1109,10 @@
           "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
         },
         "ansi-styles": {
-          "version": "4.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
-          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
           "requires": {
-            "@types/color-name": "^1.1.1",
             "color-convert": "^2.0.1"
           }
         },
@@ -1189,9 +1213,9 @@
           }
         },
         "yargs": {
-          "version": "15.3.1",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.3.1.tgz",
-          "integrity": "sha512-92O1HWEjw27sBfgmXiixJWT5hRBp2eobqXicLtPBIDBhYB+1HpwZlXmbW2luivBJHBzki+7VyCLRtAkScbTBQA==",
+          "version": "15.4.1",
+          "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz",
+          "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==",
           "requires": {
             "cliui": "^6.0.0",
             "decamelize": "^1.2.0",
@@ -1203,7 +1227,7 @@
             "string-width": "^4.2.0",
             "which-module": "^2.0.0",
             "y18n": "^4.0.0",
-            "yargs-parser": "^18.1.1"
+            "yargs-parser": "^18.1.2"
           }
         },
         "yargs-parser": {
@@ -1218,20 +1242,20 @@
       }
     },
     "db-migrate-base": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/db-migrate-base/-/db-migrate-base-2.3.0.tgz",
-      "integrity": "sha512-mxaCkSe7JC2uksvI/rKs+wOQGBSZ6B87xa4b3i+QhB+XRBpGdpMzldKE6INf+EnM6kwhbIPKjyJZgyxui9xBfQ==",
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/db-migrate-base/-/db-migrate-base-2.3.1.tgz",
+      "integrity": "sha512-HewYQ3HPmy7NOWmhhMLg9TzN1StEtSqGL3w8IbBRCxEsJ+oM3bDUQ/z5fqpYKfIUK07mMXieCmZYwFpwWkSHDw==",
       "requires": {
         "bluebird": "^3.1.1"
       }
     },
     "db-migrate-mysql": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/db-migrate-mysql/-/db-migrate-mysql-2.1.1.tgz",
-      "integrity": "sha512-dqyH+N8NyfDP1NSUHmkOuUDBhscTt3obubJ3VKz4BRyKtB/NOdx/hKxREV/rpytFGZbI2LT+ELeINSXU8N5IKQ==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/db-migrate-mysql/-/db-migrate-mysql-2.1.2.tgz",
+      "integrity": "sha512-dNfYWCDKVv1QjnIBjd4ZFIBEwpvCYuX26CyjgOupTjSNawC8zIOAaxzJVGXrRkes5xWm22sVVJnnUgJYtkIo9A==",
       "requires": {
         "bluebird": "^3.2.1",
-        "db-migrate-base": "^2.1.1",
+        "db-migrate-base": "^2.3.1",
         "moment": "^2.11.2",
         "mysql": "^2.17.1"
       }
@@ -1242,9 +1266,9 @@
       "integrity": "sha512-65k86bVeHaMxb2L0Gw3y5V+CgZSRwhVQMwDMydmw5MvIpHHwD6SmBciqIwHsZfzJ9yzV/yYhdRefRM6FV5/siw=="
     },
     "debug": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
-      "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
+      "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
       "requires": {
         "ms": "2.1.2"
       }
@@ -2442,9 +2466,9 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
     "ini": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
     },
     "ipaddr.js": {
       "version": "2.0.0",
@@ -2629,6 +2653,49 @@
       "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz",
       "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA="
     },
+    "jsonwebtoken": {
+      "version": "8.5.1",
+      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz",
+      "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==",
+      "requires": {
+        "jws": "^3.2.2",
+        "lodash.includes": "^4.3.0",
+        "lodash.isboolean": "^3.0.3",
+        "lodash.isinteger": "^4.0.4",
+        "lodash.isnumber": "^3.0.3",
+        "lodash.isplainobject": "^4.0.6",
+        "lodash.isstring": "^4.0.1",
+        "lodash.once": "^4.0.0",
+        "ms": "^2.1.1",
+        "semver": "^5.6.0"
+      },
+      "dependencies": {
+        "jwa": {
+          "version": "1.4.1",
+          "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
+          "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==",
+          "requires": {
+            "buffer-equal-constant-time": "1.0.1",
+            "ecdsa-sig-formatter": "1.0.11",
+            "safe-buffer": "^5.0.1"
+          }
+        },
+        "jws": {
+          "version": "3.2.2",
+          "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz",
+          "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==",
+          "requires": {
+            "jwa": "^1.4.1",
+            "safe-buffer": "^5.0.1"
+          }
+        },
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
+        }
+      }
+    },
     "jsprim": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
@@ -2683,9 +2750,9 @@
       }
     },
     "ldapjs": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/ldapjs/-/ldapjs-2.2.0.tgz",
-      "integrity": "sha512-9+ekbj97nxRYQMRgEm/HYFhFLWSRKah2PnReUfhfM5f62XBeUSFolB+AQ2Jij5tqowpksTnrdNCZLP6C+V3uag==",
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/ldapjs/-/ldapjs-2.2.3.tgz",
+      "integrity": "sha512-143MayI+cSo1PEngge0HMVj3Fb0TneX4Qp9yl9bKs45qND3G64B75GMSxtZCfNuVsvg833aOp1UWG8peFu1LrQ==",
       "requires": {
         "abstract-logging": "^2.0.0",
         "asn1": "^0.2.4",
@@ -2747,6 +2814,41 @@
       "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
       "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E="
     },
+    "lodash.includes": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
+      "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8="
+    },
+    "lodash.isboolean": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
+      "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY="
+    },
+    "lodash.isinteger": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
+      "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M="
+    },
+    "lodash.isnumber": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
+      "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w="
+    },
+    "lodash.isplainobject": {
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
+      "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs="
+    },
+    "lodash.isstring": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
+      "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE="
+    },
+    "lodash.once": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
+      "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w="
+    },
     "log-symbols": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz",
@@ -2842,9 +2944,9 @@
       "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
     },
     "mime": {
-      "version": "2.4.6",
-      "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz",
-      "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA=="
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-2.5.0.tgz",
+      "integrity": "sha512-ft3WayFSFUVBuJj7BMLKAQcSlItKtfjsKDDsii3rqFDAZ7t11zRe8ASw/GlmivGwVUYtwkQrxiGGpL6gFvB0ag=="
     },
     "mime-db": {
       "version": "1.43.0",
@@ -3027,14 +3129,14 @@
       }
     },
     "moment": {
-      "version": "2.29.0",
-      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.0.tgz",
-      "integrity": "sha512-z6IJ5HXYiuxvFTI6eiQ9dm77uE0gyy1yXNApVHqTcnIKfY9tIwEjlzsZ6u1LQXvVgKeTnv9Xm7NDvJ7lso3MtA=="
+      "version": "2.29.1",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz",
+      "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ=="
     },
     "moment-timezone": {
-      "version": "0.5.31",
-      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.31.tgz",
-      "integrity": "sha512-+GgHNg8xRhMXfEbv81iDtrVeTcWt0kWmTEY1XQK14dICTXnWJnT0dxdlPspwqF3keKMVPXwayEsk1DI0AA/jdA==",
+      "version": "0.5.32",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.32.tgz",
+      "integrity": "sha512-Z8QNyuQHQAmWucp8Knmgei8YNo28aLjJq6Ma+jy1ZSpSk5nyfRT8xgUbSQvD2+2UajISfenndwvFuH3NGS+nvA==",
       "requires": {
         "moment": ">= 2.9.0"
       }
@@ -3115,6 +3217,28 @@
         }
       }
     },
+    "mustache": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/mustache/-/mustache-3.2.1.tgz",
+      "integrity": "sha512-RERvMFdLpaFfSRIEe632yDm5nsd0SDKn8hGmcUwswnyiE5mtdZLDybtHAz6hjJhawokF0hXvGLtx9mrQfm6FkA=="
+    },
+    "mustache-express": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/mustache-express/-/mustache-express-1.3.0.tgz",
+      "integrity": "sha512-JWG8Rzxh9tpoLEH0NZ2u/caDiwhIkW+50IOBrcO+lHya3tCYj41bYPDEHCxPbKXvPrSyMNpI6ly4xdU2zpNQtg==",
+      "requires": {
+        "async": "~3.1.0",
+        "lru-cache": "~5.1.1",
+        "mustache": "^3.1.0"
+      },
+      "dependencies": {
+        "async": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/async/-/async-3.1.1.tgz",
+          "integrity": "sha512-X5Dj8hK1pJNC2Wzo2Rcp9FBVdJMGRR/S7V+lH46s8GVFhtbo5O4Le5GECCF/8PISVdkUA6mMPvgz7qTTD1rf1g=="
+        }
+      }
+    },
     "mute-stream": {
       "version": "0.0.8",
       "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
@@ -3366,9 +3490,9 @@
       }
     },
     "nodemailer": {
-      "version": "6.4.11",
-      "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.4.11.tgz",
-      "integrity": "sha512-BVZBDi+aJV4O38rxsUh164Dk1NCqgh6Cm0rQSb9SK/DHGll/DrCMnycVDD7msJgZCnmVa8ASo8EZzR7jsgTukQ=="
+      "version": "6.4.17",
+      "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.4.17.tgz",
+      "integrity": "sha512-89ps+SBGpo0D4Bi5ZrxcrCiRFaMmkCt+gItMXQGzEtZVR3uAD3QAQIDoxTWnx3ky0Dwwy/dhFrQ+6NNGXpw/qQ=="
     },
     "nodemailer-fetch": {
       "version": "1.6.0",
@@ -3692,11 +3816,6 @@
         "pinkie": "^2.0.0"
       }
     },
-    "pkginfo": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.4.1.tgz",
-      "integrity": "sha1-tUGO8EOd5UJfxJlQQtztFPsqhP8="
-    },
     "pngjs": {
       "version": "3.4.0",
       "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz",
@@ -3704,13 +3823,13 @@
     },
     "precond": {
       "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/precond/-/precond-0.2.3.tgz",
+      "resolved": false,
       "integrity": "sha1-qpWRvKokkj8eD0hJ0kD0fvwQdaw="
     },
     "pretty-bytes": {
-      "version": "5.4.1",
-      "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.4.1.tgz",
-      "integrity": "sha512-s1Iam6Gwz3JI5Hweaz4GoCD1WUNUIyzePFy5+Js2hjwGVt2Z79wNN+ZKOZ2vB6C+Xs6njyB84Z1IthQg8d9LxA=="
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.5.0.tgz",
+      "integrity": "sha512-p+T744ZyjjiaFlMUZZv6YPC5JrkNj8maRmPaQCWFJFplUAzpIUTRaTcS+7wmZtUoFXHtESJb23ISliaWyz3SHA=="
     },
     "process-nextick-args": {
       "version": "2.0.1",
@@ -3738,16 +3857,15 @@
       }
     },
     "prompt": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/prompt/-/prompt-1.0.0.tgz",
-      "integrity": "sha1-jlcSPDlquYiJf7Mn/Trtw+c15P4=",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/prompt/-/prompt-1.1.0.tgz",
+      "integrity": "sha512-ec1vUPXCplDBDUVD8uPa3XGA+OzLrO40Vxv3F1uxoiZGkZhdctlK2JotcHq5X6ExjocDOGwGdCSXloGNyU5L1Q==",
       "requires": {
         "colors": "^1.1.2",
-        "pkginfo": "0.x.x",
         "read": "1.0.x",
         "revalidator": "0.1.x",
         "utile": "0.3.x",
-        "winston": "2.1.x"
+        "winston": "2.x"
       }
     },
     "propagate": {
@@ -3956,9 +4074,9 @@
       }
     },
     "readdirp": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz",
-      "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==",
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz",
+      "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==",
       "requires": {
         "picomatch": "^2.2.1"
       }
@@ -4673,9 +4791,9 @@
       }
     },
     "tar-stream": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz",
-      "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==",
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
+      "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
       "requires": {
         "bl": "^4.0.3",
         "end-of-stream": "^1.4.1",
@@ -4695,12 +4813,12 @@
           }
         },
         "buffer": {
-          "version": "5.6.0",
-          "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
-          "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+          "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
           "requires": {
-            "base64-js": "^1.0.2",
-            "ieee754": "^1.1.4"
+            "base64-js": "^1.3.1",
+            "ieee754": "^1.1.13"
           }
         },
         "readable-stream": {
@@ -4880,9 +4998,9 @@
       }
     },
     "underscore": {
-      "version": "1.11.0",
-      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.11.0.tgz",
-      "integrity": "sha512-xY96SsN3NA461qIRKZ/+qox37YXPtSBswMGfiNptr+wrt6ds4HaMw23TP612fEyGekRE6LNRiLYr/aqbHXNedw=="
+      "version": "1.12.0",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.0.tgz",
+      "integrity": "sha512-21rQzss/XPMjolTiIezSu3JAjgagXKROtNrYFEOWK109qY1Uv2tVjPTZ1ci2HgvQDA16gHYSthQIJfB+XId/rQ=="
     },
     "unique-string": {
       "version": "1.0.0",
@@ -5066,16 +5184,15 @@
       }
     },
     "winston": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz",
-      "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=",
+      "version": "2.4.5",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.5.tgz",
+      "integrity": "sha512-TWoamHt5yYvsMarGlGEQE59SbJHqGsZV8/lwC+iCcGeAe0vUaOh+Lv6SYM17ouzC/a/LB1/hz/7sxFBtlu1l4A==",
       "requires": {
         "async": "~1.0.0",
         "colors": "1.0.x",
         "cycle": "1.0.x",
         "eyes": "0.1.x",
         "isstream": "0.1.x",
-        "pkginfo": "0.3.x",
         "stack-trace": "0.0.x"
       },
       "dependencies": {
@@ -5088,11 +5205,6 @@
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
           "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs="
-        },
-        "pkginfo": {
-          "version": "0.3.1",
-          "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.3.1.tgz",
-          "integrity": "sha1-Wyn2qB9wcXFC4J52W76rl7T4HiE="
         }
       }
     },
@@ -5127,9 +5239,9 @@
       }
     },
     "ws": {
-      "version": "7.3.1",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz",
-      "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA=="
+      "version": "7.4.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.2.tgz",
+      "integrity": "sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA=="
     },
     "xdg-basedir": {
       "version": "3.0.0",

package.json

@@ -18,17 +18,19 @@
     "@google-cloud/storage": "^2.5.0",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
     "async": "^2.6.3",
-    "aws-sdk": "^2.759.0",
+    "aws-sdk": "^2.828.0",
+    "basic-auth": "^2.0.1",
     "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.6.0",
+    "cloudron-manifestformat": "^5.10.1",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.0.0",
     "connect-timeout": "^1.9.0",
+    "cookie-parser": "^1.4.5",
     "cookie-session": "^1.4.0",
     "cron": "^1.8.2",
-    "db-migrate": "^0.11.11",
-    "db-migrate-mysql": "^2.1.1",
-    "debug": "^4.2.0",
+    "db-migrate": "^0.11.12",
+    "db-migrate-mysql": "^2.1.2",
+    "debug": "^4.3.1",
     "dockerode": "^2.5.8",
     "ejs": "^2.6.1",
     "ejs-cli": "^2.2.1",
@@ -36,23 +38,25 @@
     "ipaddr.js": "^2.0.0",
     "js-yaml": "^3.14.0",
     "json": "^9.0.6",
-    "ldapjs": "^2.2.0",
+    "jsonwebtoken": "^8.5.1",
+    "ldapjs": "^2.2.3",
     "lodash": "^4.17.20",
     "lodash.chunk": "^4.2.0",
-    "mime": "^2.4.6",
-    "moment": "^2.29.0",
-    "moment-timezone": "^0.5.31",
+    "mime": "^2.5.0",
+    "moment": "^2.29.1",
+    "moment-timezone": "^0.5.32",
     "morgan": "^1.10.0",
     "multiparty": "^4.2.2",
+    "mustache-express": "^1.3.0",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.4.11",
+    "nodemailer": "^6.4.17",
     "nodemailer-smtp-transport": "^2.7.4",
     "once": "^1.4.0",
-    "pretty-bytes": "^5.4.1",
+    "pretty-bytes": "^5.5.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
     "qrcode": "^1.4.4",
-    "readdirp": "^3.4.0",
+    "readdirp": "^3.5.0",
     "request": "^2.88.2",
     "rimraf": "^2.6.3",
     "s3-block-read-stream": "^0.5.0",
@@ -64,12 +68,12 @@
     "superagent": "^5.3.1",
     "supererror": "^0.7.2",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
-    "tar-stream": "^2.1.4",
+    "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
-    "underscore": "^1.11.0",
+    "underscore": "^1.12.0",
     "uuid": "^3.4.0",
     "validator": "^11.0.0",
-    "ws": "^7.3.1",
+    "ws": "^7.4.2",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {

runTests

@@ -2,11 +2,11 @@
 
 set -eu
 
-readonly SOURCE_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly DATA_DIR="${HOME}/.cloudron_test"
 readonly DEFAULT_TESTS="./src/test/*-test.js ./src/routes/test/*-test.js"
 
-! "${SOURCE_dir}/src/test/checkInstall" && exit 1
+! "${source_dir}/src/test/checkInstall" && exit 1
 
 # cleanup old data dirs some of those docker container data requires sudo to be removed
 echo "=> Provide root password to purge any leftover data in ${DATA_DIR} and load apparmor profile:"
@@ -22,19 +22,27 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
-mkdir -p platformdata/addons/mail platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
+mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com boxdata/sftp/ssh
+mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
+sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
+
+# translations
+mkdir -p box/dashboard/dist/translation
+cp -r ${source_dir}/../dashboard/dist/translation/* box/dashboard/dist/translation
 
 # put cert
 echo "=> Generating a localhost selfsigned cert"
 openssl req -x509 -newkey rsa:2048 -keyout platformdata/nginx/cert/host.key -out platformdata/nginx/cert/host.cert -days 3650 -subj '/CN=localhost' -nodes -config <(cat /etc/ssl/openssl.cnf <(printf "\n[SAN]\nsubjectAltName=DNS:*.localhost"))
 
+# generate legacy key format for sftp
+ssh-keygen -m PEM -t rsa -f boxdata/sftp/ssh/ssh_host_rsa_key -q -N ""
+
 # clear out any containers
 echo "=> Delete all docker containers first"
 docker ps -qa | xargs --no-run-if-empty docker rm -f
 
 # create docker network (while the infra code does this, most tests skip infra setup)
-docker network create --subnet=172.18.0.0/16 cloudron || true
+docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
 
 # create the same mysql server version to test with
 OUT=`docker inspect mysql-server` || true
@@ -59,7 +67,7 @@ echo "=> Ensure database"
 mysql -h"${MYSQL_IP}" -uroot -ppassword -e 'CREATE DATABASE IF NOT EXISTS box'
 
 echo "=> Run database migrations"
-cd "${SOURCE_dir}"
+cd "${source_dir}"
 BOX_ENV=test DATABASE_URL=mysql://root:password@${MYSQL_IP}/box node_modules/.bin/db-migrate up
 
 echo "=> Run tests with mocha"

scripts/cloudron-setup

@@ -43,31 +43,37 @@ fi
 initBaseImage="true"
 provider="generic"
 requestedVersion=""
+installServerOrigin="https://api.cloudron.io"
 apiServerOrigin="https://api.cloudron.io"
 webServerOrigin="https://cloudron.io"
 sourceTarballUrl=""
 rebootServer="true"
+setupToken=""
 
-args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,source-tarball-url:,version:,env:,skip-reboot" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot,generate-setup-token" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
     --help) echo "See https://docs.cloudron.io/installation/ on how to install Cloudron"; exit 0;;
     --provider) provider="$2"; shift 2;;
-    --source-tarball-url) sourceTarballUrl="$2"; shift 2;;
     --version) requestedVersion="$2"; shift 2;;
     --env)
         if [[ "$2" == "dev" ]]; then
             apiServerOrigin="https://api.dev.cloudron.io"
             webServerOrigin="https://dev.cloudron.io"
+            installServerOrigin="https://api.dev.cloudron.io"
         elif [[ "$2" == "staging" ]]; then
             apiServerOrigin="https://api.staging.cloudron.io"
             webServerOrigin="https://staging.cloudron.io"
+            installServerOrigin="https://api.staging.cloudron.io"
+        elif [[ "$2" == "unstable" ]]; then
+            installServerOrigin="https://api.dev.cloudron.io"
         fi
         shift 2;;
     --skip-baseimage-init) initBaseImage="false"; shift;;
     --skip-reboot) rebootServer="false"; shift;;
+    --generate-setup-token) setupToken="$(openssl rand -hex 10)"; shift;;
     --) break;;
     *) echo "Unknown option $1"; exit 1;;
     esac
@@ -101,43 +107,33 @@ echo " Join us at https://forum.cloudron.io for any questions."
 echo ""
 
 if [[ "${initBaseImage}" == "true" ]]; then
-    echo "=> Installing software-properties-common"
-    if ! apt-get install -y software-properties-common &>> "${LOG_FILE}"; then
-        echo "Could not install software-properties-common (for add-apt-repository below). See ${LOG_FILE}"
-        exit 1
-    fi
-
     echo "=> Updating apt and installing script dependencies"
     if ! apt-get update &>> "${LOG_FILE}"; then
         echo "Could not update package repositories. See ${LOG_FILE}"
         exit 1
     fi
 
-    if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install curl python3 ubuntu-standard -y &>> "${LOG_FILE}"; then
+    if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install --no-install-recommends curl python3 ubuntu-standard software-properties-common -y &>> "${LOG_FILE}"; then
         echo "Could not install setup dependencies (curl). See ${LOG_FILE}"
         exit 1
     fi
 fi
 
-if [[ "$sourceTarballUrl" == "" ]]; then
-    echo "=> Checking version"
-    if ! releaseJson=$($curl -s "${apiServerOrigin}/api/v1/releases?boxVersion=${requestedVersion}"); then
-        echo "Failed to get release information"
-        exit 1
-    fi
+echo "=> Checking version"
+if ! releaseJson=$($curl -s "${installServerOrigin}/api/v1/releases?boxVersion=${requestedVersion}"); then
+    echo "Failed to get release information"
+    exit 1
+fi
 
-    if [[ "$requestedVersion" == "" ]]; then
-        version=$(echo "${releaseJson}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["version"])')
-    else
-        version="${requestedVersion}"
-    fi
-
-    if ! sourceTarballUrl=$(echo "${releaseJson}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["info"]["sourceTarballUrl"])'); then
-        echo "No source code for version '${requestedVersion:-latest}'"
-        exit 1
-    fi
+if [[ "$requestedVersion" == "" ]]; then
+    version=$(echo "${releaseJson}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["version"])')
 else
-    version=${requestedVersion}
+    version="${requestedVersion}"
+fi
+
+if ! sourceTarballUrl=$(echo "${releaseJson}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["info"]["sourceTarballUrl"])'); then
+    echo "No source code for version '${requestedVersion:-latest}'"
+    exit 1
 fi
 
 echo "=> Downloading version ${version} ..."
@@ -162,6 +158,7 @@ fi
 echo "=> Installing version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
 echo "${provider}" > /etc/cloudron/PROVIDER
+[[ ! -z "${setupToken}" ]] && echo "${setupToken}" > /etc/cloudron/SETUP_TOKEN
 
 if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
     echo "Failed to install cloudron. See ${LOG_FILE} for details"
@@ -183,7 +180,12 @@ done
 if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
     ip='<IP>'
 fi
-echo -e "\n\n${GREEN}After reboot, visit https://${ip} and accept the self-signed certificate to finish setup.${DONE}\n"
+if [[ -z "${setupToken}" ]]; then
+    url="https://${ip}"
+else
+    url="https://${ip}/?setupToken=${setupToken}"
+fi
+echo -e "\n\n${GREEN}After reboot, visit ${url} and accept the self-signed certificate to finish setup.${DONE}\n"
 
 if [[ "${rebootServer}" == "true" ]]; then
     systemctl stop box mysql # sometimes mysql ends up having corrupt privilege tables

scripts/cloudron-translation-update

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+# This script downloads new translation data from weblate at https://translate.cloudron.io
+
+OUT="/home/yellowtent/box/dashboard/dist/translation"
+
+# We require root
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root. Run with sudo"
+    exit 1
+fi
+
+echo "=> Downloading new translation files..."
+curl https://translate.cloudron.io/download/cloudron/dashboard/?format=zip -o /tmp/lang.zip
+
+echo "=> Unpacking..."
+unzip -jo /tmp/lang.zip -d $OUT
+chown -R yellowtent:yellowtent $OUT
+# unzip put very restrictive permissions
+chmod ua+r $OUT/*
+
+echo "=> Cleanup..."
+rm /tmp/lang.zip
+
+echo "=> Done"
+
+echo ""
+echo "Reload the dashboard to see the new translations"
+echo ""

scripts/installer.sh

@@ -60,16 +60,15 @@ fi
 readonly nginx_version=$(nginx -v 2>&1)
 if [[ "${nginx_version}" != *"1.18."* ]]; then
     echo "==> installer: installing nginx 1.18"
-    curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-1~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
+    curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
     # apt install with install deps (as opposed to dpkg -i)
     apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
     rm /tmp/nginx.deb
 fi
 
-if ! which ipset; then
-    echo "==> installer: installing ipset"
-    apt install -y ipset
-fi
+# Cloudron 6 on ubuntu 20 installed recommended packages of collectd -> libinotify -> gnome->shell
+apt remove -y gnome-shell || true
+apt -y autoremove || true
 
 echo "==> installer: updating node"
 if [[ "$(node --version)" != "v10.18.1" ]]; then

setup/start.sh

@@ -19,6 +19,7 @@ readonly json="$(realpath ${script_dir}/../node_modules/.bin/json)"
 readonly ubuntu_version=$(lsb_release -rs)
 
 cp -f "${script_dir}/../scripts/cloudron-support" /usr/bin/cloudron-support
+cp -f "${script_dir}/../scripts/cloudron-translation-update" /usr/bin/cloudron-translation-update
 
 # this needs to match the cloudron/base:2.0.0 gid
 if ! getent group media; then
@@ -31,7 +32,8 @@ systemctl enable apparmor
 systemctl restart apparmor
 
 usermod ${USER} -a -G docker
-docker network create --subnet=172.18.0.0/16 cloudron || true
+# unbound (which starts after box code) relies on this interface to exist. dockerproxy also relies on this.
+docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
 
 mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
@@ -63,6 +65,7 @@ mkdir -p "${BOX_DATA_DIR}/certs"
 mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
 mkdir -p "${BOX_DATA_DIR}/mail/dkim"
 mkdir -p "${BOX_DATA_DIR}/well-known" # .well-known documents
+mkdir -p "${BOX_DATA_DIR}/sftp/ssh" # sftp keys
 
 # ensure backups folder exists and is writeable
 mkdir -p /var/backups
@@ -101,14 +104,13 @@ unbound-anchor -a /var/lib/unbound/root.key
 
 echo "==> Adding systemd services"
 cp -r "${script_dir}/start/systemd/." /etc/systemd/system/
-systemctl disable cloudron.target || true
-rm -f /etc/systemd/system/cloudron.target
 [[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/MemoryMax/MemoryLimit/g' -i /etc/systemd/system/box.service
 systemctl daemon-reload
+systemctl enable --now cloudron-syslog
 systemctl enable unbound
-systemctl enable cloudron-syslog
 systemctl enable box
 systemctl enable cloudron-firewall
+systemctl enable --now cloudron-disable-thp
 
 # update firewall rules
 systemctl restart cloudron-firewall
@@ -218,12 +220,15 @@ else
     cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem"
 fi
 
-# old installations used to create appdata/<app>/redis which is now part of old backups and prevents restore
-echo "==> Cleaning up stale redis directories"
-find "${APPS_DATA_DIR}" -maxdepth 2 -type d -name redis -exec rm -rf {} +
-
-echo "==> Cleaning up old logs"
-rm -f /home/yellowtent/platformdata/logs/*/*.log.* || true
+if [[ ! -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" ]]; then
+    # the key format in Ubuntu 20 changed, so we create keys in legacy format. for older ubuntu, just re-use the host keys
+    # see https://github.com/proftpd/proftpd/issues/793
+    if [[ "${ubuntu_version}" == "20.04" ]]; then
+        ssh-keygen -m PEM -t rsa -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" -q -N ""
+    else
+        cp /etc/ssh/ssh_host_rsa_key* ${BOX_DATA_DIR}/sftp/ssh
+    fi
+fi
 
 echo "==> Changing ownership"
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change

setup/start/cloudron-disable-thp.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -eu
+
+echo "==> Disabling THP"
+
+# https://docs.couchbase.com/server/current/install/thp-disable.html
+if [[ -d /sys/kernel/mm/transparent_hugepage ]]; then
+    echo "never" > /sys/kernel/mm/transparent_hugepage/enabled
+    echo "never" > /sys/kernel/mm/transparent_hugepage/defrag
+else
+    echo "==> kernel does not have THP"
+fi
+

setup/start/cloudron-firewall.sh

@@ -26,6 +26,10 @@ if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_
     [[ -n "${allowed_tcp_ports}" ]] && iptables -A CLOUDRON -p tcp -m tcp -m multiport --dports "${allowed_tcp_ports}" -j ACCEPT
 fi
 
+if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(','))" 2>/dev/null); then
+    [[ -n "${allowed_tcp_ports}" ]] && iptables -A CLOUDRON -p udp -m udp -m multiport --dports "${allowed_tcp_ports}" -j ACCEPT
+fi
+
 # turn and stun service
 iptables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
 iptables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT

setup/start/cloudron-motd

@@ -10,10 +10,17 @@ if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
     fi
     echo "${ip}" > /tmp/.cloudron-motd-cache
 
+    if [[ ! -f /etc/cloudron/SETUP_TOKEN ]]; then
+        url="https://${ip}"
+    else
+        setupToken="$(cat /etc/cloudron/SETUP_TOKEN)"
+        url="https://${ip}/?setupToken=${setupToken}"
+    fi
+
     printf "\t\t\tWELCOME TO CLOUDRON\n"
     printf "\t\t\t-------------------\n"
 
-    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit https://${ip} on your browser and accept the self-signed certificate to finish setup."
+    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit ${url} on your browser and accept the self-signed certificate to finish setup."
     printf "Cloudron overview - https://docs.cloudron.io/ \n"
     printf "Cloudron setup - https://docs.cloudron.io/installation/#setup \n"
 else

setup/start/collectd/df.py

@@ -6,7 +6,7 @@ disks = []
 
 def init():
     global disks
-    lines = [s.split() for s in subprocess.check_output(["df", "--type=ext4", "--output=source,target,size,used,avail"]).splitlines()]
+    lines = [s.split() for s in subprocess.check_output(["df", "--type=ext4", "--output=source,target,size,used,avail"]).decode('utf-8').splitlines()]
     disks = lines[1:] # strip header
     collectd.info('custom df plugin initialized with %s' % disks)
 

setup/start/mysql.cnf

@@ -6,7 +6,7 @@
 performance_schema=OFF
 max_connections=50
 # on ec2, without this we get a sporadic connection drop when doing the initial migration
-max_allowed_packet=32M
+max_allowed_packet=64M
 
 # https://mathiasbynens.be/notes/mysql-utf8mb4
 character-set-server = utf8mb4

setup/start/systemd/cloudron-disable-thp.service

@@ -0,0 +1,15 @@
+# https://docs.mongodb.com/manual/tutorial/transparent-huge-pages/
+[Unit]
+Description=Disable Transparent Huge Pages (THP)
+DefaultDependencies=no
+After=sysinit.target local-fs.target
+Before=docker.service
+
+[Service]
+Type=oneshot
+ExecStart="/home/yellowtent/box/setup/start/cloudron-disable-thp.sh"
+RemainAfterExit=yes
+
+[Install]
+WantedBy=basic.target
+

setup/start/systemd/unbound.service

@@ -2,7 +2,7 @@
 
 [Unit]
 Description=Unbound DNS Resolver
-After=network.target
+After=network.target docker.service
 
 [Service]
 PIDFile=/run/unbound.pid

setup/start/unbound.conf

@@ -1,5 +1,7 @@
 server:
-        interface: 0.0.0.0
+        port: 53
+        interface: 127.0.0.1
+        interface: 172.18.0.1
         do-ip6: no
         access-control: 127.0.0.1 allow
         access-control: 172.18.0.1/16 allow

src/appdb.js

@@ -1,32 +1,32 @@
 'use strict';
 
 exports = module.exports = {
-    get: get,
-    getByHttpPort: getByHttpPort,
-    getByContainerId: getByContainerId,
-    add: add,
-    exists: exists,
-    del: del,
-    update: update,
-    getAll: getAll,
-    getPortBindings: getPortBindings,
-    delPortBinding: delPortBinding,
+    get,
+    add,
+    exists,
+    del,
+    update,
+    getAll,
+    getPortBindings,
+    delPortBinding,
 
-    setAddonConfig: setAddonConfig,
-    getAddonConfig: getAddonConfig,
-    getAddonConfigByAppId: getAddonConfigByAppId,
-    getAddonConfigByName: getAddonConfigByName,
-    unsetAddonConfig: unsetAddonConfig,
-    unsetAddonConfigByAppId: unsetAddonConfigByAppId,
-    getAppIdByAddonConfigValue: getAppIdByAddonConfigValue,
+    setAddonConfig,
+    getAddonConfig,
+    getAddonConfigByAppId,
+    getAddonConfigByName,
+    unsetAddonConfig,
+    unsetAddonConfigByAppId,
+    getAppIdByAddonConfigValue,
+    getByIpAddress,
 
-    setHealth: setHealth,
-    setTask: setTask,
-    getAppStoreIds: getAppStoreIds,
+    setHealth,
+    setTask,
+    getAppStoreIds,
 
     // subdomain table types
     SUBDOMAIN_TYPE_PRIMARY: 'primary',
     SUBDOMAIN_TYPE_REDIRECT: 'redirect',
+    SUBDOMAIN_TYPE_ALIAS: 'alias',
 
     _clear: clear
 };
@@ -38,17 +38,14 @@ var assert = require('assert'),
     safe = require('safetydance'),
     util = require('util');
 
-var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
-    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'subdomains.subdomain AS location', 'subdomains.domain',
-    'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
+const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
+    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson',
-    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup',
+    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp',
     'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
     'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(',');
 
-var PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
-
-const SUBDOMAIN_FIELDS = [ 'appId', 'domain', 'subdomain', 'type' ].join(',');
+const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
 
 function postProcess(result) {
     assert.strictEqual(typeof result, 'object');
@@ -89,6 +86,7 @@ function postProcess(result) {
     result.sso = !!result.sso; // make it bool
     result.enableBackup = !!result.enableBackup; // make it bool
     result.enableAutomaticUpdate = !!result.enableAutomaticUpdate; // make it bool
+    result.proxyAuth = !!result.proxyAuth;
 
     assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
     result.debugMode = safe.JSON.parse(result.debugModeJson);
@@ -98,11 +96,23 @@ function postProcess(result) {
     result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
     delete result.servicesConfigJson;
 
-    result.alternateDomains = result.alternateDomains || [];
-    result.alternateDomains.forEach(function (d) {
-        delete d.appId;
-        delete d.type;
-    });
+    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
+    delete result.subdomains;
+    delete result.domains;
+    delete result.subdomainTypes;
+
+    result.alternateDomains = [];
+    result.aliasDomains = [];
+    for (let i = 0; i < subdomainTypes.length; i++) {
+        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
+            result.location = subdomains[i];
+            result.domain = domains[i];
+        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
+            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
+        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
+            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
+        }
+    }
 
     let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
     delete result.envNames;
@@ -125,121 +135,54 @@ function postProcess(result) {
     result.taskId = result.taskId ? String(result.taskId) : null;
 }
 
+// each query simply join apps table with another table by id. we then join the full result together
+const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
+const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
+const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
+const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
+const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
+    + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
+    + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
+    + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
+    + ` LEFT JOIN (${MOUNTS_QUERY}) AS q4 on q4.id = apps.id`;
+
 function get(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes, '
-        + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues,'
-        + 'JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys '
-        + ' FROM apps'
-        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
-        + '  LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId'
-        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?'
-        + '  LEFT OUTER JOIN appMounts ON apps.id = appMounts.appId'
-        + ' WHERE apps.id = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, id ], function (error, result) {
+    database.query(`${APPS_QUERY} WHERE apps.id = ?`, [ id ], function (error, result) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
         if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
 
-        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE appId = ? AND type = ?', [ id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        postProcess(result[0]);
 
-            result[0].alternateDomains = alternateDomains;
-
-            postProcess(result[0]);
-
-            callback(null, result[0]);
-        });
+        callback(null, result[0]);
     });
 }
 
-function getByHttpPort(httpPort, callback) {
-    assert.strictEqual(typeof httpPort, 'number');
+function getByIpAddress(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-    + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes,'
-    + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues,'
-    + 'JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys '
-        + ' FROM apps'
-        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
-        + '  LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId'
-        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?'
-        + '  LEFT OUTER JOIN appMounts ON apps.id = appMounts.appId'
-        + ' WHERE httpPort = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, httpPort ], function (error, result) {
+    database.query(`${APPS_QUERY} WHERE apps.containerIp = ?`, [ ip ], function (error, result) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
         if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
 
-        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE appId = ? AND type = ?', [ result[0].id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        postProcess(result[0]);
 
-            result[0].alternateDomains = alternateDomains;
-            postProcess(result[0]);
-
-            callback(null, result[0]);
-        });
-    });
-}
-
-function getByContainerId(containerId, callback) {
-    assert.strictEqual(typeof containerId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes,'
-        + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues,'
-        + 'JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys '
-        + ' FROM apps'
-        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
-        + '  LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId'
-        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?'
-        + '  LEFT OUTER JOIN appMounts ON apps.id = appMounts.appId'
-        + ' WHERE containerId = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, containerId ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE appId = ? AND type = ?', [ result[0].id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            result[0].alternateDomains = alternateDomains;
-            postProcess(result[0]);
-
-            callback(null, result[0]);
-        });
+        callback(null, result[0]);
     });
 }
 
 function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes,'
-        + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues,'
-        + 'JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys '
-        + ' FROM apps'
-        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
-        + '  LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId'
-        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?'
-        + '  LEFT OUTER JOIN appMounts ON apps.id = appMounts.appId'
-        + ' GROUP BY apps.id ORDER BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY ], function (error, results) {
+    database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ], function (error, results) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
-        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE type = ?', [ exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        results.forEach(postProcess);
 
-            alternateDomains.forEach(function (d) {
-                var domain = results.find(function (a) { return d.appId === a.id; });
-                if (!domain) return;
-
-                domain.alternateDomains = domain.alternateDomains || [];
-                domain.alternateDomains.push(d);
-            });
-
-            results.forEach(postProcess);
-
-            callback(null, results);
-        });
+        callback(null, results);
     });
 }
 
@@ -311,6 +254,15 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
         });
     }
 
+    if (data.aliasDomains) {
+        data.aliasDomains.forEach(function (d) {
+            queries.push({
+                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
+            });
+        });
+    }
+
     database.transaction(queries, function (error) {
         if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
         if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'no such domain'));
@@ -408,6 +360,7 @@ function updateWithConstraints(id, app, constraints, callback) {
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
     assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
     assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
+    assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
     assert(!('tags' in app) || Array.isArray(app.tags));
     assert(!('env' in app) || typeof app.env === 'object');
 
@@ -443,6 +396,12 @@ function updateWithConstraints(id, app, constraints, callback) {
                 queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
             });
         }
+
+        if ('aliasDomains' in app) {
+            app.aliasDomains.forEach(function (d) {
+                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
+            });
+        }
     }
 
     if ('mounts' in app) {
@@ -457,7 +416,7 @@ function updateWithConstraints(id, app, constraints, callback) {
         if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig') {
             fields.push(`${p}Json = ?`);
             values.push(JSON.stringify(app[p]));
-        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'env' && p !== 'mounts') {
+        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
             fields.push(p + ' = ?');
             values.push(app[p]);
         }

src/apphealthmonitor.js

@@ -21,6 +21,7 @@ const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be sm
 const UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes
 
 const OOM_EVENT_LIMIT = 60 * 60 * 1000; // 60 minutes
+const gStartTime = new Date(); // time when apphealthmonitor was started
 let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago
 
 function debugApp(app) {
@@ -34,7 +35,8 @@ function setHealth(app, health, callback) {
     assert.strictEqual(typeof health, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let now = new Date(), healthTime = app.healthTime, curHealth = app.health;
+    let now = new Date(), curHealth = app.health;
+    let healthTime = gStartTime > app.healthTime ? gStartTime : app.healthTime; // on box restart, clamp value to start time
 
     if (health === apps.HEALTH_HEALTHY) {
         healthTime = now;
@@ -85,8 +87,7 @@ function checkAppHealth(app, callback) {
         // non-appstore apps may not have healthCheckPath
         if (!manifest.healthCheckPath) return setHealth(app, apps.HEALTH_HEALTHY, callback);
 
-        // poll through docker network instead of nginx to bypass any potential oauth proxy
-        var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
+        const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
         superagent
             .get(healthCheckUrl)
             .set('Host', app.fqdn) // required for some apache configs with rewrite rules
@@ -96,7 +97,7 @@ function checkAppHealth(app, callback) {
             .end(function (error, res) {
                 if (error && !error.response) {
                     setHealth(app, apps.HEALTH_UNHEALTHY, callback);
-                } else if (res.statusCode >= 400) { // 2xx and 3xx are ok
+                } else if (res.statusCode >= 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
                     setHealth(app, apps.HEALTH_UNHEALTHY, callback);
                 } else {
                     setHealth(app, apps.HEALTH_HEALTHY, callback);
@@ -119,7 +120,7 @@ function getContainerInfo(containerId, callback) {
 
 /*
     OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:0.10.0 /bin/bash
+        docker run -ti -m 100M cloudron/base:2.0.0 /bin/bash
         apt-get update && apt-get install stress
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */

src/apps.js

@@ -6,7 +6,6 @@ exports = module.exports = {
     removeRestrictedFields,
 
     get,
-    getByContainerId,
     getByIpAddress,
     getByFqdn,
     getAll,
@@ -34,6 +33,7 @@ exports = module.exports = {
 
     restore,
     importApp,
+    exportApp,
     clone,
 
     update,
@@ -63,6 +63,7 @@ exports = module.exports = {
 
     getDataDir,
     getIconPath,
+    getMemoryLimit,
 
     downloadFile,
     uploadFile,
@@ -135,7 +136,6 @@ var appdb = require('./appdb.js'),
     superagent = require('superagent'),
     tasks = require('./tasks.js'),
     TransformStream = require('stream').Transform,
-    updateChecker = require('./updatechecker.js'),
     users = require('./users.js'),
     util = require('util'),
     uuid = require('uuid'),
@@ -155,7 +155,6 @@ function validatePortBindings(portBindings, manifest) {
     const RESERVED_PORTS = [
         22, /* ssh */
         25, /* smtp */
-        53, /* dns */
         80, /* http */
         143, /* imap */
         202, /* alternate ssh */
@@ -168,7 +167,7 @@ function validatePortBindings(portBindings, manifest) {
         2004, /* graphite (lo) */
         2514, /* cloudron-syslog (lo) */
         constants.PORT, /* app server (lo) */
-        constants.SYSADMIN_PORT, /* sysadmin app server (lo) */
+        constants.AUTHWALL_PORT, /* protected sites */
         constants.INTERNAL_SMTP_PORT, /* internal smtp port (lo) */
         constants.LDAP_PORT,
         3306, /* mysql (lo) */
@@ -192,7 +191,7 @@ function validatePortBindings(portBindings, manifest) {
         if (!Number.isInteger(hostPort)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not an integer`, { field: 'portBindings', portName: portName });
         if (RESERVED_PORTS.indexOf(hostPort) !== -1) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
         if (RESERVED_PORT_RANGES.find(range => (hostPort >= range[0] && hostPort <= range[1]))) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
-        if (hostPort <= 1023 || hostPort > 65535) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not in permitted range`, { field: 'portBindings', portName: portName });
+        if (hostPort !== 53 && (hostPort <= 1023 || hostPort > 65535)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not in permitted range`, { field: 'portBindings', portName: portName }); // dns 53 is special and adblocker apps can use them
     }
 
     // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
@@ -385,7 +384,7 @@ function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, port
 
     // check if any of the port bindings conflict
     for (let portName in portBindings) {
-        if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is reserved`, { portName });
+        if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is in use`, { portName });
     }
 
     if (match[2] === 'dataDir') {
@@ -407,13 +406,13 @@ function removeInternalFields(app) {
         'location', 'domain', 'fqdn', 'mailboxName', 'mailboxDomain',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts');
+        'label', 'alternateDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts');
 }
 
 // non-admins can only see these
 function removeRestrictedFields(app) {
     return _.pick(app,
-        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'alternateDomains', 'sso',
+        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'alternateDomains', 'aliasDomains', 'sso',
         'location', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup');
 }
 
@@ -447,6 +446,20 @@ function getIconPath(app, options, callback) {
     callback(new BoxError(BoxError.NOT_FOUND, 'No icon'));
 }
 
+function getMemoryLimit(app) {
+    assert.strictEqual(typeof app, 'object');
+
+    let memoryLimit = app.memoryLimit || app.manifest.memoryLimit || 0;
+
+    if (memoryLimit === -1) { // unrestricted
+        memoryLimit = 0;
+    } else if (memoryLimit === 0 || memoryLimit < constants.DEFAULT_MEMORY_LIMIT) { // ensure we never go below minimum (in case we change the default)
+        memoryLimit = constants.DEFAULT_MEMORY_LIMIT;
+    }
+
+    return memoryLimit;
+}
+
 function postProcess(app, domainObjectMap) {
     let result = {};
     for (let portName in app.portBindings) {
@@ -457,6 +470,7 @@ function postProcess(app, domainObjectMap) {
     app.iconUrl = getIconUrlSync(app);
     app.fqdn = domains.fqdn(app.location, domainObjectMap[app.domain]);
     app.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    app.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 }
 
 function hasAccessTo(app, user, callback) {
@@ -499,25 +513,6 @@ function get(appId, callback) {
         if (error) return callback(error);
 
         appdb.get(appId, function (error, app) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'No such app'));
-            if (error) return callback(error);
-
-            postProcess(app, domainObjectMap);
-
-            callback(null, app);
-        });
-    });
-}
-
-function getByContainerId(containerId, callback) {
-    assert.strictEqual(typeof containerId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    getDomainObjectMap(function (error, domainObjectMap) {
-        if (error) return callback(error);
-
-        appdb.getByContainerId(containerId, function (error, app) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'No such app'));
             if (error) return callback(error);
 
             postProcess(app, domainObjectMap);
@@ -535,16 +530,15 @@ function getByIpAddress(ip, callback) {
     // this is only used by the ldap test. the apps tests still uses proper docker
     if (constants.TEST && exports._MOCK_GET_BY_IP_APP_ID) return get(exports._MOCK_GET_BY_IP_APP_ID, callback);
 
-    docker.getContainerIdByIp(ip, function (error, containerId) {
+    appdb.getByIpAddress(ip, function (error, app) {
         if (error) return callback(error);
 
-        docker.inspect(containerId, function (error, result) {
+        getDomainObjectMap(function (error, domainObjectMap) {
             if (error) return callback(error);
 
-            const appId = safe.query(result, 'Config.Labels.appId', null);
-            if (!appId) return callback(new BoxError(BoxError.NOT_FOUND, 'No such app'));
+            postProcess(app, domainObjectMap);
 
-            get(appId, callback);
+            callback(null, app);
         });
     });
 }
@@ -626,20 +620,31 @@ function scheduleTask(appId, installationState, taskId, callback) {
     assert.strictEqual(typeof taskId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    appTaskManager.scheduleTask(appId, taskId, function (error) {
-        debug(`scheduleTask: task ${taskId} of ${appId} completed`);
-        if (error && (error.code === tasks.ECRASHED || error.code === tasks.ESTOPPED)) { // if task crashed, update the error
-            debug(`Apptask crashed/stopped: ${error.message}`);
-            let boxError = new BoxError(BoxError.TASK_ERROR, error.message);
-            boxError.details.crashed = error.code === tasks.ECRASHED;
-            boxError.details.stopped = error.code === tasks.ESTOPPED;
-            // see also apptask makeTaskError
-            boxError.details.taskId = taskId;
-            boxError.details.installationState = installationState;
-            appdb.update(appId, { installationState: exports.ISTATE_ERROR, error: boxError.toPlainObject(), taskId: null }, callback);
-        } else if (!(installationState === exports.ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall
-            appdb.update(appId, { taskId: null }, callback);
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(error);
+
+        let memoryLimit = 400;
+        if (installationState === exports.ISTATE_PENDING_BACKUP || installationState === exports.ISTATE_PENDING_CLONE || installationState === exports.ISTATE_PENDING_RESTORE) {
+            memoryLimit = 'memoryLimit' in backupConfig ? Math.max(backupConfig.memoryLimit/1024/1024, 400) : 400;
         }
+
+        const options = { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit };
+
+        appTaskManager.scheduleTask(appId, taskId, options, function (error) {
+            debug(`scheduleTask: task ${taskId} of ${appId} completed`);
+            if (error && (error.code === tasks.ECRASHED || error.code === tasks.ESTOPPED)) { // if task crashed, update the error
+                debug(`Apptask crashed/stopped: ${error.message}`);
+                let boxError = new BoxError(BoxError.TASK_ERROR, error.message);
+                boxError.details.crashed = error.code === tasks.ECRASHED;
+                boxError.details.stopped = error.code === tasks.ESTOPPED;
+                // see also apptask makeTaskError
+                boxError.details.taskId = taskId;
+                boxError.details.installationState = installationState;
+                appdb.update(appId, { installationState: exports.ISTATE_ERROR, error: boxError.toPlainObject(), taskId: null }, callback);
+            } else if (!(installationState === exports.ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall
+                appdb.update(appId, { taskId: null }, callback);
+            }
+        });
     });
 }
 
@@ -701,7 +706,13 @@ function validateLocations(locations, callback) {
         for (let location of locations) {
             if (!(location.domain in domainObjectMap)) return callback(new BoxError(BoxError.BAD_FIELD, 'No such domain', { field: 'location', domain: location.domain, subdomain: location.subdomain }));
 
-            error = domains.validateHostname(location.subdomain, domainObjectMap[location.domain]);
+            let subdomain = location.subdomain;
+            if (location.type === 'alias' && subdomain.startsWith('*')) {
+                if (subdomain === '*') continue;
+                subdomain = subdomain.replace(/^\*\./, ''); // remove *.
+            }
+
+            error = domains.validateHostname(subdomain, domainObjectMap[location.domain]);
             if (error) return callback(new BoxError(BoxError.BAD_FIELD, 'Bad location: ' + error.message, { field: 'location', domain: location.domain, subdomain: location.subdomain }));
         }
 
@@ -733,6 +744,7 @@ function install(data, auditSource, callback) {
         enableBackup = 'enableBackup' in data ? data.enableBackup : true,
         enableAutomaticUpdate = 'enableAutomaticUpdate' in data ? data.enableAutomaticUpdate : true,
         alternateDomains = data.alternateDomains || [],
+        aliasDomains = data.aliasDomains || [],
         env = data.env || {},
         label = data.label || null,
         tags = data.tags || [],
@@ -766,7 +778,7 @@ function install(data, auditSource, callback) {
 
     if ('sso' in data && !('optionalSso' in manifest)) return callback(new BoxError(BoxError.BAD_FIELD, 'sso can only be specified for apps with optionalSso'));
     // if sso was unspecified, enable it by default if possible
-    if (sso === null) sso = !!manifest.addons['ldap'] || !!manifest.addons['oauth'];
+    if (sso === null) sso = !!manifest.addons['ldap'] || !!manifest.addons['proxyAuth'];
 
     error = validateEnv(env);
     if (error) return callback(error);
@@ -785,7 +797,10 @@ function install(data, auditSource, callback) {
         }
     }
 
-    const locations = [{subdomain: location, domain}].concat(alternateDomains);
+    const locations = [{ subdomain: location, domain, type: 'primary' }]
+        .concat(alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
+        .concat(aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));
+
     validateLocations(locations, function (error, domainObjectMap) {
         if (error) return callback(error);
 
@@ -797,18 +812,19 @@ function install(data, auditSource, callback) {
         debug('Will install app with id : ' + appId);
 
         var data = {
-            accessRestriction: accessRestriction,
-            memoryLimit: memoryLimit,
-            sso: sso,
-            debugMode: debugMode,
-            mailboxName: mailboxName,
-            mailboxDomain: mailboxDomain,
-            enableBackup: enableBackup,
-            enableAutomaticUpdate: enableAutomaticUpdate,
-            alternateDomains: alternateDomains,
-            env: env,
-            label: label,
-            tags: tags,
+            accessRestriction,
+            memoryLimit,
+            sso,
+            debugMode,
+            mailboxName,
+            mailboxDomain,
+            enableBackup,
+            enableAutomaticUpdate,
+            alternateDomains,
+            aliasDomains,
+            env,
+            label,
+            tags,
             runState: exports.RSTATE_RUNNING,
             installationState: exports.ISTATE_PENDING_INSTALL
         };
@@ -838,6 +854,7 @@ function install(data, auditSource, callback) {
                     const newApp = _.extend({}, data, { appStoreId, manifest, location, domain, portBindings });
                     newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
                     newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+                    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
                     eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId, app: newApp, taskId: result.taskId });
 
@@ -1193,7 +1210,8 @@ function setLocation(app, data, auditSource, callback) {
         domain: data.domain.toLowerCase(),
         // these are intentionally reset, if not set
         portBindings: null,
-        alternateDomains: []
+        alternateDomains: [],
+        aliasDomains: []
     };
 
     if ('portBindings' in data) {
@@ -1213,14 +1231,20 @@ function setLocation(app, data, auditSource, callback) {
         values.alternateDomains = data.alternateDomains;
     }
 
-    const locations = [{subdomain: values.location, domain: values.domain}].concat(values.alternateDomains);
+    if ('aliasDomains' in data) {
+        values.aliasDomains = data.aliasDomains;
+    }
+
+    const locations = [{ subdomain: values.location, domain: values.domain, type: 'primary' }]
+        .concat(values.alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
+        .concat(values.aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));
 
     validateLocations(locations, function (error, domainObjectMap) {
         if (error) return callback(error);
 
         const task = {
             args: {
-                oldConfig: _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'portBindings'),
+                oldConfig: _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'aliasDomains', 'portBindings'),
                 overwriteDns: !!data.overwriteDns
             },
             values
@@ -1231,6 +1255,7 @@ function setLocation(app, data, auditSource, callback) {
 
             values.fqdn = domains.fqdn(values.location, domainObjectMap[values.domain]);
             values.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+            values.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
             eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, _.extend({ appId, app, taskId: result.taskId }, values));
 
@@ -1274,7 +1299,8 @@ function update(app, data, auditSource, callback) {
 
     const skipBackup = !!data.skipBackup,
         appId = app.id,
-        manifest = data.manifest;
+        manifest = data.manifest,
+        appStoreId = data.appStoreId;
 
     let values = {};
 
@@ -1289,14 +1315,12 @@ function update(app, data, auditSource, callback) {
     error = checkManifestConstraints(manifest);
     if (error) return callback(error);
 
-    var updateConfig = { skipBackup, manifest };
+    var updateConfig = { skipBackup, manifest, appStoreId }; // this will clear appStoreId when updating from a repo and set it if passed in for update route
 
     // prevent user from installing a app with different manifest id over an existing app
     // this allows cloudron install -f --app <appid> for an app installed from the appStore
     if (app.manifest.id !== updateConfig.manifest.id) {
         if (!data.force) return callback(new BoxError(BoxError.BAD_FIELD, 'manifest id does not match. force to override'));
-        // clear appStoreId so that this app does not get updates anymore
-        updateConfig.appStoreId = '';
     }
 
     // suffix '0' if prerelease is missing for semver.lte to work as expected
@@ -1343,9 +1367,6 @@ function update(app, data, auditSource, callback) {
 
         eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId, app, skipBackup, toManifest: manifest, fromManifest: app.manifest, force: data.force, taskId: result.taskId });
 
-        // clear update indicator, if update fails, it will come back through the update checker
-        updateChecker.resetAppUpdateInfo(appId);
-
         callback(null, { taskId: result.taskId });
     });
 }
@@ -1571,6 +1592,26 @@ function importApp(app, data, auditSource, callback) {
     });
 }
 
+function exportApp(app, data, auditSource, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const appId = app.id;
+
+    let error = checkAppState(app, exports.ISTATE_PENDING_BACKUP);
+    if (error) return callback(error);
+
+    const task = {
+        args: { snapshotOnly: true },
+        values: {}
+    };
+    addTask(appId, exports.ISTATE_PENDING_BACKUP, task, (error, result) => {
+        if (error) return callback(error);
+
+        callback(null, { taskId: result.taskId });
+    });
+}
+
 function purchaseApp(data, callback) {
     assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -1625,7 +1666,7 @@ function clone(app, data, user, auditSource, callback) {
         let mailboxName = hasMailAddon(manifest) ? mailboxNameForLocation(location, manifest) : null;
         let mailboxDomain = hasMailAddon(manifest) ? domain : null;
 
-        const locations = [{subdomain: location, domain}];
+        const locations = [{ subdomain: location, domain, type: 'primary' }];
         validateLocations(locations, function (error, domainObjectMap) {
             if (error) return callback(error);
 
@@ -1642,7 +1683,8 @@ function clone(app, data, user, auditSource, callback) {
                 enableBackup: app.enableBackup,
                 reverseProxyConfig: app.reverseProxyConfig,
                 env: app.env,
-                alternateDomains: []
+                alternateDomains: [],
+                aliasDomains: []
             };
 
             appdb.add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) {
@@ -1664,6 +1706,8 @@ function clone(app, data, user, auditSource, callback) {
                         const newApp = _.extend({}, data, { appStoreId, manifest, location, domain, portBindings });
                         newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
                         newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+                        newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+
                         eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, oldApp: app, newApp: newApp, taskId: result.taskId });
 
                         callback(null, { id: newAppId, taskId: result.taskId });

src/appstore.js

@@ -1,28 +1,28 @@
 'use strict';
 
 exports = module.exports = {
-    getFeatures: getFeatures,
+    getFeatures,
 
-    getApps: getApps,
-    getApp: getApp,
-    getAppVersion: getAppVersion,
+    getApps,
+    getApp,
+    getAppVersion,
 
-    trackBeginSetup: trackBeginSetup,
-    trackFinishedSetup: trackFinishedSetup,
+    trackBeginSetup,
+    trackFinishedSetup,
 
-    registerWithLoginCredentials: registerWithLoginCredentials,
+    registerWithLoginCredentials,
 
-    purchaseApp: purchaseApp,
-    unpurchaseApp: unpurchaseApp,
+    purchaseApp,
+    unpurchaseApp,
 
-    getUserToken: getUserToken,
-    getSubscription: getSubscription,
-    isFreePlan: isFreePlan,
+    getUserToken,
+    getSubscription,
+    isFreePlan,
 
-    getAppUpdate: getAppUpdate,
-    getBoxUpdate: getBoxUpdate,
+    getAppUpdate,
+    getBoxUpdate,
 
-    createTicket: createTicket
+    createTicket
 };
 
 var apps = require('./apps.js'),
@@ -45,6 +45,8 @@ var apps = require('./apps.js'),
 // Keep in sync with appstore/routes/cloudrons.js
 let gFeatures = {
     userMaxCount: 5,
+    userGroups: false,
+    userRoles: false,
     domainMaxCount: 1,
     externalLdap: false,
     privateDockerRegistry: false,
@@ -247,13 +249,13 @@ function getBoxUpdate(options, callback) {
             if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
             if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
             if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 204) return callback(null); // no update
+            if (result.statusCode === 204) return callback(null, null); // no update
             if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
             var updateInfo = result.body;
 
             if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Invalid update version: %s %s', result.statusCode, result.text)));
+                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', result.statusCode, result.text)));
             }
 
             // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }

src/apptask.js

@@ -6,7 +6,6 @@ exports = module.exports = {
     run: run,
 
     // exported for testing
-    _reserveHttpPort: reserveHttpPort,
     _configureReverseProxy: configureReverseProxy,
     _unconfigureReverseProxy: unconfigureReverseProxy,
     _createAppDir: createAppDir,
@@ -17,8 +16,7 @@ exports = module.exports = {
     _waitForDnsPropagation: waitForDnsPropagation
 };
 
-var addons = require('./addons.js'),
-    appdb = require('./appdb.js'),
+const appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
@@ -34,14 +32,15 @@ var addons = require('./addons.js'),
     ejs = require('ejs'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
+    iputils = require('./iputils.js'),
     manifestFormat = require('cloudron-manifestformat'),
-    net = require('net'),
     os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
     reverseProxy = require('./reverseproxy.js'),
     rimraf = require('rimraf'),
     safe = require('safetydance'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     superagent = require('superagent'),
@@ -89,22 +88,16 @@ function updateApp(app, values, callback) {
     });
 }
 
-function reserveHttpPort(app, callback) {
+function allocateContainerIp(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let server = net.createServer();
-    server.listen(0, function () {
-        let port = server.address().port;
-
-        updateApp(app, { httpPort: port }, function (error) {
-            server.close(function (/* closeError */) {
-                if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Failed to allocate http port ${port}: ${error.message}`));
-
-                callback(null);
-            });
-        });
-    });
+    async.retry({ times: 10 }, function (retryCallback) {
+        const iprange = iputils.intFromIp('172.18.20.255') - iputils.intFromIp('172.18.16.1');
+        let rnd = Math.floor(Math.random() * iprange);
+        const containerIp = iputils.ipFromInt(iputils.intFromIp('172.18.16.1') + rnd);
+        updateApp(app, { containerIp }, retryCallback);
+    }, callback);
 }
 
 function configureReverseProxy(app, callback) {
@@ -348,9 +341,9 @@ function registerSubdomains(app, overwrite, callback) {
     sysinfo.getServerIp(function (error, ip) {
         if (error) return callback(error);
 
-        const allDomains = [ { subdomain: app.location, domain: app.domain }].concat(app.alternateDomains);
+        const allDomains = [ { subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains);
 
-        debug(`registerSubdomain: Will register ${JSON.stringify(allDomains)}`);
+        debugApp(app, `registerSubdomain: Will register ${JSON.stringify(allDomains)}`);
 
         async.eachSeries(allDomains, function (domain, iteratorDone) {
             async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
@@ -370,7 +363,7 @@ function registerSubdomains(app, overwrite, callback) {
 
                     domains.upsertDnsRecords(domain.subdomain, domain.domain, 'A', [ ip ], function (error) {
                         if (error && (error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                            debug('registerSubdomains: Upsert error. Will retry.', error.message);
+                            debugApp(app, 'registerSubdomains: Upsert error. Will retry.', error.message);
                             return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain })); // try again
                         }
 
@@ -401,7 +394,7 @@ function unregisterSubdomains(app, allDomains, callback) {
                 domains.removeDnsRecords(domain.subdomain, domain.domain, 'A', [ ip ], function (error) {
                     if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, null);
                     if (error && (error.reason === BoxError.SBUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                        debug('registerSubdomains: Remove error. Will retry.', error.message);
+                        debugApp(app, 'registerSubdomains: Remove error. Will retry.', error.message);
                         return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain })); // try again
                     }
 
@@ -431,8 +424,8 @@ function waitForDnsPropagation(app, callback) {
         domains.waitForDnsRecord(app.location, app.domain, 'A', ip, { times: 240 }, function (error) {
             if (error) return callback(new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: app.location, domain: app.domain }));
 
-            // now wait for alternateDomains, if any
-            async.eachSeries(app.alternateDomains, function (domain, iteratorCallback) {
+            // now wait for alternateDomains and aliasDomains, if any
+            async.eachSeries(app.alternateDomains.concat(app.aliasDomains), function (domain, iteratorCallback) {
                 domains.waitForDnsRecord(domain.subdomain, domain.domain, 'A', ip, { times: 240 }, function (error) {
                     if (error) return callback(new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: domain.subdomain, domain: domain.domain }));
 
@@ -451,7 +444,7 @@ function moveDataDir(app, targetDir, callback) {
     let resolvedSourceDir = apps.getDataDir(app, app.dataDir);
     let resolvedTargetDir = apps.getDataDir(app, targetDir);
 
-    debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
+    debugApp(app, `moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
 
     if (resolvedSourceDir === resolvedTargetDir) return callback();
 
@@ -484,6 +477,8 @@ function downloadImage(manifest, callback) {
 }
 
 function startApp(app, callback){
+    debugApp(app, 'startApp: starting container');
+
     if (app.runState === apps.RSTATE_STOPPED) return callback();
 
     docker.startContainer(app.id, callback);
@@ -517,7 +512,7 @@ function install(app, args, progressCallback, callback) {
                 addonsToRemove = app.manifest.addons;
             }
 
-            addons.teardownAddons(app, addonsToRemove, next);
+            services.teardownAddons(app, addonsToRemove, next);
         },
 
         function deleteAppDirIfNeeded(done) {
@@ -532,7 +527,8 @@ function install(app, args, progressCallback, callback) {
             docker.deleteImage(oldManifest, done);
         },
 
-        reserveHttpPort.bind(null, app),
+        // allocating container ip here, lets the users "repair" an app if allocation fails at appdb.add time
+        allocateContainerIp.bind(null, app),
 
         progressCallback.bind(null, { percent: 20, message: 'Downloading icon' }),
         downloadIcon.bind(null, app),
@@ -550,24 +546,24 @@ function install(app, args, progressCallback, callback) {
             if (!restoreConfig) {
                 async.series([
                     progressCallback.bind(null, { percent: 60, message: 'Setting up addons' }),
-                    addons.setupAddons.bind(null, app, app.manifest.addons),
+                    services.setupAddons.bind(null, app, app.manifest.addons),
                 ], next);
             } else if (!restoreConfig.backupId) { // in-place import
                 async.series([
                     progressCallback.bind(null, { percent: 60, message: 'Importing addons in-place' }),
-                    addons.setupAddons.bind(null, app, app.manifest.addons),
-                    addons.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
-                    addons.restoreAddons.bind(null, app, app.manifest.addons),
+                    services.setupAddons.bind(null, app, app.manifest.addons),
+                    services.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
+                    services.restoreAddons.bind(null, app, app.manifest.addons),
                 ], next);
             } else {
                 async.series([
                     progressCallback.bind(null, { percent: 65, message: 'Download backup and restoring addons' }),
-                    addons.setupAddons.bind(null, app, app.manifest.addons),
-                    addons.clearAddons.bind(null, app, app.manifest.addons),
+                    services.setupAddons.bind(null, app, app.manifest.addons),
+                    services.clearAddons.bind(null, app, app.manifest.addons),
                     backups.downloadApp.bind(null, app, restoreConfig, (progress) => {
                         progressCallback({ percent: 65, message: progress.message });
                     }),
-                    addons.restoreAddons.bind(null, app, app.manifest.addons)
+                    services.restoreAddons.bind(null, app, app.manifest.addons)
                 ], next);
             }
         },
@@ -602,7 +598,7 @@ function backup(app, args, progressCallback, callback) {
 
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Backing up' }),
-        backups.backupApp.bind(null, app, { /* options */ }, (progress) => {
+        backups.backupApp.bind(null, app, { snapshotOnly: !!args.snapshotOnly }, (progress) => {
             progressCallback({ percent: 30, message: progress.message });
         }),
 
@@ -630,7 +626,7 @@ function create(app, args, progressCallback, callback) {
 
         // FIXME: re-setup addons only because sendmail addon to re-inject env vars on mailboxName change
         progressCallback.bind(null, { percent: 30, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        services.setupAddons.bind(null, app, app.manifest.addons),
 
         progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
         createContainer.bind(null, app),
@@ -667,6 +663,12 @@ function changeLocation(app, args, progressCallback, callback) {
                 return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
             });
 
+            if (oldConfig.aliasDomains) {
+                obsoleteDomains = obsoleteDomains.concat(oldConfig.aliasDomains.filter(function (o) {
+                    return !app.aliasDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
+                }));
+            }
+
             if (locationChanged) obsoleteDomains.push({ subdomain: oldConfig.location, domain: oldConfig.domain });
 
             if (obsoleteDomains.length === 0) return next();
@@ -679,7 +681,7 @@ function changeLocation(app, args, progressCallback, callback) {
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        services.setupAddons.bind(null, app, app.manifest.addons),
 
         progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
         createContainer.bind(null, app),
@@ -721,7 +723,7 @@ function migrateDataDir(app, args, progressCallback, callback) {
 
         // re-setup addons since this creates the localStorage volume
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),
+        services.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),
 
         progressCallback.bind(null, { percent: 60, message: 'Moving data dir' }),
         moveDataDir.bind(null, app, newDataDir),
@@ -754,7 +756,6 @@ function configure(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
         unconfigureReverseProxy.bind(null, app),
         deleteContainers.bind(null, app, { managedOnly: true }),
-        reserveHttpPort.bind(null, app),
 
         progressCallback.bind(null, { percent: 20, message: 'Downloading icon' }),
         downloadIcon.bind(null, app),
@@ -767,7 +768,7 @@ function configure(app, args, progressCallback, callback) {
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        services.setupAddons.bind(null, app, app.manifest.addons),
 
         progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
         createContainer.bind(null, app),
@@ -789,7 +790,6 @@ function configure(app, args, progressCallback, callback) {
     });
 }
 
-// nginx configuration is skipped because app.httpPort is expected to be available
 function update(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
@@ -801,7 +801,10 @@ function update(app, args, progressCallback, callback) {
 
     // app does not want these addons anymore
     // FIXME: this does not handle option changes (like multipleDatabases)
-    var unusedAddons = _.omit(app.manifest.addons, Object.keys(updateConfig.manifest.addons));
+    const unusedAddons = _.omit(app.manifest.addons, Object.keys(updateConfig.manifest.addons));
+    const httpPathsChanged = app.manifest.httpPaths !== updateConfig.manifest.httpPaths;
+    const httpPortChanged = app.manifest.httpPort !== updateConfig.manifest.httpPort;
+    const proxyAuthChanged = !_.isEqual(safe.query(app.manifest, 'addons.proxyAuth'), safe.query(updateConfig.manifest, 'addons.proxyAuth'));
 
     async.series([
         // this protects against the theoretical possibility of an app being marked for update from
@@ -840,7 +843,7 @@ function update(app, args, progressCallback, callback) {
         },
 
         // only delete unused addons after backup
-        addons.teardownAddons.bind(null, app, unusedAddons),
+        services.teardownAddons.bind(null, app, unusedAddons),
 
         // free unused ports
         function (next) {
@@ -852,7 +855,7 @@ function update(app, args, progressCallback, callback) {
                 if (newTcpPorts[portName] || newUdpPorts[portName]) return callback(null); // port still in use
 
                 appdb.delPortBinding(currentPorts[portName], apps.PORT_TYPE_TCP, function (error) {
-                    if (error && error.reason === BoxError.NOT_FOUND) debug('update: portbinding does not exist in database', error);
+                    if (error && error.reason === BoxError.NOT_FOUND) debugApp(app, 'update: portbinding does not exist in database', error);
                     else if (error) return next(error);
 
                     // also delete from app object for further processing (the db is updated in the next step)
@@ -869,13 +872,20 @@ function update(app, args, progressCallback, callback) {
         downloadIcon.bind(null, app),
 
         progressCallback.bind(null, { percent: 60, message: 'Updating addons' }),
-        addons.setupAddons.bind(null, app, updateConfig.manifest.addons),
+        services.setupAddons.bind(null, app, updateConfig.manifest.addons),
 
         progressCallback.bind(null, { percent: 70, message: 'Creating container' }),
         createContainer.bind(null, app),
 
         startApp.bind(null, app),
 
+        progressCallback.bind(null, { percent: 90, message: 'Configuring reverse proxy' }),
+        function (next) {
+            if (!httpPathsChanged && !proxyAuthChanged && !httpPortChanged) return next();
+
+            configureReverseProxy(app, next);
+        },
+
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, updateTime: new Date() })
     ], function seriesDone(error) {
@@ -899,13 +909,16 @@ function start(app, args, progressCallback, callback) {
 
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Starting app services' }),
-        addons.startAppServices.bind(null, app),
+        services.startAppServices.bind(null, app),
 
         progressCallback.bind(null, { percent: 35, message: 'Starting container' }),
         docker.startContainer.bind(null, app.id),
 
+        progressCallback.bind(null, { percent: 60, message: 'Adding collectd profile' }),
+        addCollectdProfile.bind(null, app),
+
         // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
-        progressCallback.bind(null, { percent: 60, message: 'Configuring reverse proxy' }),
+        progressCallback.bind(null, { percent: 80, message: 'Configuring reverse proxy' }),
         configureReverseProxy.bind(null, app),
 
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
@@ -930,7 +943,10 @@ function stop(app, args, progressCallback, callback) {
         docker.stopContainers.bind(null, app.id),
 
         progressCallback.bind(null, { percent: 50, message: 'Stopping app services' }),
-        addons.stopAppServices.bind(null, app),
+        services.stopAppServices.bind(null, app),
+
+        progressCallback.bind(null, { percent: 80, message: 'Removing collectd profile' }),
+        removeCollectdProfile.bind(null, app),
 
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
@@ -976,7 +992,7 @@ function uninstall(app, args, progressCallback, callback) {
         deleteContainers.bind(null, app, {}),
 
         progressCallback.bind(null, { percent: 30, message: 'Teardown addons' }),
-        addons.teardownAddons.bind(null, app, app.manifest.addons),
+        services.teardownAddons.bind(null, app, app.manifest.addons),
 
         progressCallback.bind(null, { percent: 40, message: 'Cleanup file manager' }),
 
@@ -987,7 +1003,7 @@ function uninstall(app, args, progressCallback, callback) {
         docker.deleteImage.bind(null, app.manifest),
 
         progressCallback.bind(null, { percent: 70, message: 'Unregistering domains' }),
-        unregisterSubdomains.bind(null, app, [ { subdomain: app.location, domain: app.domain } ].concat(app.alternateDomains)),
+        unregisterSubdomains.bind(null, app, [ { subdomain: app.location, domain: app.domain } ].concat(app.alternateDomains).concat(app.aliasDomains)),
 
         progressCallback.bind(null, { percent: 80, message: 'Cleanup icon' }),
         removeIcon.bind(null, app),

src/apptaskmanager.js

@@ -1,7 +1,7 @@
 'use strict';
 
 exports = module.exports = {
-    scheduleTask: scheduleTask
+    scheduleTask
 };
 
 let assert = require('assert'),
@@ -12,7 +12,8 @@ let assert = require('assert'),
     safe = require('safetydance'),
     path = require('path'),
     paths = require('./paths.js'),
-    sftp = require('./sftp.js'),
+    scheduler = require('./scheduler.js'),
+    services = require('./services.js'),
     tasks = require('./tasks.js');
 
 let gActiveTasks = { }; // indexed by app id
@@ -36,9 +37,10 @@ function initializeSync() {
 }
 
 // callback is called when task is finished
-function scheduleTask(appId, taskId, callback) {
+function scheduleTask(appId, taskId, options, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof taskId, 'string');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     if (!gInitialized) initializeSync();
@@ -69,17 +71,17 @@ function scheduleTask(appId, taskId, callback) {
 
     if (!fs.existsSync(path.dirname(logFile))) safe.fs.mkdirSync(path.dirname(logFile)); // ensure directory
 
-    // TODO: set memory limit for app backup task
-    tasks.startTask(taskId, { logFile, timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15 }, function (error, result) {
+    scheduler.suspendJobs(appId);
+
+    tasks.startTask(taskId, Object.assign(options, { logFile }), function (error, result) {
         callback(error, result);
 
         delete gActiveTasks[appId];
         locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
 
         // post app task hooks
-        sftp.rebuild(function (error) {
-            if (error) debug('Unable to rebuild sftp:', error);
-        });
+        services.rebuildService('sftp', error => { if (error) debug('Unable to rebuild sftp:', error); });
+        scheduler.resumeJobs(appId);
     });
 }
 

src/backups.js

@@ -1,36 +1,36 @@
 'use strict';
 
 exports = module.exports = {
-    testConfig: testConfig,
-    testProviderConfig: testProviderConfig,
+    testConfig,
+    testProviderConfig,
 
     getByIdentifierAndStatePaged,
 
-    get: get,
+    get,
 
-    startBackupTask: startBackupTask,
+    startBackupTask,
 
-    restore: restore,
+    restore,
 
-    backupApp: backupApp,
-    downloadApp: downloadApp,
+    backupApp,
+    downloadApp,
 
-    backupBoxAndApps: backupBoxAndApps,
+    backupBoxAndApps,
 
-    upload: upload,
+    upload,
 
-    startCleanupTask: startCleanupTask,
-    cleanup: cleanup,
-    cleanupCacheFilesSync: cleanupCacheFilesSync,
+    startCleanupTask,
+    cleanup,
+    cleanupCacheFilesSync,
 
-    injectPrivateFields: injectPrivateFields,
-    removePrivateFields: removePrivateFields,
+    injectPrivateFields,
+    removePrivateFields,
 
-    checkConfiguration: checkConfiguration,
+    checkConfiguration,
 
-    configureCollectd: configureCollectd,
+    configureCollectd,
 
-    generateEncryptionKeysSync: generateEncryptionKeysSync,
+    generateEncryptionKeysSync,
 
     BACKUP_IDENTIFIER_BOX: 'box',
 
@@ -48,8 +48,7 @@ exports = module.exports = {
     _applyBackupRetentionPolicy: applyBackupRetentionPolicy
 };
 
-var addons = require('./addons.js'),
-    apps = require('./apps.js'),
+const apps = require('./apps.js'),
     async = require('async'),
     assert = require('assert'),
     backupdb = require('./backupdb.js'),
@@ -72,6 +71,7 @@ var addons = require('./addons.js'),
     progressStream = require('progress-stream'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     syncer = require('./syncer.js'),
     tar = require('tar-fs'),
@@ -1045,7 +1045,7 @@ function snapshotApp(app, progressCallback, callback) {
         return callback(new BoxError(BoxError.FS_ERROR, 'Error creating config.json: ' + safe.error.message));
     }
 
-    addons.backupAddons(app, app.manifest.addons, function (error) {
+    services.backupAddons(app, app.manifest.addons, function (error) {
         if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
 
         debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`);
@@ -1175,6 +1175,8 @@ function backupApp(app, options, progressCallback, callback) {
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
+    if (options.snapshotOnly) return snapshotApp(app, progressCallback, callback);
+
     const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
 
     debug(`backupApp - Backing up ${app.fqdn} with tag ${tag}`);

src/cert/acme2.js

@@ -139,7 +139,7 @@ Acme2.prototype.updateContact = function (registrationUri, callback) {
     const that = this;
     this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 200, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         debug(`updateContact: contact of user updated to ${that.email}`);
 
@@ -160,7 +160,7 @@ Acme2.prototype.registerUser = function (callback) {
     this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
         // 200 if already exists. 201 for new accounts
-        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register new account. Expecting 200 or 201, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         debug(`registerUser: user registered keyid: ${result.headers.location}`);
 
@@ -185,8 +185,8 @@ Acme2.prototype.newOrder = function (domain, callback) {
 
     this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
-        if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending signed request: ${result.body.detail}`));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         debug('newOrder: created order %s %j', domain, result.body);
 
@@ -259,7 +259,7 @@ Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
 
     this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 200, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         callback();
     });
@@ -313,7 +313,7 @@ Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, cal
     this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
         // 429 means we reached the cert limit for this domain
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 200, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         return callback(null);
     });
@@ -362,7 +362,7 @@ Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
         that.postAsGet(certUrl, function (error, result) {
             if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
             if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
-            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
             const fullChainPem = result.body; // buffer
 
@@ -586,34 +586,34 @@ Acme2.prototype.getDirectory = function (callback) {
     });
 };
 
-Acme2.prototype.getCertificate = function (hostname, domain, callback) {
-    assert.strictEqual(typeof hostname, 'string');
+Acme2.prototype.getCertificate = function (vhost, domain, callback) {
+    assert.strictEqual(typeof vhost, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`getCertificate: start acme flow for ${hostname} from ${this.caDirectory}`);
+    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
 
-    if (hostname !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
-        hostname = domains.makeWildcard(hostname);
-        debug(`getCertificate: will get wildcard cert for ${hostname}`);
+    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
+        vhost = domains.makeWildcard(vhost);
+        debug(`getCertificate: will get wildcard cert for ${vhost}`);
     }
 
     const that = this;
     this.getDirectory(function (error) {
         if (error) return callback(error);
 
-        that.acmeFlow(hostname, domain, function (error) {
+        that.acmeFlow(vhost, domain, function (error) {
             if (error) return callback(error);
 
             var outdir = paths.APP_CERTS_DIR;
-            const certName = hostname.replace('*.', '_.');
+            const certName = vhost.replace('*.', '_.');
             callback(null, path.join(outdir, `${certName}.cert`), path.join(outdir, `${certName}.key`));
         });
     });
 };
 
-function getCertificate(hostname, domain, options, callback) {
-    assert.strictEqual(typeof hostname, 'string');
+function getCertificate(vhost, domain, options, callback) {
+    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -623,6 +623,6 @@ function getCertificate(hostname, domain, options, callback) {
         debug(`getCertificate: attempt ${attempt++}`);
 
         let acme = new Acme2(options || { });
-        acme.getCertificate(hostname, domain, retryCallback);
+        acme.getCertificate(vhost, domain, retryCallback);
     }, callback);
 }

src/cloudron.js

@@ -21,8 +21,7 @@ exports = module.exports = {
     runSystemChecks
 };
 
-var addons = require('./addons.js'),
-    apps = require('./apps.js'),
+const apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
     async = require('async'),
@@ -43,6 +42,7 @@ var addons = require('./addons.js'),
     platform = require('./platform.js'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     spawn = require('child_process').spawn,
@@ -350,7 +350,7 @@ function updateDashboardDomain(domain, auditSource, callback) {
     setDashboardDomain(domain, auditSource, function (error) {
         if (error) return callback(error);
 
-        addons.rebuildService('turn', NOOP_CALLBACK); // to update the realm variable
+        services.rebuildService('turn', NOOP_CALLBACK); // to update the realm variable
 
         callback(null);
     });

src/collectd/volume.ejs

@@ -0,0 +1,9 @@
+<Plugin python>
+  <Module du>
+    <Path>
+        Instance "<%= volumeId %>"
+        Dir "<%= hostPath %>"
+    </Path>
+  </Module>
+</Plugin>
+

src/constants.js

@@ -26,7 +26,7 @@ exports = module.exports = {
 
     PORT: CLOUDRON ? 3000 : 5454,
     INTERNAL_SMTP_PORT: 2525, // this value comes from the mail container
-    SYSADMIN_PORT: 3001, // unused
+    AUTHWALL_PORT: 3001,
     LDAP_PORT: 3002,
     DOCKER_PROXY_PORT: 3003,
 
@@ -37,7 +37,14 @@ exports = module.exports = {
     DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024), // see also client.js
 
     DEMO_USERNAME: 'cloudron',
-    DEMO_BLACKLISTED_APPS: [ 'com.github.cloudtorrent', 'net.alltubedownload.cloudronapp' ],
+    DEMO_BLACKLISTED_APPS: [
+        'com.github.cloudtorrent',
+        'net.alltubedownload.cloudronapp',
+        'com.adguard.home.cloudronapp',
+        'com.transmissionbt.cloudronapp',
+        'io.github.sickchill.cloudronapp',
+        'to.couchpota.cloudronapp'
+    ],
 
     AUTOUPDATE_PATTERN_NEVER: 'never',
 
@@ -50,6 +57,6 @@ exports = module.exports = {
 
     FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
 
-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '5.1.1-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '6.0.1-test'
 };
 

src/dns/linode.js

@@ -119,9 +119,10 @@ function get(domainObject, location, type, callback) {
         zoneName = domainObject.zoneName,
         name = domains.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, { records }) {
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
 
+        const { records } = result;
         var tmp = records.map(function (record) { return record.target; });
 
         debug('get: %j', tmp);
@@ -143,9 +144,10 @@ function upsert(domainObject, location, type, values, callback) {
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, { zoneId, records }) {
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
 
+        const { zoneId, records } = result;
         let i = 0, recordIds = []; // used to track available records to update instead of create
 
         async.eachSeries(values, function (value, iteratorCallback) {
@@ -222,9 +224,10 @@ function del(domainObject, location, type, values, callback) {
         zoneName = domainObject.zoneName,
         name = domains.getName(domainObject, location, type) || '';
 
-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, { zoneId, records }) {
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
 
+        const { zoneId, records } = result;
         if (records.length === 0) return callback(null);
 
         var tmp = records.filter(function (record) { return values.some(function (value) { return value === record.target; }); });
@@ -287,7 +290,7 @@ function verifyDnsConfig(domainObject, callback) {
         if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
 
         if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) {
-            debug('verifyDnsConfig: %j does not contains DO NS', nameservers);
+            debug('verifyDnsConfig: %j does not contains linode NS', nameservers);
             return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode', { field: 'nameservers' }));
         }
 

src/dns/netcup.js

@@ -0,0 +1,303 @@
+'use strict';
+
+exports = module.exports = {
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    constants = require('../constants.js'),
+    debug = require('debug')('box:dns/netcup'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
+    superagent = require('superagent'),
+    util = require('util'),
+    waitForDns = require('./waitfordns.js');
+
+var API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON';
+
+function formatError(response) {
+    if (response.body) return util.format('Netcup DNS error [%s] %s', response.body.statuscode, response.body.longmessage);
+    else return util.format('Netcup DNS error [%s] %s', response.statusCode, response.text);
+}
+
+function removePrivateFields(domainObject) {
+    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    return domainObject;
+}
+
+function injectPrivateFields(newConfig, currentConfig) {
+    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+}
+
+// returns a api session id
+function login(dnsConfig, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const data = {
+        action: 'login',
+        param:{
+            apikey: dnsConfig.apiKey,
+            apipassword: dnsConfig.apiPassword,
+            customernumber: dnsConfig.customerNumber
+        }
+    };
+
+    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+        callback(null, result.body.responsedata.apisessionid);
+    });
+}
+
+function getAllRecords(dnsConfig, apiSessionId, zoneName, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof apiSessionId, 'string');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`getAllRecords: getting dns records of ${zoneName}`);
+
+    const data = {
+        action: 'infoDnsRecords',
+        param:{
+            apikey: dnsConfig.apiKey,
+            apisessionid: apiSessionId,
+            customernumber: dnsConfig.customerNumber,
+            domainname: zoneName,
+        }
+    };
+
+    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+        debug('getAllRecords:', JSON.stringify(result.body.responsedata.dnsrecords || []));
+
+        callback(null, result.body.responsedata.dnsrecords || []);
+    });
+}
+
+function upsert(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '@';
+
+    debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
+
+    login(dnsConfig, function (error, apiSessionId) {
+        if (error) return callback(error);
+
+        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
+            if (error) return callback(error);
+
+            let records = [];
+
+            values.forEach(function (value) {
+                // remove possible quotation
+                if (value.charAt(0) === '"') value = value.slice(1);
+                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
+
+                let priority = null;
+                if (type === 'MX') {
+                    priority = parseInt(value.split(' ')[0], 10);
+                    value = value.split(' ')[1];
+                }
+
+                let record = result.find(function (r) { return r.hostname === name && r.type === type; });
+                if (!record) record = { hostname: name, type: type, destination: value, deleterecord: false };
+                else record.destination = value;
+
+                if (priority !== null) record.priority = priority;
+
+                records.push(record);
+            });
+
+            const data = {
+                action: 'updateDnsRecords',
+                param:{
+                    apikey: dnsConfig.apiKey,
+                    apisessionid: apiSessionId,
+                    customernumber: dnsConfig.customerNumber,
+                    domainname: zoneName,
+                    dnsrecordset: {
+                        dnsrecords: records
+                    }
+                }
+            };
+
+            debug('upserting', JSON.stringify(data));
+
+            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                debug('upsert:', result.body);
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function get(domainObject, location, type, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '@';
+
+    debug('get: %s for zone %s of type %s', name, zoneName, type);
+
+    login(dnsConfig, function (error, apiSessionId) {
+        if (error) return callback(error);
+
+        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
+            if (error) return callback(error);
+
+            // We only return the value string
+            callback(null, result.filter(function (r) { return r.hostname === name && r.type === type; }).map(function (r) { return r.destination; }));
+        });
+    });
+}
+
+function del(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '@';
+
+    debug('del: %s for zone %s of type %s with values %j', name, zoneName, type, values);
+
+    login(dnsConfig, function (error, apiSessionId) {
+        if (error) return callback(error);
+
+        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
+            if (error) return callback(error);
+
+            let records = [];
+
+            values.forEach(function (value) {
+                // remove possible quotation
+                if (value.charAt(0) === '"') value = value.slice(1);
+                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
+
+                let record = result.find(function (r) { return r.hostname === name && r.type === type && r.destination === value; });
+                if (!record) return;
+
+                record.deleterecord = true;
+
+                records.push(record);
+            });
+
+            if (records.length === 0) return callback(null);
+
+            const data = {
+                action: 'updateDnsRecords',
+                param:{
+                    apikey: dnsConfig.apiKey,
+                    apisessionid: apiSessionId,
+                    customernumber: dnsConfig.customerNumber,
+                    domainname: zoneName,
+                    dnsrecordset: {
+                        dnsrecords: records
+                    }
+                }
+            };
+
+            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                debug('del:', result.body.responsedata);
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function wait(domainObject, location, type, value, options, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+    assert.strictEqual(typeof callback, 'function');
+
+    const fqdn = domains.fqdn(location, domainObject);
+
+    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+}
+
+function verifyDnsConfig(domainObject, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    if (!dnsConfig.customerNumber || typeof dnsConfig.customerNumber !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'customerNumber must be a non-empty string', { field: 'customerNumber' }));
+    if (!dnsConfig.apiKey || typeof dnsConfig.apiKey !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string', { field: 'apiKey' }));
+    if (!dnsConfig.apiPassword || typeof dnsConfig.apiPassword !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiPassword must be a non-empty string', { field: 'apiPassword' }));
+
+    const ip = '127.0.0.1';
+
+    var credentials = {
+        customerNumber: dnsConfig.customerNumber,
+        apiKey: dnsConfig.apiKey,
+        apiPassword: dnsConfig.apiPassword,
+    };
+
+    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+
+    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
+        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
+        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+
+        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) {
+            debug('verifyDnsConfig: %j does not contains Netcup NS', nameservers);
+            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup', { field: 'nameservers' }));
+        }
+
+        const location = 'cloudrontestdns';
+
+        upsert(domainObject, location, 'A', [ ip ], function (error) {
+            if (error) return callback(error);
+
+            debug('verifyDnsConfig: Test A record added');
+
+            del(domainObject, location, 'A', [ ip ], function (error) {
+                if (error) return callback(error);
+
+                debug('verifyDnsConfig: Test A record removed again');
+
+                callback(null, credentials);
+            });
+        });
+    });
+}

src/docker.js

@@ -1,37 +1,39 @@
 'use strict';
 
 exports = module.exports = {
-    testRegistryConfig: testRegistryConfig,
-    setRegistryConfig: setRegistryConfig,
-    injectPrivateFields: injectPrivateFields,
-    removePrivateFields: removePrivateFields,
+    testRegistryConfig,
+    setRegistryConfig,
+    injectPrivateFields,
+    removePrivateFields,
 
-    ping: ping,
+    ping,
 
-    info: info,
-    downloadImage: downloadImage,
-    createContainer: createContainer,
-    startContainer: startContainer,
-    restartContainer: restartContainer,
-    stopContainer: stopContainer,
+    info,
+    downloadImage,
+    createContainer,
+    startContainer,
+    restartContainer,
+    stopContainer,
     stopContainerByName: stopContainer,
-    stopContainers: stopContainers,
-    deleteContainer: deleteContainer,
-    deleteImage: deleteImage,
-    deleteContainers: deleteContainers,
-    createSubcontainer: createSubcontainer,
-    getContainerIdByIp: getContainerIdByIp,
-    inspect: inspect,
+    stopContainers,
+    deleteContainer,
+    deleteImage,
+    deleteContainers,
+    createSubcontainer,
+    getContainerIdByIp,
+    inspect,
+    getContainerIp,
     inspectByName: inspect,
-    execContainer: execContainer,
-    getEvents: getEvents,
-    memoryUsage: memoryUsage,
-    createVolume: createVolume,
-    removeVolume: removeVolume,
-    clearVolume: clearVolume
+    execContainer,
+    getEvents,
+    memoryUsage,
+    createVolume,
+    removeVolume,
+    clearVolume,
+    update
 };
 
-var addons = require('./addons.js'),
+const apps = require('./apps.js'),
     async = require('async'),
     assert = require('assert'),
     BoxError = require('./boxerror.js'),
@@ -39,10 +41,13 @@ var addons = require('./addons.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:docker'),
     Docker = require('dockerode'),
+    os = require('os'),
     path = require('path'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     safe = require('safetydance'),
+    system = require('./system.js'),
     util = require('util'),
     volumes = require('./volumes.js'),
     _ = require('underscore');
@@ -208,6 +213,18 @@ function getBinds(app, callback) {
     });
 }
 
+function getLowerUpIp() { // see getifaddrs and IFF_LOWER_UP and netdevice
+    const ni = os.networkInterfaces(); // { lo: [], eth0: [] }
+    for (const iname of Object.keys(ni)) {
+        if (iname === 'lo') continue;
+        for (const address of ni[iname]) {
+            if (!address.internal && address.family === 'IPv4') return address.address;
+        }
+    }
+
+    return null;
+}
+
 function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof name, 'string');
@@ -233,11 +250,6 @@ function createSubcontainer(app, name, cmd, options, callback) {
         `${envPrefix}APP_DOMAIN=${domain}`
     ];
 
-    // docker portBindings requires ports to be exposed
-    exposedPorts[manifest.httpPort + '/tcp'] = {};
-
-    dockerPortBindings[manifest.httpPort + '/tcp'] = [ { HostIp: '127.0.0.1', HostPort: app.httpPort + '' } ];
-
     var portEnv = [];
     for (let portName in app.portBindings) {
         const hostPort = app.portBindings[portName];
@@ -246,29 +258,24 @@ function createSubcontainer(app, name, cmd, options, callback) {
 
         var containerPort = ports[portName].containerPort || hostPort;
 
+        // docker portBindings requires ports to be exposed
         exposedPorts[`${containerPort}/${portType}`] = {};
         portEnv.push(`${portName}=${hostPort}`);
 
-        dockerPortBindings[`${containerPort}/${portType}`] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
+        const hostIp = hostPort === 53 ? getLowerUpIp() : '0.0.0.0'; // port 53 is special because it is possibly taken by systemd-resolved
+        dockerPortBindings[`${containerPort}/${portType}`] = [ { HostIp: hostIp, HostPort: hostPort + '' } ];
     }
 
     let appEnv = [];
     Object.keys(app.env).forEach(function (name) { appEnv.push(`${name}=${app.env[name]}`); });
 
-    // first check db record, then manifest
-    var memoryLimit = app.memoryLimit || manifest.memoryLimit || 0;
-
-    if (memoryLimit === -1) { // unrestricted
-        memoryLimit = 0;
-    } else if (memoryLimit === 0 || memoryLimit < constants.DEFAULT_MEMORY_LIMIT) { // ensure we never go below minimum (in case we change the default)
-        memoryLimit = constants.DEFAULT_MEMORY_LIMIT;
-    }
+    let memoryLimit = apps.getMemoryLimit(app);
 
     // give scheduler tasks twice the memory limit since background jobs take more memory
     // if required, we can make this a manifest and runtime argument later
     if (!isAppContainer) memoryLimit *= 2;
 
-    addons.getEnvironment(app, function (error, addonEnv) {
+    services.getEnvironment(app, function (error, addonEnv) {
         if (error) return callback(error);
 
         getBinds(app, function (error, binds) {
@@ -292,7 +299,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                     'isCloudronManaged': String(true)
                 },
                 HostConfig: {
-                    Mounts: addons.getMountsSync(app, app.manifest.addons),
+                    Mounts: services.getMountsSync(app, app.manifest.addons),
                     Binds: binds, // ideally, we have to use 'Mounts' but we have to create volumes then
                     LogConfig: {
                         Type: 'syslog',
@@ -302,7 +309,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                             'syslog-format': 'rfc5424'
                         }
                     },
-                    Memory: memoryLimit / 2,
+                    Memory: system.getMemoryAllocation(memoryLimit),
                     MemorySwap: memoryLimit, // Memory + Swap
                     PortBindings: isAppContainer ? dockerPortBindings : { },
                     PublishAllPorts: false,
@@ -334,12 +341,15 @@ function createSubcontainer(app, name, cmd, options, callback) {
                 containerOptions.NetworkingConfig = {
                     EndpointsConfig: {
                         cloudron: {
+                            IPAMConfig: {
+                                IPv4Address: app.containerIp
+                            },
                             Aliases: [ name ] // adds hostname entry with container name
                         }
                     }
                 };
             } else {
-                containerOptions.HostConfig.NetworkMode = `container:${app.containerId}`;
+                containerOptions.HostConfig.NetworkMode = `container:${app.containerId}`; // scheduler containers must have same IP as app for various addon auth
             }
 
             var capabilities = manifest.capabilities || [];
@@ -546,6 +556,22 @@ function inspect(containerId, callback) {
     });
 }
 
+function getContainerIp(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (constants.TEST) return callback(null, '127.0.5.5');
+
+    inspect(containerId, function (error, result) {
+        if (error) return callback(error);
+
+        const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
+        if (!ip) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error getting container IP'));
+
+        callback(null, ip);
+    });
+}
+
 function execContainer(containerId, options, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof options, 'object');
@@ -668,3 +694,17 @@ function info(callback) {
         callback(null, result);
     });
 }
+
+function update(name, memory, memorySwap, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof memory, 'number');
+    assert.strictEqual(typeof memorySwap, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    const args = `update --memory ${memory} --memory-swap ${memorySwap} ${name}`.split(' ');
+    // scale back db containers, if possible. this is retried because updating memory constraints can fail
+    // with failed to write to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/xx/memory.memsw.limit_in_bytes: device or resource busy
+    async.retry({ times: 10, interval: 60 * 1000 }, function (retryCallback) {
+        shell.spawn(`update(${name})`, '/usr/bin/docker', args, { }, retryCallback);
+    }, callback);
+}

src/domaindb.js

@@ -16,14 +16,18 @@ var assert = require('assert'),
     database = require('./database.js'),
     safe = require('safetydance');
 
-var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson' ].join(',');
+var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson' ].join(',');
 
 function postProcess(data) {
     data.config = safe.JSON.parse(data.configJson);
-    data.tlsConfig = safe.JSON.parse(data.tlsConfigJson);
     delete data.configJson;
+
+    data.tlsConfig = safe.JSON.parse(data.tlsConfigJson);
     delete data.tlsConfigJson;
 
+    data.wellKnown = safe.JSON.parse(data.wellKnownJson);
+    delete data.wellKnownJson;
+
     return data;
 }
 
@@ -86,6 +90,9 @@ function update(name, domain, callback) {
         } else if (k === 'tlsConfig') {
             fields.push('tlsConfigJson = ?');
             args.push(JSON.stringify(domain[k]));
+        } else if (k === 'wellKnown') {
+            fields.push('wellKnownJson = ?');
+            args.push(JSON.stringify(domain[k]));
         } else {
             fields.push(k + ' = ?');
             args.push(domain[k]);

src/domains.js

@@ -1,32 +1,32 @@
 'use strict';
 
 module.exports = exports = {
-    add: add,
-    get: get,
-    getAll: getAll,
-    update: update,
-    del: del,
-    clear: clear,
+    add,
+    get,
+    getAll,
+    update,
+    del,
+    clear,
 
-    fqdn: fqdn,
-    getName: getName,
+    fqdn,
+    getName,
 
-    getDnsRecords: getDnsRecords,
-    upsertDnsRecords: upsertDnsRecords,
-    removeDnsRecords: removeDnsRecords,
+    getDnsRecords,
+    upsertDnsRecords,
+    removeDnsRecords,
 
-    waitForDnsRecord: waitForDnsRecord,
+    waitForDnsRecord,
 
-    removePrivateFields: removePrivateFields,
-    removeRestrictedFields: removeRestrictedFields,
+    removePrivateFields,
+    removeRestrictedFields,
 
-    validateHostname: validateHostname,
+    validateHostname,
 
-    makeWildcard: makeWildcard,
+    makeWildcard,
 
-    parentDomain: parentDomain,
+    parentDomain,
 
-    checkDnsRecords: checkDnsRecords
+    checkDnsRecords
 };
 
 var assert = require('assert'),
@@ -60,6 +60,7 @@ function api(provider) {
     case 'linode': return require('./dns/linode.js');
     case 'namecom': return require('./dns/namecom.js');
     case 'namecheap': return require('./dns/namecheap.js');
+    case 'netcup': return require('./dns/netcup.js');
     case 'noop': return require('./dns/noop.js');
     case 'manual': return require('./dns/manual.js');
     case 'wildcard': return require('./dns/wildcard.js');
@@ -152,6 +153,12 @@ function validateTlsConfig(tlsConfig, dnsProvider) {
     return null;
 }
 
+function validateWellKnown(wellKnown) {
+    assert.strictEqual(typeof wellKnown, 'object');
+
+    return null;
+}
+
 function add(domain, data, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof data.zoneName, 'string');
@@ -178,7 +185,7 @@ function add(domain, data, auditSource, callback) {
         if (error) return callback(error);
     } else {
         fallbackCertificate = reverseProxy.generateFallbackCertificateSync({ domain, config });
-        if (fallbackCertificate.error) return callback(error);
+        if (fallbackCertificate.error) return callback(fallbackCertificate.error);
     }
 
     let error = validateTlsConfig(tlsConfig, provider);
@@ -246,7 +253,7 @@ function update(domain, data, auditSource, callback) {
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
+    let { zoneName, provider, config, fallbackCertificate, tlsConfig, wellKnown } = data;
 
     if (settings.isDemo() && (domain === settings.adminDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
 
@@ -267,6 +274,9 @@ function update(domain, data, auditSource, callback) {
         error = validateTlsConfig(tlsConfig, provider);
         if (error) return callback(error);
 
+        error = validateWellKnown(wellKnown, provider);
+        if (error) return callback(error);
+
         if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config);
 
         verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
@@ -274,9 +284,10 @@ function update(domain, data, auditSource, callback) {
 
             let newData = {
                 config: sanitizedConfig,
-                zoneName: zoneName,
-                provider: provider,
-                tlsConfig: tlsConfig
+                zoneName,
+                provider,
+                tlsConfig,
+                wellKnown
             };
 
             domaindb.update(domain, newData, function (error) {
@@ -431,7 +442,7 @@ function waitForDnsRecord(location, domain, type, value, options, callback) {
 
 // removes all fields that are strictly private and should never be returned by API calls
 function removePrivateFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate');
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate', 'wellKnown');
     return api(result.provider).removePrivateFields(result);
 }
 
@@ -444,10 +455,11 @@ function removeRestrictedFields(domain) {
     return result;
 }
 
-function makeWildcard(hostname) {
-    assert.strictEqual(typeof hostname, 'string');
+function makeWildcard(vhost) {
+    assert.strictEqual(typeof vhost, 'string');
 
-    let parts = hostname.split('.');
+    // if the vhost is like *.example.com, this function will do nothing
+    let parts = vhost.split('.');
     parts[0] = '*';
     return parts.join('.');
 }

src/eventlog.js

@@ -1,11 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    add: add,
-    get: get,
-    getAllPaged: getAllPaged,
-    getByCreationTime: getByCreationTime,
-    cleanup: cleanup,
+    add,
+    upsert,
+    get,
+    getAllPaged,
+    getByCreationTime,
+    cleanup,
 
     // keep in sync with webadmin index.js filter
     ACTION_ACTIVATE: 'cloudron.activate',
@@ -57,6 +58,7 @@ exports = module.exports = {
 
     ACTION_USER_ADD: 'user.add',
     ACTION_USER_LOGIN: 'user.login',
+    ACTION_USER_LOGOUT: 'user.logout',
     ACTION_USER_REMOVE: 'user.remove',
     ACTION_USER_UPDATE: 'user.update',
     ACTION_USER_TRANSFER: 'user.transfer',
@@ -90,9 +92,24 @@ function add(action, source, data, callback) {
 
     callback = callback || NOOP_CALLBACK;
 
-    // we do only daily upserts for login actions, so they don't spam the db
-    var api = action === exports.ACTION_USER_LOGIN ? eventlogdb.upsert : eventlogdb.add;
-    api(uuid.v4(), action, source, data, function (error, id) {
+    eventlogdb.add(uuid.v4(), action, source, data, function (error, id) {
+        if (error) return callback(error);
+
+        callback(null, { id: id });
+
+        notifications.onEvent(id, action, source, data, NOOP_CALLBACK);
+    });
+}
+
+function upsert(action, source, data, callback) {
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert(!callback || typeof callback === 'function');
+
+    callback = callback || NOOP_CALLBACK;
+
+    eventlogdb.upsert(uuid.v4(), action, source, data, function (error, id) {
         if (error) return callback(error);
 
         callback(null, { id: id });

src/graphite.js

@@ -1,23 +1,27 @@
 'use strict';
 
 exports = module.exports = {
-    startGraphite: startGraphite
+    start,
+
+    DEFAULT_MEMORY_LIMIT: 256 * 1024 * 1024
 };
 
 var assert = require('assert'),
     async = require('async'),
     infra = require('./infra_version.js'),
     paths = require('./paths.js'),
-    shell = require('./shell.js');
+    shell = require('./shell.js'),
+    system = require('./system.js');
 
-function startGraphite(existingInfra, callback) {
+function start(existingInfra, serviceConfig, callback) {
     assert.strictEqual(typeof existingInfra, 'object');
+    assert.strictEqual(typeof serviceConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     const tag = infra.images.graphite.tag;
     const dataDir = paths.PLATFORM_DATA_DIR;
-
-    if (existingInfra.version === infra.version && infra.images.graphite.tag === existingInfra.images.graphite.tag) return callback();
+    const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT;
+    const memory = system.getMemoryAllocation(memoryLimit);
 
     const cmd = `docker run --restart=always -d --name="graphite" \
                 --hostname graphite \
@@ -27,8 +31,8 @@ function startGraphite(existingInfra, callback) {
                 --log-opt syslog-address=udp://127.0.0.1:2514 \
                 --log-opt syslog-format=rfc5424 \
                 --log-opt tag=graphite \
-                -m 150m \
-                --memory-swap 150m \
+                -m ${memory} \
+                --memory-swap ${memoryLimit} \
                 --dns 172.18.0.1 \
                 --dns-search=. \
                 -p 127.0.0.1:2003:2003 \

src/groupdb.js

@@ -196,6 +196,7 @@ function setMembers(groupId, userIds, callback) {
 
     database.transaction(queries, function (error) {
         if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.CONFLICT, 'Duplicate member in list'));
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
         callback(error);
@@ -227,6 +228,7 @@ function setMembership(userId, groupIds, callback) {
 
     database.transaction(queries, function (error) {
         if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, error.message));
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.CONFLICT, 'Already member'));
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
         callback(null);

src/groups.js

@@ -1,25 +1,25 @@
 'use strict';
 
 exports = module.exports = {
-    create: create,
-    remove: remove,
-    get: get,
-    getByName: getByName,
-    update: update,
-    getWithMembers: getWithMembers,
-    getAll: getAll,
-    getAllWithMembers: getAllWithMembers,
+    create,
+    remove,
+    get,
+    getByName,
+    update,
+    getWithMembers,
+    getAll,
+    getAllWithMembers,
 
-    getMembers: getMembers,
-    addMember: addMember,
-    setMembers: setMembers,
-    removeMember: removeMember,
-    isMember: isMember,
+    getMembers,
+    addMember,
+    setMembers,
+    removeMember,
+    isMember,
 
-    setMembership: setMembership,
-    getMembership: getMembership,
+    setMembership,
+    getMembership,
 
-    count: count
+    count
 };
 
 var assert = require('assert'),

src/infra_version.js

@@ -6,22 +6,22 @@
 
 exports = module.exports = {
     // a version change recreates all containers with latest docker config
-    'version': '48.17.2',
+    'version': '48.18.0',
 
     'baseImages': [
-        { repo: 'cloudron/base-arm64', tag: 'cloudron/base-arm64:2.0.0@sha256:cc336184d5968636804951a0ab44f8d2c8cdd19b94f045753d312d81705b5806' }
+        { repo: 'cloudron/base', tag: 'cloudron/base:2.0.0@sha256:f9fea80513aa7c92fe2e7bf3978b54c8ac5222f47a9a32a7f8833edf0eb5a4f4' }
     ],
 
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'turn': { repo: 'cloudron/turn-arm64', tag: 'cloudron/turn-arm64:1.1.0@sha256:198577e2105fbc5f69588f84eda9ce27420e62c9fc957faddb037cd5708edf01' },
-        'mysql': { repo: 'cloudron/mysql-arm64', tag: 'cloudron/mysql-arm64:2.3.2@sha256:695670fa5438b0c7b44784995014a570165b6f6c990d9778c44e833b511a7bdc' },
-        'postgresql': { repo: 'cloudron/postgresql-arm64', tag: 'cloudron/postgresql-arm64:3.3.0@sha256:76b79a99dc968bc2de6e6b20f889e41584b7c671d0953ade9e13ebcea9a5b80b' },
-        'mongodb': { repo: 'cloudron/mongodb-arm64', tag: 'cloudron/mongodb-arm64:3.0.0@sha256:a57769b6d8f94c26548a019712edb261edf34c2bee68cb929287501d2cc6a10d' },
-        'redis': { repo: 'cloudron/redis-arm64', tag: 'cloudron/redis-arm64:2.3.0@sha256:674ba9135c2bc3b7ecd6791f4fa04cb7196c5b73997acf1f31df644978042d69' },
-        'mail': { repo: 'cloudron/mail-arm64', tag: 'cloudron/mail-arm64:2.10.0@sha256:acb67fbd1ad0346fd1634fa2673633a2e9f6088df2f854f3336390c7ceefd04d' },
-        'graphite': { repo: 'cloudron/graphite-arm64', tag: 'cloudron/graphite-arm64:2.3.0@sha256:181c8c7ac0e9cd35cc93013e4e44b41de1b5c9f4454b1c0f8a8acde7080558e1' },
-        'sftp': { repo: 'cloudron/sftp-arm64', tag: 'cloudron/sftp-arm64:3.0.0@sha256:a77554569a039495b0fbdb1ef0265ade53a9f8be48d9346ed64291d72b9be6ac' }
+        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.2.0@sha256:4359aae80050a92bae3be30600fb93ef4dbaec6dc9254bda353c0b131a36f969' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.3.2@sha256:dd624870c7f8ba9b2759f93ce740d1e092a1ac4b2d6af5007a01b30ad6b316d0' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:3.3.0@sha256:0daf1be5320c095077392bf21d247b93ceaddca46c866c17259a335c80d2f357' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:3.0.0@sha256:59e50b1f55e433ffdf6d678f8c658812b4119f631db8325572a52ee40d3bc562' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.3.0@sha256:0e31ec817e235b1814c04af97b1e7cf0053384aca2569570ce92bef0d95e94d2' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.1.0@sha256:18e0d75ad88a3e66849de2c4c01f794e8df9235befd74544838e34b65f487740' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.3.0@sha256:b7bc1ca4f4d0603a01369a689129aa273a938ce195fe43d00d42f4f2d5212f50' },
+        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.0.0@sha256:7e0165f17789192fd4f92efb34aa373450fa859e3b502684b2b121a5582965bf' }
     }
 };

src/iputils.js

@@ -0,0 +1,35 @@
+'use strict';
+
+exports = module.exports = {
+    ipFromInt,
+    intFromIp
+};
+
+const assert = require('assert');
+
+function intFromIp(address) {
+    assert.strictEqual(typeof address, 'string');
+
+    const parts = address.split('.');
+
+    if (parts.length !== 4) return null;
+
+    return (parseInt(parts[0], 10) << (8*3)) & 0xFF000000 |
+           (parseInt(parts[1], 10) << (8*2)) & 0x00FF0000 |
+           (parseInt(parts[2], 10) << (8*1)) & 0x0000FF00 |
+           (parseInt(parts[3], 10) << (8*0)) & 0x000000FF;
+}
+
+function ipFromInt(input) {
+    assert.strictEqual(typeof input, 'number');
+
+    let output = [];
+
+    for (let i = 3; i >= 0; --i) {
+        const octet = (input >> (i*8)) & 0x000000FF;
+        output.push(octet);
+    }
+
+    return output.join('.');
+}
+

src/ldap.js

@@ -1,12 +1,11 @@
 'use strict';
 
 exports = module.exports = {
-    start: start,
-    stop: stop
+    start,
+    stop
 };
 
-var addons = require('./addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     async = require('async'),
@@ -14,11 +13,13 @@ var addons = require('./addons.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:ldap'),
     eventlog = require('./eventlog.js'),
+    groups = require('./groups.js'),
     ldap = require('ldapjs'),
     mail = require('./mail.js'),
     mailboxdb = require('./mailboxdb.js'),
     path = require('path'),
     safe = require('safetydance'),
+    services = require('./services.js'),
     users = require('./users.js');
 
 var gServer = null;
@@ -133,8 +134,8 @@ function userSearch(req, res, next) {
 
             var dn = ldap.parseDN('cn=' + user.id + ',ou=users,dc=cloudron');
 
-            var groups = [ GROUP_USERS_DN ];
-            if (users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) groups.push(GROUP_ADMINS_DN);
+            var memberof = [ GROUP_USERS_DN ];
+            if (users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) memberof.push(GROUP_ADMINS_DN);
 
             var displayName = user.displayName || user.username || ''; // displayName can be empty and username can be null
             var nameParts = displayName.split(' ');
@@ -155,7 +156,7 @@ function userSearch(req, res, next) {
                     givenName: firstName,
                     username: user.username,
                     samaccountname: user.username,      // to support ActiveDirectory clients
-                    memberof: groups
+                    memberof: memberof
                 }
             };
 
@@ -287,14 +288,14 @@ function mailboxSearch(req, res, next) {
     } else if (req.dn.rdns[0].attrs.domain) { // legacy ldap mailbox search for old sogo
         var domain = req.dn.rdns[0].attrs.domain.value.toLowerCase();
 
-        mailboxdb.listMailboxes(domain, 1, 1000, function (error, result) {
+        mailboxdb.listMailboxes(domain, 1, 1000, function (error, mailboxes) {
 
             if (error) return next(new ldap.OperationsError(error.toString()));
 
             var results = [];
 
             // send mailbox objects
-            result.forEach(function (mailbox) {
+            mailboxes.forEach(function (mailbox) {
                 var dn = ldap.parseDN(`cn=${mailbox.name}@${domain},domain=${domain},ou=mailboxes,dc=cloudron`);
 
                 var obj = {
@@ -328,7 +329,9 @@ function mailboxSearch(req, res, next) {
             async.eachSeries(mailboxes, function (mailbox, callback) {
                 var dn = ldap.parseDN(`cn=${mailbox.name}@${mailbox.domain},ou=mailboxes,dc=cloudron`);
 
-                users.get(mailbox.ownerId, function (error, userObject) {
+                let getFunc = mailbox.ownerType === mail.OWNERTYPE_USER ? users.get : groups.get;
+
+                getFunc(mailbox.ownerId, function (error, ownerObject) {
                     if (error) return callback(); // skip mailboxes with unknown owner
 
                     var obj = {
@@ -336,30 +339,26 @@ function mailboxSearch(req, res, next) {
                         attributes: {
                             objectclass: ['mailbox'],
                             objectcategory: 'mailbox',
-                            displayname: userObject.displayName,
+                            displayname: mailbox.ownerType === mail.OWNERTYPE_USER ? ownerObject.displayName : ownerObject.name,
                             cn: `${mailbox.name}@${mailbox.domain}`,
                             uid: `${mailbox.name}@${mailbox.domain}`,
                             mail: `${mailbox.name}@${mailbox.domain}`
                         }
                     };
 
-                    mailboxdb.getAliasesForName(mailbox.name, mailbox.domain, function (error, aliases) {
-                        if (error) return callback(error);
-
-                        aliases.forEach(function (a, idx) {
-                            obj.attributes['mail' + idx] = `${a.name}@${a.domain}`;
-                        });
-
-                        // ensure all filter values are also lowercase
-                        var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
-                        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
-                        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                            results.push(obj);
-                        }
-
-                        callback();
+                    mailbox.aliases.forEach(function (a, idx) {
+                        obj.attributes['mail' + idx] = `${a.name}@${a.domain}`;
                     });
+
+                    // ensure all filter values are also lowercase
+                    var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+                    if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+                    if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                        results.push(obj);
+                    }
+
+                    callback();
                 });
             }, function (error) {
                 if (error) return next(new ldap.OperationsError(error.toString()));
@@ -483,18 +482,42 @@ function authorizeUserForApp(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
     assert.strictEqual(typeof req.app, 'object');
 
-    apps.hasAccessTo(req.app, req.user, function (error, result) {
+    apps.hasAccessTo(req.app, req.user, function (error, hasAccess) {
         if (error) return next(new ldap.OperationsError(error.toString()));
 
         // we return no such object, to avoid leakage of a users existence
-        if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (!hasAccess) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: req.app.id }, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+        eventlog.upsert(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: req.app.id }, { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
         res.end();
     });
 }
 
+function verifyMailboxPassword(mailbox, password, callback) {
+    assert.strictEqual(typeof mailbox, 'object');
+    assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (mailbox.ownerType === mail.OWNERTYPE_USER) return users.verify(mailbox.ownerId, password, users.AP_MAIL /* identifier */, callback);
+
+    groups.getMembers(mailbox.ownerId, function (error, userIds) {
+        if (error) return callback(error);
+
+        let verifiedUser = null;
+        async.someSeries(userIds, function iterator(userId, iteratorDone) {
+            users.verify(userId, password, users.AP_MAIL /* identifier */, function (error, result) {
+                if (error) return iteratorDone(null, false);
+                verifiedUser = result;
+                iteratorDone(null, true);
+            });
+        }, function (error, result) {
+            if (!result) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            callback(null, verifiedUser);
+        });
+    });
+}
+
 function authenticateUserMailbox(req, res, next) {
     debug('user mailbox auth: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
 
@@ -514,12 +537,12 @@ function authenticateUserMailbox(req, res, next) {
             if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
             if (error) return next(new ldap.OperationsError(error.message));
 
-            users.verify(mailbox.ownerId, req.credentials || '', users.AP_MAIL, function (error, result) {
+            verifyMailboxPassword(mailbox, req.credentials || '', function (error, result) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
 
-                eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
+                eventlog.upsert(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
                 res.end();
             });
         });
@@ -549,11 +572,10 @@ function authenticateSftp(req, res, next) {
 }
 
 function loadSftpConfig(req, res, next) {
-    addons.getServicesConfig('sftp', function (error, service, servicesConfig) {
+    services.getServiceConfig('sftp', function (error, serviceConfig) {
         if (error) return next(new ldap.OperationsError(error.toString()));
 
-        const serviceConfig = servicesConfig['sftp'];
-        req.requireAdmin = 'requireAdmin' in serviceConfig ? serviceConfig.requireAdmin : true;
+        req.requireAdmin = serviceConfig.requireAdmin;
 
         next();
     });
@@ -607,16 +629,37 @@ function userSearchSftp(req, res, next) {
     });
 }
 
+function verifyAppMailboxPassword(addonId, username, password, callback) {
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof username, 'string');
+    assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const pattern = addonId === 'sendmail' ? 'MAIL_SMTP' : 'MAIL_IMAP';
+    appdb.getAppIdByAddonConfigValue(addonId, `%${pattern}_PASSWORD`, password, function (error, appId) { // search by password because this is unique for each app
+        if (error) return callback(error);
+
+        appdb.getAddonConfig(appId, addonId, function (error, result) {
+            if (error) return callback(error);
+
+            if (!result.some(r => r.name.endsWith(`${pattern}_USERNAME`) && r.value === username)) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+
+            callback(null);
+        });
+    });
+}
+
 function authenticateMailAddon(req, res, next) {
     debug('mail addon auth: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
 
     if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-    var email = req.dn.rdns[0].attrs.cn.value.toLowerCase();
-    var parts = email.split('@');
+    const email = req.dn.rdns[0].attrs.cn.value.toLowerCase();
+    const parts = email.split('@');
     if (parts.length !== 2) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
     const addonId = req.dn.rdns[1].attrs.ou.value.toLowerCase(); // 'sendmail' or 'recvmail'
+    if (addonId !== 'sendmail' && addonId !== 'recvmail') return next(new ldap.OperationsError('Invalid DN'));
 
     mail.getDomain(parts[1], function (error, domain) {
         if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
@@ -624,26 +667,22 @@ function authenticateMailAddon(req, res, next) {
 
         if (addonId === 'recvmail' && !domain.enabled) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-        let namePattern; // manifest v2 has a CLOUDRON_ prefix for names
-        if (addonId === 'sendmail') namePattern = '%MAIL_SMTP_PASSWORD';
-        else if (addonId === 'recvmail') namePattern = '%MAIL_IMAP_PASSWORD';
-        else return next(new ldap.OperationsError('Invalid DN'));
+        verifyAppMailboxPassword(addonId, email, req.credentials || '', function (error) {
+            if (!error) return res.end(); // validated as app
 
-        // note: with sendmail addon, apps can send mail without a mailbox (unlike users)
-        appdb.getAppIdByAddonConfigValue(addonId, namePattern, req.credentials || '', function (error, appId) {
+            if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
             if (error && error.reason !== BoxError.NOT_FOUND) return next(new ldap.OperationsError(error.message));
-            if (appId) return res.end();
 
             mailboxdb.getMailbox(parts[0], parts[1], function (error, mailbox) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
 
-                users.verify(mailbox.ownerId, req.credentials || '', users.AP_MAIL, function (error, result) {
+                verifyMailboxPassword(mailbox, req.credentials || '', function (error, result) {
                     if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                     if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                     if (error) return next(new ldap.OperationsError(error.message));
 
-                    eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
+                    eventlog.upsert(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
                     res.end();
                 });
             });
@@ -674,13 +713,13 @@ function start(callback) {
     gServer.bind('ou=users,dc=cloudron', authenticateApp, authenticateUser, authorizeUserForApp);
 
     // http://www.ietf.org/proceedings/43/I-D/draft-srivastava-ldap-mail-00.txt
-    gServer.search('ou=mailboxes,dc=cloudron', mailboxSearch); // haraka, dovecot
+    gServer.search('ou=mailboxes,dc=cloudron', mailboxSearch); // haraka (address translation), dovecot (LMTP), sogo (mailbox search)
     gServer.bind('ou=mailboxes,dc=cloudron', authenticateUserMailbox); // apps like sogo can use domain=${domain} to authenticate a mailbox
     gServer.search('ou=mailaliases,dc=cloudron', mailAliasSearch); // haraka
     gServer.search('ou=mailinglists,dc=cloudron', mailingListSearch); // haraka
 
-    gServer.bind('ou=recvmail,dc=cloudron', authenticateMailAddon); // dovecot
-    gServer.bind('ou=sendmail,dc=cloudron', authenticateMailAddon); // haraka
+    gServer.bind('ou=recvmail,dc=cloudron', authenticateMailAddon); // dovecot (IMAP auth)
+    gServer.bind('ou=sendmail,dc=cloudron', authenticateMailAddon); // haraka (MSA auth)
 
     gServer.bind('ou=sftp,dc=cloudron', authenticateSftp);    // sftp
     gServer.search('ou=sftp,dc=cloudron', loadSftpConfig, userSearchSftp);

src/mail.js

@@ -52,11 +52,16 @@ exports = module.exports = {
     removeList,
     resolveList,
 
+    OWNERTYPE_USER: 'user',
+    OWNERTYPE_GROUP: 'group',
+
+    DEFAULT_MEMORY_LIMIT: 512 * 1024 * 1024,
+
     _removeMailboxes: removeMailboxes,
     _readDkimPublicKeySync: readDkimPublicKeySync
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
     cloudron = require('./cloudron.js'),
@@ -75,12 +80,15 @@ var assert = require('assert'),
     nodemailer = require('nodemailer'),
     path = require('path'),
     paths = require('./paths.js'),
+    request = require('request'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     smtpTransport = require('nodemailer-smtp-transport'),
     sysinfo = require('./sysinfo.js'),
+    system = require('./system.js'),
     tasks = require('./tasks.js'),
     users = require('./users.js'),
     validator = require('validator'),
@@ -619,9 +627,10 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
     });
 }
 
-function configureMail(mailFqdn, mailDomain, callback) {
+function configureMail(mailFqdn, mailDomain, serviceConfig, callback) {
     assert.strictEqual(typeof mailFqdn, 'string');
     assert.strictEqual(typeof mailDomain, 'string');
+    assert.strictEqual(typeof serviceConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     // mail (note: 2525 is hardcoded in mail container and app use this port)
@@ -630,7 +639,8 @@ function configureMail(mailFqdn, mailDomain, callback) {
     // mail container uses /app/data for backed up data and /run for restart-able data
 
     const tag = infra.images.mail.tag;
-    const memoryLimit = 4 * 256;
+    const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT;
+    const memory = system.getMemoryAllocation(memoryLimit);
     const cloudronToken = hat(8 * 128), relayToken = hat(8 * 128);
 
     reverseProxy.getCertificate(mailFqdn, mailDomain, function (error, bundle) {
@@ -661,8 +671,8 @@ function configureMail(mailFqdn, mailDomain, callback) {
                             --log-opt syslog-address=udp://127.0.0.1:2514 \
                             --log-opt syslog-format=rfc5424 \
                             --log-opt tag=mail \
-                            -m ${memoryLimit}m \
-                            --memory-swap ${memoryLimit * 2}m \
+                            -m ${memory} \
+                            --memory-swap ${memoryLimit} \
                             --dns 172.18.0.1 \
                             --dns-search=. \
                             -e CLOUDRON_MAIL_TOKEN="${cloudronToken}" \
@@ -709,8 +719,12 @@ function restartMail(callback) {
 
     if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return callback();
 
-    debug(`restartMail: restarting mail container with ${settings.mailFqdn()} ${settings.adminDomain()}`);
-    configureMail(settings.mailFqdn(), settings.adminDomain(), callback);
+    services.getServiceConfig('mail', function (error, serviceConfig) {
+        if (error) return callback(error);
+
+        debug(`restartMail: restarting mail container with ${settings.mailFqdn()} ${settings.adminDomain()}`);
+        configureMail(settings.mailFqdn(), settings.adminDomain(), serviceConfig, callback);
+    });
 }
 
 function restartMailIfActivated(callback) {
@@ -763,7 +777,7 @@ function txtRecordsWithSpf(domain, mailFqdn, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     domains.getDnsRecords('', domain, 'TXT', function (error, txtRecords) {
-        if (error) return error;
+        if (error) return callback(error);
 
         debug('txtRecordsWithSpf: current txt records - %j', txtRecords);
 
@@ -934,7 +948,10 @@ function changeLocation(auditSource, progressCallback, callback) {
                 progressCallback({ percent: progress, message: `Updating DNS of ${domainObject.domain}` });
                 progress += Math.round(70/allDomains.length);
 
-                upsertDnsRecords(domainObject.domain, fqdn, iteratorDone);
+                upsertDnsRecords(domainObject.domain, fqdn, function (error) { // ignore any errors. we anyway report dns errors in status tab
+                    progressCallback({ percent: progress, message: `Updated DNS of ${domainObject.domain}: ${error ? error.message : 'success'}` });
+                    iteratorDone();
+                });
             }, function (error) {
                 if (error) return callback(error);
 
@@ -1162,10 +1179,11 @@ function getMailbox(name, domain, callback) {
     });
 }
 
-function addMailbox(name, domain, userId, auditSource, callback) {
+function addMailbox(name, domain, ownerId, ownerType, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -1174,31 +1192,53 @@ function addMailbox(name, domain, userId, auditSource, callback) {
     var error = validateName(name);
     if (error) return callback(error);
 
-    mailboxdb.addMailbox(name, domain, userId, function (error) {
+    if (ownerType !== exports.OWNERTYPE_USER && ownerType !== exports.OWNERTYPE_GROUP) return callback(new BoxError(BoxError.BAD_FIELD, 'bad owner type'));
+
+    mailboxdb.addMailbox(name, domain, ownerId, ownerType, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, userId });
+        eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType });
 
         callback(null);
     });
 }
 
-function updateMailboxOwner(name, domain, userId, auditSource, callback) {
+function updateMailboxOwner(name, domain, ownerId, ownerType, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     name = name.toLowerCase();
 
+    if (ownerType !== exports.OWNERTYPE_USER && ownerType !== exports.OWNERTYPE_GROUP) return callback(new BoxError(BoxError.BAD_FIELD, 'bad owner type'));
+
     getMailbox(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        mailboxdb.updateMailboxOwner(name, domain, userId, function (error) {
+        mailboxdb.updateMailboxOwner(name, domain, ownerId, ownerType, function (error) {
             if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: result.userId, userId });
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: result.userId, ownerId, ownerType });
+
+            callback(null);
+        });
+    });
+}
+
+function removeSolrIndex(mailbox, callback) {
+    assert.strictEqual(typeof mailbox, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
+        if (error) return callback(error);
+
+        request.post(`https://${addonDetails.ip}:3000/solr_delete_index?access_token=${addonDetails.token}`, { timeout: 2000, rejectUnauthorized: false, json: { mailbox } }, function (error, response) {
+            if (error) return callback(error);
+
+            if (response.statusCode !== 200) return callback(new Error(`Error removing solr index - ${response.statusCode} ${JSON.stringify(response.body)}`));
 
             callback(null);
         });
@@ -1212,7 +1252,8 @@ function removeMailbox(name, domain, options, auditSource, callback) {
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    const deleteMailFunc = options.deleteMails ? shell.sudo.bind(null, 'removeMailbox', [ REMOVE_MAILBOX, `${name}@${domain}` ], {}) : (next) => next();
+    const mailbox =`${name}@${domain}`;
+    const deleteMailFunc = options.deleteMails ? shell.sudo.bind(null, 'removeMailbox', [ REMOVE_MAILBOX, mailbox ], {}) : (next) => next();
 
     deleteMailFunc(function (error) {
         if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error removing mailbox: ${error.message}`));
@@ -1220,6 +1261,7 @@ function removeMailbox(name, domain, options, auditSource, callback) {
         mailboxdb.del(name, domain, function (error) {
             if (error) return callback(error);
 
+            removeSolrIndex(mailbox, NOOP_CALLBACK);
             eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
 
             callback();

src/mail_templates/app_updates_available.ejs

@@ -3,18 +3,14 @@
 Dear Cloudron Admin,
 
 <% for (var i = 0; i < apps.length; i++) { -%>
-A new version <%= apps[i].updateInfo.manifest.version %> of the app '<%= apps[i].app.manifest.title %>' installed at <%= apps[i].app.fqdn %> is available.
+The app '<%= apps[i].app.manifest.title %>' installed at <%= apps[i].app.fqdn %> has an update available.
 
-Changes:
+<%= apps[i].app.manifest.title %> v<%= apps[i].updateInfo.manifest.version %> changes:
 <%= apps[i].updateInfo.manifest.changelog %>
 
 <% } -%>
 
-<% if (!hasSubscription) { -%>
-*Keep your Cloudron automatically up-to-date and secure by upgrading to a paid plan at* <%= webadminUrl %>/#/settings
-<% } else { -%>
 Update now at <%= webadminUrl %>
-<% } -%>
 
 Powered by https://cloudron.io
 
@@ -33,24 +29,20 @@ Sent at: <%= new Date().toUTCString() %>
     <div style="width: 650px; text-align: left;">
 <% for (var i = 0; i < apps.length; i++) { -%>
         <p>
-            A new version <%= apps[i].updateInfo.manifest.version %> of the app '<%= apps[i].app.manifest.title %>' installed at <a href="https://<%= apps[i].app.fqdn %>"><%= apps[i].app.fqdn %></a> is available.
+            The app '<%= apps[i].app.manifest.title %>' installed at <a href="https://<%= apps[i].app.fqdn %>"><%= apps[i].app.fqdn %></a> has an update available.
         </p>
 
-        <h5>Changelog:</h5>
+        <h5><%= apps[i].app.manifest.title %> v<%= apps[i].updateInfo.manifest.version %> changes:</h5>
         <%- apps[i].changelogHTML %>
 
         <br/>
 <% } -%>
 
-<% if (!hasSubscription) { -%>
-        <p>Keep your Cloudron automatically up-to-date and secure by upgrading to a <a href="<%= webadminUrl %>/#/settings">paid plan</a>.</p>
-<% } else { -%>
         <p>
             <br/>
             <center><a href="<%= webadminUrl %>">Update now</a></center>
             <br/>
         </p>
-<% } -%>
     </div>
 
     <div style="font-size: 10px; color: #333333; background: #ffffff;">

src/mail_templates/box_update_available.ejs

@@ -0,0 +1,45 @@
+<%if (format === 'text') { %>
+
+Dear <%= cloudronName %> Admin,
+
+Cloudron v<%= newBoxVersion %> is now available!
+
+Changes:
+<% for (var i = 0; i < changelog.length; i++) { %>
+  * <%- changelog[i] %>
+<% } %>
+
+Powered by https://cloudron.io
+
+Sent at: <%= new Date().toUTCString() %>
+
+<% } else { %>
+
+<center>
+
+<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
+
+<h3>Dear <%= cloudronName %> Admin,</h3>
+
+<div style="width: 650px; text-align: left;">
+    <p>
+        Cloudron v<%= newBoxVersion %> is now available!
+    </p>
+
+    <h5>Changes:</h5>
+    <ul>
+    <% for (var i = 0; i < changelogHTML.length; i++) { %>
+        <li><%- changelogHTML[i] %></li>
+    <% } %>
+    </ul>
+
+    <br/>
+</div>
+
+<div style="font-size: 10px; color: #333333; background: #ffffff;">
+    Powered by <a href="https://cloudron.io">Cloudron</a>.
+</div>
+
+</center>
+
+<% } %>

src/mail_templates/oom_event.ejs

@@ -2,12 +2,13 @@
 
 Dear <%= cloudronName %> Admin,
 
-<%= program %> was restarted now as it ran out of memory.
-
-If this message appears repeatedly, give the app more memory.
-
-* To increase an app's memory limit - https://docs.cloudron.io/apps/#memory-limit
-* To increase a service's memory limit - https://docs.cloudron.io/troubleshooting/#services
+<%if (app) { %>
+The application at <%= app.fqdn %> ran out of memory. The application has been restarted automatically. If you see this notification often,
+consider increasing the memory limit - <%= webadminUrl %>/#/app/<%= app.id %>/resources .
+<% } else { %>
+The addon <%= addon.name %> service ran out of memory. The service has been restarted automatically. If you see this notification often,
+consider increasing the memory limit - <%= webadminUrl %>/#/services .
+<% } %>
 
 Out of memory event:
 

src/mail_templates/password_reset-html.ejs

@@ -0,0 +1,24 @@
+<center>
+
+<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
+
+<h3>{{ passwordResetEmail.salutation }}</h3>
+
+<p>{{ passwordResetEmail.description }}</p>
+
+<p>
+  <a href="<%= resetLink %>">{{ passwordResetEmail.resetAction }}</a>
+</p>
+
+<br/>
+
+{{ passwordResetEmail.expireNote }}
+
+<br/>
+<br/>
+
+<div style="font-size: 10px; color: #333333; background: #ffffff;">
+  Powered by <a href="https://cloudron.io">Cloudron</a>
+</div>
+
+</center>

src/mail_templates/password_reset-text.ejs

@@ -0,0 +1,9 @@
+{{ passwordResetEmail.salutation }}
+
+{{ passwordResetEmail.description }}
+
+{{ passwordResetEmail.resetActionText }}
+
+{{ passwordResetEmail.expireNote }}
+
+Powered by https://cloudron.io

src/mail_templates/password_reset.ejs

@@ -1,45 +0,0 @@
-<%if (format === 'text') { %>
-
-Hi <%= user.displayName || user.username || user.email %>,
-
-Someone, hopefully you, has requested your account's password
-be reset. If you did not request this reset, please ignore this message.
-
-To reset your password, please visit the following page:
-<%- resetLink %>
-
-Please note that the password reset link will expire in 24 hours.
-
-Powered by https://cloudron.io
-
-<% } else { %>
-
-<center>
-
-<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
-
-<h3>Hi <%= user.displayName || user.username || user.email %>,</h3>
-
-<p>
-    Someone, hopefully you, has requested your account's password be reset.<br/>
-    If you did not request this reset, please ignore this message.
-</p>
-
-<p>
-    <a href="<%= resetLink %>">Click to reset your password</a>
-</p>
-
-<br/>
-
-Please note that the password reset link will expire in 24 hours.
-
-<br/>
-<br/>
-
-<div style="font-size: 10px; color: #333333; background: #ffffff;">
-    Powered by <a href="https://cloudron.io">Cloudron</a>
-</div>
-
-</center>
-
-<% } %>

src/mail_templates/user_added.ejs

@@ -1,30 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear <%= cloudronName %> Admin,
-
-A new user with email <%= user.email %> was added to <%= cloudronName %>.
-
-Powered by https://cloudron.io
-
-<% } else { %>
-
-<center>
-
-<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
-
-<h3>Dear <%= cloudronName %> Admin,</h3>
-
-<p>
-    A new user with email <%= user.email %> was added to <%= cloudronName %>.
-</p>
-
-<br/>
-<br/>
-
-<div style="font-size: 10px; color: #333333; background: #ffffff;">
-    Powered by <a href="https://cloudron.io">Cloudron</a>.
-</div>
-
-</center>
-
-<% } %>

src/mail_templates/user_event.ejs

@@ -1,14 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear Cloudron Admin,
-
-User <%= user.username || user.email %> <%= event %>.
-
-
-Powered by https://cloudron.io
-
-Sent at: <%= new Date().toUTCString() %>
-
-<% } else { %>
-
-<% } %>

src/mail_templates/welcome_user-html.ejs

@@ -0,0 +1,28 @@
+<center>
+
+<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
+
+<h3>{{ welcomeEmail.salutation }}</h3>
+<h2>{{ welcomeEmail.welcomeTo }}</h2>
+
+<p>
+  <a href="<%= inviteLink %>">{{ welcomeEmail.inviteLinkAction }}</a>
+</p>
+
+<br/>
+<br/>
+
+<div style="font-size: 10px; color: #333333; background: #ffffff;">
+<% if (invitor) { -%>
+  {{ welcomeEmail.invitor }}
+<% } -%>
+
+  <br/>
+
+  {{ welcomeEmail.expireNote }}
+  <br/>
+
+  Powered by <a href="https://cloudron.io">Cloudron</a>
+</div>
+
+</center>

src/mail_templates/welcome_user-text.ejs

@@ -0,0 +1,13 @@
+{{ welcomeEmail.salutation }}
+
+{{ welcomeEmail.welcomeTo }}
+
+{{ welcomeEmail.inviteLinkActionText }}
+
+<% if (invitor) { %>
+{{ welcomeEmail.invitor }}
+<% } %>
+
+{{ welcomeEmail.expireNote }}
+
+Powered by https://cloudron.io

src/mail_templates/welcome_user.ejs

@@ -1,50 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear <%= user.displayName || user.username || user.email %>,
-
-Welcome to <%= cloudronName %>!
-
-Follow the link to get started.
-<%- inviteLink %>
-
-<% if (invitor && invitor.email) { %>
-You are receiving this email because you were invited by <%= invitor.email %>.
-<% } %>
-
-Please note that the invite link will expire in 7 days.
-
-Powered by https://cloudron.io
-
-<% } else { %>
-
-<center>
-
-<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
-
-<h3>Hi <%= user.displayName || user.username || user.email %>,</h3>
-
-<h2>Welcome to <%= cloudronName %>!</h2>
-
-<p>
-    <a href="<%= inviteLink %>">Get started</a>.
-</p>
-
-<br/>
-<br/>
-
-<div style="font-size: 10px; color: #333333; background: #ffffff;">
-    <% if (invitor && invitor.email) { %>
-        You are receiving this email because you were invited by <%= invitor.email %>.
-    <% } %>
-
-    <br/>
-
-        Please note that the invite link will expire in 7 days.
-    <br/>
-
-    Powered by <a href="https://cloudron.io">Cloudron</a>
-</div>
-
-</center>
-
-<% } %>

src/mailboxdb.js

@@ -42,7 +42,7 @@ var assert = require('assert'),
     safe = require('safetydance'),
     util = require('util');
 
-var MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain' ].join(',');
+var MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'ownerType', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain' ].join(',');
 
 function postProcess(data) {
     data.members = safe.JSON.parse(data.membersJson) || [ ];
@@ -53,13 +53,24 @@ function postProcess(data) {
     return data;
 }
 
-function addMailbox(name, domain, ownerId, callback) {
+function postProcessAliases(data) {
+    const aliasNames = JSON.parse(data.aliasNames), aliasDomains = JSON.parse(data.aliasDomains);
+    delete data.aliasNames;
+    delete data.aliasDomains;
+    data.aliases = [];
+    for (let i = 0; i < aliasNames.length; i++) { // NOTE: aliasNames is [ null ] when no aliases
+        if (aliasNames[i]) data.aliases[i] = { name: aliasNames[i], domain: aliasDomains[i] };
+    }
+}
+
+function addMailbox(name, domain, ownerId, ownerType, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('INSERT INTO mailboxes (name, type, domain, ownerId) VALUES (?, ?, ?, ?)', [ name, exports.TYPE_MAILBOX, domain, ownerId ], function (error) {
+    database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?)', [ name, exports.TYPE_MAILBOX, domain, ownerId, ownerType ], function (error) {
         if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists'));
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
@@ -67,13 +78,14 @@ function addMailbox(name, domain, ownerId, callback) {
     });
 }
 
-function updateMailboxOwner(name, domain, ownerId, callback) {
+function updateMailboxOwner(name, domain, ownerId, ownerType, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('UPDATE mailboxes SET ownerId = ? WHERE name = ? AND domain = ?', [ ownerId, name, domain ], function (error, result) {
+    database.query('UPDATE mailboxes SET ownerId = ?, ownerType = ? WHERE name = ? AND domain = ?', [ ownerId, ownerType, name, domain ], function (error, result) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
         if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
 
@@ -88,8 +100,8 @@ function addList(name, domain, members, membersOnly, callback) {
     assert.strictEqual(typeof membersOnly, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('INSERT INTO mailboxes (name, type, domain, ownerId, membersJson, membersOnly) VALUES (?, ?, ?, ?, ?, ?)',
-        [ name, exports.TYPE_LIST, domain, 'admin', JSON.stringify(members), membersOnly ], function (error) {
+    database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, membersJson, membersOnly) VALUES (?, ?, ?, ?, ?, ?, ?)',
+        [ name, exports.TYPE_LIST, domain, 'admin', 'user', JSON.stringify(members), membersOnly ], function (error) {
             if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists'));
             if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
@@ -223,14 +235,22 @@ function listMailboxes(domain, search, page, perPage, callback) {
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    let query = `SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? AND domain = ?`;
-    if (search) query += ' AND (name LIKE ' + mysql.escape('%' + search + '%') + ')';
-    query += 'ORDER BY name LIMIT ?,?';
+    const escapedSearch = mysql.escape('%' + search + '%'); // this also quotes the string
+    const searchQuery = search ? ` HAVING (name LIKE ${escapedSearch} OR aliasNames LIKE ${escapedSearch} OR aliasDomains LIKE ${escapedSearch})` : ''; // having instead of where because of aggregated columns use
 
-    database.query(query, [ exports.TYPE_MAILBOX, domain, (page-1)*perPage, perPage ], function (error, results) {
+    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains '
+        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
+        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
+        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
+        + ' WHERE m1.domain = ?'
+        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
+        + searchQuery
+        + ' ORDER BY name LIMIT ?,?';
+
+    database.query(query, [ domain, (page-1)*perPage, perPage ], function (error, results) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
-        results.forEach(function (result) { postProcess(result); });
+        results.forEach(postProcessAliases);
 
         callback(null, results);
     });
@@ -241,14 +261,20 @@ function listAllMailboxes(page, perPage, callback) {
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? ORDER BY name LIMIT ?,?`,
-        [ exports.TYPE_MAILBOX, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains '
+        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1` +
+        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2` +
+        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
+        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
+        + ' ORDER BY name LIMIT ?,?';
 
-            results.forEach(function (result) { postProcess(result); });
+    database.query(query, [ (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
-            callback(null, results);
-        });
+        results.forEach(postProcessAliases);
+
+        callback(null, results);
+    });
 }
 
 function getLists(domain, search, page, perPage, callback) {
@@ -314,8 +340,8 @@ function setAliasesForName(name, domain, aliases, callback) {
         // clear existing aliases
         queries.push({ query: 'DELETE FROM mailboxes WHERE aliasName = ? AND aliasDomain = ? AND type = ?', args: [ name, domain, exports.TYPE_ALIAS ] });
         aliases.forEach(function (alias) {
-            queries.push({ query: 'INSERT INTO mailboxes (name, domain, type, aliasName, aliasDomain, ownerId) VALUES (?, ?, ?, ?, ?, ?)',
-                args: [ alias.name, alias.domain, exports.TYPE_ALIAS, name, domain, results[0].ownerId ] });
+            queries.push({ query: 'INSERT INTO mailboxes (name, domain, type, aliasName, aliasDomain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?, ?, ?)',
+                args: [ alias.name, alias.domain, exports.TYPE_ALIAS, name, domain, results[0].ownerId, results[0].ownerType ] });
         });
 
         database.transaction(queries, function (error) {

src/mailer.js

@@ -1,25 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    userAdded: userAdded,
-    userRemoved: userRemoved,
-    roleChanged: roleChanged,
-    passwordReset: passwordReset,
-    appUpdatesAvailable: appUpdatesAvailable,
+    passwordReset,
+    boxUpdateAvailable,
+    appUpdatesAvailable,
 
-    sendInvite: sendInvite,
+    sendInvite,
 
-    appUp: appUp,
-    appDied: appDied,
-    appUpdated: appUpdated,
-    oomEvent: oomEvent,
+    appUp,
+    appDied,
+    appUpdated,
+    oomEvent,
 
-    backupFailed: backupFailed,
+    backupFailed,
 
-    certificateRenewalError: certificateRenewalError,
-    boxUpdateError: boxUpdateError,
+    certificateRenewalError,
+    boxUpdateError,
 
-    sendTestMail: sendTestMail,
+    sendTestMail,
 
     _mailQueue: [] // accumulate mails in test mode
 };
@@ -34,8 +32,8 @@ var assert = require('assert'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     showdown = require('showdown'),
-    smtpTransport = require('nodemailer-smtp-transport'),
-    util = require('util');
+    translation = require('./translation.js'),
+    smtpTransport = require('nodemailer-smtp-transport');
 
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
@@ -91,14 +89,21 @@ function sendMail(mailOptions, callback) {
     });
 }
 
-function render(templateFile, params) {
+function render(templateFile, params, translationAssets) {
     assert.strictEqual(typeof templateFile, 'string');
     assert.strictEqual(typeof params, 'object');
 
     var content = null;
+    var raw = safe.fs.readFileSync(path.join(MAIL_TEMPLATES_DIR, templateFile), 'utf8');
+    if (raw === null) {
+        debug(`Error loading ${templateFile}`);
+        return '';
+    }
+
+    if (typeof translationAssets === 'object') raw = translation.translate(raw, translationAssets.translations || {}, translationAssets.fallback || {});
 
     try {
-        content = ejs.render(safe.fs.readFileSync(path.join(MAIL_TEMPLATES_DIR, templateFile), 'utf8'), params);
+        content = ejs.render(raw, params);
     } catch (e) {
         debug(`Error rendering ${templateFile}`, e);
     }
@@ -106,25 +111,6 @@ function render(templateFile, params) {
     return content;
 }
 
-function mailUserEvent(mailTo, user, event) {
-    assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof event, 'string');
-
-    getMailConfig(function (error, mailConfig) {
-        if (error) return debug('Error getting mail details:', error);
-
-        var mailOptions = {
-            from: mailConfig.notificationFrom,
-            to: mailTo,
-            subject: util.format('[%s] %s %s', mailConfig.cloudronName, user.username || user.fallbackEmail || user.email, event),
-            text: render('user_event.ejs', { user: user, event: event, format: 'text' }),
-        };
-
-        sendMail(mailOptions);
-    });
-}
-
 function sendInvite(user, invitor, inviteLink) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof invitor, 'object');
@@ -135,84 +121,31 @@ function sendInvite(user, invitor, inviteLink) {
     getMailConfig(function (error, mailConfig) {
         if (error) return debug('Error getting mail details:', error);
 
-        var templateData = {
-            user: user,
-            webadminUrl: settings.adminOrigin(),
-            inviteLink: inviteLink,
-            invitor: invitor,
-            cloudronName: mailConfig.cloudronName,
-            cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
-        };
+        translation.getTranslations(function (error, translationAssets) {
+            if (error) return debug('Error getting translations:', error);
 
-        var templateDataText = JSON.parse(JSON.stringify(templateData));
-        templateDataText.format = 'text';
+            var templateData = {
+                user: user.displayName || user.username || user.email,
+                webadminUrl: settings.adminOrigin(),
+                inviteLink: inviteLink,
+                invitor: invitor ? invitor.email : null,
+                cloudronName: mailConfig.cloudronName,
+                cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
+            };
 
-        var templateDataHTML = JSON.parse(JSON.stringify(templateData));
-        templateDataHTML.format = 'html';
+            var mailOptions = {
+                from: mailConfig.notificationFrom,
+                to: user.fallbackEmail,
+                subject: ejs.render(translation.translate('{{ welcomeEmail.subject }}', translationAssets.translations || {}, translationAssets.fallback || {}), { cloudron: mailConfig.cloudronName }),
+                text: render('welcome_user-text.ejs', templateData, translationAssets),
+                html: render('welcome_user-html.ejs', templateData, translationAssets)
+            };
 
-        var mailOptions = {
-            from: mailConfig.notificationFrom,
-            to: user.fallbackEmail,
-            subject: util.format('Welcome to %s', mailConfig.cloudronName),
-            text: render('welcome_user.ejs', templateDataText),
-            html: render('welcome_user.ejs', templateDataHTML)
-        };
-
-        sendMail(mailOptions);
+            sendMail(mailOptions);
+        });
     });
 }
 
-function userAdded(mailTo, user) {
-    assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof user, 'object');
-
-    debug(`userAdded: Sending mail for added users ${user.fallbackEmail} to ${mailTo}`);
-
-    getMailConfig(function (error, mailConfig) {
-        if (error) return debug('Error getting mail details:', error);
-
-        var templateData = {
-            user: user,
-            cloudronName: mailConfig.cloudronName,
-            cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
-        };
-
-        var templateDataText = JSON.parse(JSON.stringify(templateData));
-        templateDataText.format = 'text';
-
-        var templateDataHTML = JSON.parse(JSON.stringify(templateData));
-        templateDataHTML.format = 'html';
-
-        var mailOptions = {
-            from: mailConfig.notificationFrom,
-            to: mailTo,
-            subject: util.format('[%s] User %s added', mailConfig.cloudronName, user.fallbackEmail),
-            text: render('user_added.ejs', templateDataText),
-            html: render('user_added.ejs', templateDataHTML)
-        };
-
-        sendMail(mailOptions);
-    });
-}
-
-function userRemoved(mailTo, user) {
-    assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof user, 'object');
-
-    debug('Sending mail for userRemoved.', user.id, user.username, user.email);
-
-    mailUserEvent(mailTo, user, 'was removed');
-}
-
-function roleChanged(mailTo, user) {
-    assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof user, 'object');
-
-    debug('Sending mail for roleChanged');
-
-    mailUserEvent(mailTo, user, `now has the role '${user.role}'`);
-}
-
 function passwordReset(user) {
     assert.strictEqual(typeof user, 'object');
 
@@ -221,28 +154,26 @@ function passwordReset(user) {
     getMailConfig(function (error, mailConfig) {
         if (error) return debug('Error getting mail details:', error);
 
-        var templateData = {
-            user: user,
-            resetLink: `${settings.adminOrigin()}/login.html?resetToken=${user.resetToken}`,
-            cloudronName: mailConfig.cloudronName,
-            cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
-        };
+        translation.getTranslations(function (error, translationAssets) {
+            if (error) return debug('Error getting translations:', error);
 
-        var templateDataText = JSON.parse(JSON.stringify(templateData));
-        templateDataText.format = 'text';
+            var templateData = {
+                user: user.displayName || user.username || user.email,
+                resetLink: `${settings.adminOrigin()}/login.html?resetToken=${user.resetToken}`,
+                cloudronName: mailConfig.cloudronName,
+                cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
+            };
 
-        var templateDataHTML = JSON.parse(JSON.stringify(templateData));
-        templateDataHTML.format = 'html';
+            var mailOptions = {
+                from: mailConfig.notificationFrom,
+                to: user.fallbackEmail,
+                subject: ejs.render(translation.translate('{{ passwordResetEmail.subject }}', translationAssets.translations || {}, translationAssets.fallback || {}), { cloudron: mailConfig.cloudronName }),
+                text: render('password_reset-text.ejs', templateData, translationAssets),
+                html: render('password_reset-html.ejs', templateData, translationAssets)
+            };
 
-        var mailOptions = {
-            from: mailConfig.notificationFrom,
-            to: user.fallbackEmail,
-            subject: util.format('[%s] Password Reset', mailConfig.cloudronName),
-            text: render('password_reset.ejs', templateDataText),
-            html: render('password_reset.ejs', templateDataHTML)
-        };
-
-        sendMail(mailOptions);
+            sendMail(mailOptions);
+        });
     });
 }
 
@@ -258,7 +189,7 @@ function appUp(mailTo, app) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] App %s is back online', mailConfig.cloudronName, app.fqdn),
+            subject: `[${mailConfig.cloudronName}] App ${app.fqdn} is back online`,
             text: render('app_up.ejs', { title: app.manifest.title, appFqdn: app.fqdn, format: 'text' })
         };
 
@@ -278,7 +209,7 @@ function appDied(mailTo, app) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] App %s is down', mailConfig.cloudronName, app.fqdn),
+            subject: `[${mailConfig.cloudronName}] App ${app.fqdn} is down`,
             text: render('app_down.ejs', { title: app.manifest.title, appFqdn: app.fqdn, supportEmail: mailConfig.supportEmail, format: 'text' })
         };
 
@@ -325,10 +256,46 @@ function appUpdated(mailTo, app, callback) {
     });
 }
 
-function appUpdatesAvailable(mailTo, apps, hasSubscription, callback) {
+function boxUpdateAvailable(mailTo, updateInfo, callback) {
+    assert.strictEqual(typeof mailTo, 'string');
+    assert.strictEqual(typeof updateInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    getMailConfig(function (error, mailConfig) {
+        if (error) return debug('Error getting mail details:', error);
+
+        var converter = new showdown.Converter();
+
+        var templateData = {
+            webadminUrl: settings.adminOrigin(),
+            newBoxVersion: updateInfo.version,
+            changelog: updateInfo.changelog,
+            changelogHTML: updateInfo.changelog.map(function (e) { return converter.makeHtml(e); }),
+            cloudronName: mailConfig.cloudronName,
+            cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
+        };
+
+        var templateDataText = JSON.parse(JSON.stringify(templateData));
+        templateDataText.format = 'text';
+
+        var templateDataHTML = JSON.parse(JSON.stringify(templateData));
+        templateDataHTML.format = 'html';
+
+        var mailOptions = {
+            from: mailConfig.notificationFrom,
+            to: mailTo,
+            subject: `[${mailConfig.cloudronName}] Cloudron update available`,
+            text: render('box_update_available.ejs', templateDataText),
+            html: render('box_update_available.ejs', templateDataHTML)
+        };
+
+        sendMail(mailOptions, callback);
+    });
+}
+
+function appUpdatesAvailable(mailTo, apps, callback) {
     assert.strictEqual(typeof mailTo, 'string');
     assert.strictEqual(typeof apps, 'object');
-    assert.strictEqual(typeof hasSubscription, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
     getMailConfig(function (error, mailConfig) {
@@ -341,7 +308,6 @@ function appUpdatesAvailable(mailTo, apps, hasSubscription, callback) {
 
         var templateData = {
             webadminUrl: settings.adminOrigin(),
-            hasSubscription: hasSubscription,
             apps: apps,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
@@ -356,7 +322,7 @@ function appUpdatesAvailable(mailTo, apps, hasSubscription, callback) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: `New app updates available for ${mailConfig.cloudronName}`,
+            subject: `[${mailConfig.cloudronName}] App update available`,
             text: render('app_updates_available.ejs', templateDataText),
             html: render('app_updates_available.ejs', templateDataHTML)
         };
@@ -374,7 +340,7 @@ function backupFailed(mailTo, errorMessage, logUrl) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] Failed to backup', mailConfig.cloudronName),
+            subject: `[${mailConfig.cloudronName}] Failed to backup`,
             text: render('backup_failed.ejs', { cloudronName: mailConfig.cloudronName, message: errorMessage, logUrl: logUrl, format: 'text' })
         };
 
@@ -393,7 +359,7 @@ function certificateRenewalError(mailTo, domain, message) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] Certificate renewal error', domain),
+            subject: `[${mailConfig.cloudronName}] Certificate renewal error`,
             text: render('certificate_renewal_error.ejs', { domain: domain, message: message, format: 'text' })
         };
 
@@ -411,7 +377,7 @@ function boxUpdateError(mailTo, message) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] Cloudron update error', mailConfig.cloudronName),
+            subject: `[${mailConfig.cloudronName}] Cloudron update error`,
             text: render('box_update_error.ejs', { message: message, format: 'text' })
         };
 
@@ -419,19 +385,30 @@ function boxUpdateError(mailTo, message) {
     });
 }
 
-function oomEvent(mailTo, program, event) {
+function oomEvent(mailTo, app, addon, containerId, event) {
     assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof program, 'string');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof addon, 'object');
+    assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof event, 'object');
 
     getMailConfig(function (error, mailConfig) {
         if (error) return debug('Error getting mail details:', error);
 
+        const templateData = {
+            webadminUrl: settings.adminOrigin(),
+            cloudronName: mailConfig.cloudronName,
+            app,
+            addon,
+            event: JSON.stringify(event),
+            format: 'text'
+        };
+
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] %s was restarted (OOM)', mailConfig.cloudronName, program),
-            text: render('oom_event.ejs', { cloudronName: mailConfig.cloudronName, program: program, event: JSON.stringify(event), format: 'text' })
+            subject: `[${mailConfig.cloudronName}] ${app ? app.fqdn : addon.name} was restarted (OOM)`,
+            text: render('oom_event.ejs', templateData)
         };
 
         sendMail(mailOptions);
@@ -450,7 +427,7 @@ function sendTestMail(domain, email, callback) {
             authUser: `no-reply@${domain}`,
             from: `"${mailConfig.cloudronName}" <no-reply@${domain}>`,
             to: email,
-            subject: util.format('Test Email from %s', mailConfig.cloudronName),
+            subject: `[${mailConfig.cloudronName}] Test Email`,
             text: render('test.ejs', { cloudronName: mailConfig.cloudronName, format: 'text'})
         };
 

src/middleware/index.js

@@ -1,10 +1,11 @@
 'use strict';
 
 exports = module.exports = {
+    cookieParser: require('cookie-parser'),
     cors: require('./cors'),
     json: require('body-parser').json,
     morgan: require('morgan'),
-    proxy: require('proxy-middleware'),
+    proxy: require('./proxy-middleware.js'),
     lastMile: require('connect-lastmile'),
     multipart: require('./multipart.js'),
     timeout: require('connect-timeout'),

src/middleware/proxy-middleware.js

@@ -0,0 +1,149 @@
+// https://github.com/cloudron-io/node-proxy-middleware
+// MIT license
+// contains https://github.com/gonzalocasas/node-proxy-middleware/pull/59
+
+var os = require('os');
+var http = require('http');
+var https = require('https');
+var owns = {}.hasOwnProperty;
+
+module.exports = function proxyMiddleware(options) {
+  //enable ability to quickly pass a url for shorthand setup
+  if(typeof options === 'string'){
+      options = require('url').parse(options);
+  }
+
+  var httpLib = options.protocol === 'https:' ? https : http;
+  var request = httpLib.request;
+
+  options = options || {};
+  options.hostname = options.hostname;
+  options.port = options.port;
+  options.pathname = options.pathname || '/';
+
+  return function (req, resp, next) {
+    var url = req.url;
+    // You can pass the route within the options, as well
+    if (typeof options.route === 'string') {
+      if (url === options.route) {
+        url = '';
+      } else if (url.slice(0, options.route.length) === options.route) {
+        url = url.slice(options.route.length);
+      } else {
+        return next();
+      }
+    }
+
+    //options for this request
+    var opts = extend({}, options);
+    if (url && url.charAt(0) === '?') { // prevent /api/resource/?offset=0
+      if (options.pathname.length > 1 && options.pathname.charAt(options.pathname.length - 1) === '/') {
+        opts.path = options.pathname.substring(0, options.pathname.length - 1) + url;
+      } else {
+        opts.path = options.pathname + url;
+      }
+    } else if (url) {
+      opts.path = slashJoin(options.pathname, url);
+    } else {
+      opts.path = options.pathname;
+    }
+    opts.method = req.method;
+    opts.headers = options.headers ? merge(req.headers, options.headers) : req.headers;
+
+    applyViaHeader(req.headers, opts, opts.headers);
+
+    if (!options.preserveHost) {
+      // Forwarding the host breaks dotcloud
+      delete opts.headers.host;
+    }
+
+    var myReq = request(opts, function (myRes) {
+      var statusCode = myRes.statusCode
+        , headers = myRes.headers
+        , location = headers.location;
+      // Fix the location
+      if (((statusCode > 300 && statusCode < 304) || statusCode === 201) && location && location.indexOf(options.href) > -1) {
+        // absoulte path
+        headers.location = location.replace(options.href, slashJoin('/', slashJoin((options.route || ''), '')));
+      }
+      applyViaHeader(myRes.headers, opts, myRes.headers);
+      rewriteCookieHosts(myRes.headers, opts, myRes.headers, req);
+      resp.writeHead(myRes.statusCode, myRes.headers);
+      myRes.on('error', function (err) {
+        next(err);
+      });
+      myRes.on('end', function (err) {
+        next();
+      });
+      myRes.pipe(resp);
+    });
+    myReq.on('error', function (err) {
+      next(err);
+    });
+    if (!req.readable) {
+      myReq.end();
+    } else {
+      req.pipe(myReq);
+    }
+  };
+};
+
+function applyViaHeader(existingHeaders, opts, applyTo) {
+  if (!opts.via) return;
+
+  var viaName = (true === opts.via) ?  os.hostname() : opts.via;
+  var viaHeader = '1.1 ' + viaName;
+  if(existingHeaders.via) {
+    viaHeader = existingHeaders.via + ', ' + viaHeader;
+  }
+
+  applyTo.via = viaHeader;
+}
+
+function rewriteCookieHosts(existingHeaders, opts, applyTo, req) {
+  if (!opts.cookieRewrite || !owns.call(existingHeaders, 'set-cookie')) {
+    return;
+  }
+
+  var existingCookies = existingHeaders['set-cookie'],
+      rewrittenCookies = [],
+      rewriteHostname = (true === opts.cookieRewrite) ? os.hostname() : opts.cookieRewrite;
+
+  if (!Array.isArray(existingCookies)) {
+    existingCookies = [ existingCookies ];
+  }
+
+  for (var i = 0; i < existingCookies.length; i++) {
+    var rewrittenCookie = existingCookies[i].replace(/(Domain)=[a-z\.-_]*?(;|$)/gi, '$1=' + rewriteHostname + '$2');
+
+    if (!req.connection.encrypted) {
+      rewrittenCookie = rewrittenCookie.replace(/;\s*?(Secure)/i, '');
+    }
+    rewrittenCookies.push(rewrittenCookie);
+  }
+
+  applyTo['set-cookie'] = rewrittenCookies;
+}
+
+function slashJoin(p1, p2) {
+  var trailing_slash = false;
+
+  if (p1.length && p1[p1.length - 1] === '/') { trailing_slash = true; }
+  if (trailing_slash && p2.length && p2[0] === '/') {p2 = p2.substring(1); }
+
+  return p1 + p2;
+}
+
+function extend(obj, src) {
+  for (var key in src) if (owns.call(src, key)) obj[key] = src[key];
+  return obj;
+}
+
+//merges data without changing state in either argument
+function merge(src1, src2) {
+    var merged = {};
+    extend(merged, src1);
+    extend(merged, src2);
+    return merged;
+}
+

src/nginxconfig.ejs

@@ -28,7 +28,13 @@ server {
         alias /home/yellowtent/platformdata/acme/;
     }
 
-    # for default server, serve the splash page. for other endpoints, redirect to HTTPS
+    location /notfound.html {
+        root <%= sourceDir %>/dashboard/dist;
+        try_files /notfound.html =404;
+        internal;
+    }
+
+    # for default server, serve the notfound page. for other endpoints, redirect to HTTPS
     location / {
 <% if ( endpoint === 'admin' || endpoint === 'setup' ) { %>
         return 301 https://$host$request_uri;
@@ -37,8 +43,8 @@ server {
 <% } else if ( endpoint === 'redirect' ) { %>
         return 301 https://<%= redirectTo %>$request_uri;
 <% } else if ( endpoint === 'ip' ) { %>
-        root <%= sourceDir %>/dashboard/dist;
-        try_files /splash.html =404;
+        root /home/yellowtent/boxdata;
+        try_files /custom_pages/notfound.html /notfound.html;
 <% } %>
     }
 }
@@ -92,6 +98,14 @@ server {
     add_header Referrer-Policy "no-referrer-when-downgrade";
     proxy_hide_header Referrer-Policy;
 
+    # workaround caching issue after /logout. if max-age is set, browser uses cache and user thinks they have not logged out
+    # have to keep all the add_header here to avoid repeating all add_header in location block
+    <% if (proxyAuth.enabled) { %>
+    proxy_hide_header Cache-Control;
+    add_header Cache-Control no-cache;
+    add_header Set-Cookie $auth_cookie;
+    <% } %>
+
     # gzip responses that are > 50k and not images
     gzip on;
     gzip_min_length 50k;
@@ -147,86 +161,150 @@ server {
 <% if ( endpoint === 'admin' ) { %>
         proxy_pass http://127.0.0.1:3000;
 <% } else if ( endpoint === 'app' ) { %>
-        proxy_pass http://127.0.0.1:<%= port %>;
+        proxy_pass http://<%= ip %>:<%= port %>;
 <% } else if ( endpoint === 'redirect' ) { %>
         return 302 https://<%= redirectTo %>$request_uri;
 <% } %>
     }
 
     # user defined .well-known resources
-    location ~ ^/.well-known/(.*)$ {
-        root /home/yellowtent/boxdata/well-known/$host;
-        try_files /$1 @wellknown-upstream;
+    location /.well-known/ {
+        error_page 404 = @wellknown-upstream;
+        proxy_pass http://127.0.0.1:3000/well-known-handler/;
     }
 
-    location / {
-        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
-        proxy_buffer_size       128k;
-        proxy_buffers           4 256k;
-        proxy_busy_buffers_size 256k;
+<% if (proxyAuth.enabled) { %>
+    proxy_set_header X-App-ID "<%= proxyAuth.id %>";
+<% } %>
 
-        # No buffering to temp files, it fails for large downloads
-        proxy_max_temp_file_size 0;
+    # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
+    proxy_buffer_size       128k;
+    proxy_buffers           4 256k;
+    proxy_busy_buffers_size 256k;
 
-        # Disable check to allow unlimited body sizes. this allows apps to accept whatever size they want
-        client_max_body_size 0;
+    # No buffering to temp files, it fails for large downloads
+    proxy_max_temp_file_size 0;
+
+    # Disable check to allow unlimited body sizes. this allows apps to accept whatever size they want
+    client_max_body_size 0;
 
 <% if (robotsTxtQuoted) { %>
-        location = /robots.txt {
-            return 200 <%- robotsTxtQuoted %>;
-        }
+    location = /robots.txt {
+        return 200 <%- robotsTxtQuoted %>;
+    }
 <% } %>
 
 <% if ( endpoint === 'admin' || endpoint === 'setup' ) { %>
-        location /api/ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 1m;
-        }
+    location /api/ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 1m;
+    }
 
-        location ~ ^/api/v1/(developer|session)/login$ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 1m;
-            limit_req zone=admin_login burst=5;
-        }
+    location ~ ^/api/v1/(developer|session)/login$ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 1m;
+        limit_req zone=admin_login burst=5;
+    }
 
-        # the read timeout is between successive reads and not the whole connection
-        location ~ ^/api/v1/apps/.*/exec$ {
-            proxy_pass   http://127.0.0.1:3000;
-            proxy_read_timeout 30m;
-        }
+    # the read timeout is between successive reads and not the whole connection
+    location ~ ^/api/v1/apps/.*/exec$ {
+        proxy_pass   http://127.0.0.1:3000;
+        proxy_read_timeout 30m;
+    }
 
-        location ~ ^/api/v1/apps/.*/upload$ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 0;
-        }
+    location ~ ^/api/v1/apps/.*/upload$ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 0;
+    }
 
-        location ~ ^/api/v1/apps/.*/files/ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 0;
-        }
+    location ~ ^/api/v1/apps/.*/files/ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 0;
+    }
 
-        # graphite paths (uncomment block below and visit /graphite-web/dashboard)
-        # remember to comment out the CSP policy as well to access the graphite dashboard
-        # location ~ ^/graphite-web/ {
-        #     proxy_pass   http://127.0.0.1:8417;
-        #     client_max_body_size 1m;
-        # }
+    location ~ ^/api/v1/volumes/.*/files/ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 0;
+    }
 
-        location / {
-            root   <%= sourceDir %>/dashboard/dist;
-            index  index.html index.htm;
-        }
+    # graphite paths (uncomment block below and visit /graphite-web/dashboard)
+    # remember to comment out the CSP policy as well to access the graphite dashboard
+    # location ~ ^/graphite-web/ {
+    #     proxy_pass   http://127.0.0.1:8417;
+    #     client_max_body_size 1m;
+    # }
+
+    location / {
+        root   <%= sourceDir %>/dashboard/dist;
+        index  index.html index.htm;
+    }
 <% } else if ( endpoint === 'app' ) { %>
-        proxy_pass http://127.0.0.1:<%= port %>;
+    location = /appstatus.html {
+        root /home/yellowtent/box/dashboard/dist;
+    }
+
+    <% if (proxyAuth.enabled) { %>
+    location = /proxy-auth {
+        internal;
+        proxy_pass http://127.0.0.1:3001/auth;
+        proxy_pass_request_body off;
+        # repeat proxy headers since we addded proxy_set_header at this location level
+        proxy_set_header X-App-ID "<%= proxyAuth.id %>";
+        proxy_set_header Content-Length "";
+    }
+
+    location ~ ^/(login|logout)$ {
+        proxy_pass http://127.0.0.1:3001;
+    }
+
+    location @proxy-auth-login {
+        if ($http_user_agent ~* "docker-client") {
+            return 401;
+        }
+        return 302 /login?redirect=$request_uri;
+    }
+
+    location <%= proxyAuth.location %> {
+        auth_request /proxy-auth;
+        auth_request_set $auth_cookie $upstream_http_set_cookie;
+        error_page 401 = @proxy-auth-login;
+
+        proxy_pass http://<%= ip %>:<%= port %>;
+    }
+
+    <% if (proxyAuth.location !== '/') { %>
+    location / {
+        proxy_pass http://<%= ip %>:<%= port %>;
+    }
+    <% } %>
+
+    <% } else { %>
+    location / {
+        proxy_pass http://<%= ip %>:<%= port %>;
+    }
+    <% } %>
+
+    <% Object.keys(httpPaths).forEach(function (path) { -%>
+    location "<%= path %>" {
+        # the trailing / will replace part of the original URI matched by the location.
+        proxy_pass http://<%= ip %>:<%= httpPaths[path] %>/;
+    }
+    <% }); %>
+
 <% } else if ( endpoint === 'redirect' ) { %>
         # redirect everything to the app. this is temporary because there is no way
         # to clear a permanent redirect on the browser
         return 302 https://<%= redirectTo %>$request_uri;
 <% } else if ( endpoint === 'ip' ) { %>
-        location / {
-            root   <%= sourceDir %>/dashboard/dist;
-            try_files /splash.html =404;
-        }
-<% } %>
+    location /notfound.html {
+        root <%= sourceDir %>/dashboard/dist;
+        try_files /notfound.html =404;
+        internal;
     }
+
+    location / {
+        root /home/yellowtent/boxdata;
+        try_files /custom_pages/notfound.html /notfound.html;
+    }
+<% } %>
 }

src/notifications.js

@@ -1,11 +1,14 @@
 'use strict';
 
 exports = module.exports = {
-    get: get,
-    ack: ack,
-    getAllPaged: getAllPaged,
+    get,
+    ack,
+    getAllPaged,
 
-    onEvent: onEvent,
+    onEvent,
+
+    appUpdatesAvailable,
+    boxUpdateAvailable,
 
     // NOTE: if you add an alert, be sure to add title below
     ALERT_BACKUP_CONFIG: 'backupConfig',
@@ -20,11 +23,13 @@ exports = module.exports = {
     _add: add
 };
 
-let assert = require('assert'),
+let apps = require('./apps.js'),
+    assert = require('assert'),
     async = require('async'),
     auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     changelog = require('./changelog.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:notifications'),
     eventlog = require('./eventlog.js'),
     mailer = require('./mailer.js'),
@@ -94,8 +99,8 @@ function getAllPaged(userId, acknowledged, page, perPage, callback) {
 }
 
 // Calls iterator with (admin, callback)
-function actionForAllAdmins(skippingUserIds, iterator, callback) {
-    assert(Array.isArray(skippingUserIds));
+function forEachAdmin(options, iterator, callback) {
+    assert(Array.isArray(options.skip));
     assert.strictEqual(typeof iterator, 'function');
     assert.strictEqual(typeof callback, 'function');
 
@@ -104,7 +109,7 @@ function actionForAllAdmins(skippingUserIds, iterator, callback) {
         if (error) return callback(error);
 
         // filter out users we want to skip (like the user who did the action or the user the action was performed on)
-        result = result.filter(function (r) { return skippingUserIds.indexOf(r.id) === -1; });
+        result = result.filter(function (r) { return options.skip.indexOf(r.id) === -1; });
 
         async.each(result, iterator, callback);
     });
@@ -116,8 +121,7 @@ function userAdded(performedBy, eventId, user, callback) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([ performedBy, user.id ], function (admin, done) {
-        mailer.userAdded(admin.email, user);
+    forEachAdmin({ skip: [ performedBy, user.id ] }, function (admin, done) {
         add(admin.id, eventId, `User '${user.displayName}' added`, `User '${user.username || user.email || user.fallbackEmail}' was added.`, done);
     }, callback);
 }
@@ -128,8 +132,7 @@ function userRemoved(performedBy, eventId, user, callback) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([ performedBy, user.id ], function (admin, done) {
-        mailer.userRemoved(admin.email, user);
+    forEachAdmin({ skip: [ performedBy, user.id ] }, function (admin, done) {
         add(admin.id, eventId, `User '${user.displayName}' removed`, `User '${user.username || user.email || user.fallbackEmail}' was removed.`, done);
     }, callback);
 }
@@ -139,8 +142,7 @@ function roleChanged(performedBy, eventId, user, callback) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([ performedBy, user.id ], function (admin, done) {
-        mailer.roleChanged(admin.email, user);
+    forEachAdmin({ skip: [ performedBy, user.id ] }, function (admin, done) {
         add(admin.id, eventId, `User '${user.displayName}'s role changed`, `User '${user.username || user.email || user.fallbackEmail}' now has the role ${user.role}.`, done);
     }, callback);
 }
@@ -152,23 +154,19 @@ function oomEvent(eventId, app, addon, containerId, event, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let title, message, program;
+    assert(app || addon);
+
+    let title, message;
     if (app) {
-        program = `App ${app.fqdn}`;
         title = `The application at ${app.fqdn} ran out of memory.`;
-        message = 'The application has been restarted automatically. If you see this notification often, consider increasing the [memory limit](https://docs.cloudron.io/apps/#memory-limit)';
+        message = `The application has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.adminOrigin()}/#/app/${app.id}/resources)`;
     } else if (addon) {
-        program = `${addon.name} service`;
         title = `The ${addon.name} service ran out of memory`;
-        message = 'The service has been restarted automatically. If you see this notification often, consider increasing the [memory limit](https://docs.cloudron.io/troubleshooting/#services)';
-    } else { // this never happens currently
-        program = `Container ${containerId}`;
-        title = `The container ${containerId} ran out of memory`;
-        message = 'The container has been restarted automatically. Consider increasing the [memory limit](https://docs.docker.com/v17.09/edge/engine/reference/commandline/update/#update-a-containers-kernel-memory-constraints)';
+        message = `The service has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.adminOrigin()}/#/services)`;
     }
 
-    actionForAllAdmins([], function (admin, done) {
-        mailer.oomEvent(admin.email, program, event);
+    forEachAdmin({ skip: [] }, function (admin, done) {
+        mailer.oomEvent(admin.email, app, addon, containerId, event);
 
         add(admin.id, eventId, title, message, done);
     }, callback);
@@ -179,7 +177,7 @@ function appUp(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, done) {
+    forEachAdmin({ skip: [] }, function (admin, done) {
         mailer.appUp(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is back online`, `The application installed at ${app.fqdn} is back online.`, done);
     }, callback);
@@ -190,7 +188,7 @@ function appDied(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, callback) {
+    forEachAdmin({ skip: [] }, function (admin, callback) {
         mailer.appDied(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is down`, `The application installed at ${app.fqdn} is not responding.`, callback);
     }, callback);
@@ -208,7 +206,7 @@ function appUpdated(eventId, app, callback) {
     const title = upstreamVersion ? `${app.manifest.title} at ${app.fqdn} updated to ${upstreamVersion} (package version ${app.manifest.version})`
         : `${app.manifest.title} at ${app.fqdn} updated to package version ${app.manifest.version}`;
 
-    actionForAllAdmins([], function (admin, done) {
+    forEachAdmin({ skip: [] }, function (admin, done) {
         add(admin.id, eventId, title, `The application installed at https://${app.fqdn} was updated.\n\nChangelog:\n${app.manifest.changelog}\n`, function (error) {
             if (error) return callback(error);
 
@@ -220,6 +218,39 @@ function appUpdated(eventId, app, callback) {
     }, callback);
 }
 
+function boxUpdateAvailable(updateInfo, callback) {
+    assert.strictEqual(typeof updateInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getAutoupdatePattern(function (error, result) {
+        if (error) return callback(error);
+
+        if (result !== constants.AUTOUPDATE_PATTERN_NEVER) return callback();
+
+        forEachAdmin({ skip: [] }, function (admin, done) {
+            mailer.boxUpdateAvailable(admin.email, updateInfo, done);
+        }, callback);
+    });
+}
+
+function appUpdatesAvailable(appUpdates, callback) {
+    assert.strictEqual(typeof appUpdates, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getAutoupdatePattern(function (error, result) {
+        if (error) return callback(error);
+
+        // if we are auto updating, then just consider apps that cannot be auto updated
+        if (result !== constants.AUTOUPDATE_PATTERN_NEVER) appUpdates = appUpdates.filter(update => !apps.canAutoupdateApp(update.app, update.updateInfo));
+
+        if (appUpdates.length === 0) return callback();
+
+        forEachAdmin({ skip: [] }, function (admin, done) {
+            mailer.appUpdatesAvailable(admin.email, appUpdates, done);
+        }, callback);
+    });
+}
+
 function boxUpdated(eventId, oldVersion, newVersion, callback) {
     assert.strictEqual(typeof eventId, 'string');
     assert.strictEqual(typeof oldVersion, 'string');
@@ -229,7 +260,7 @@ function boxUpdated(eventId, oldVersion, newVersion, callback) {
     const changes = changelog.getChanges(newVersion);
     const changelogMarkdown = changes.map((m) => `* ${m}\n`).join('');
 
-    actionForAllAdmins([], function (admin, done) {
+    forEachAdmin({ skip: [] }, function (admin, done) {
         add(admin.id, eventId, `Cloudron updated to v${newVersion}`, `Cloudron was updated from v${oldVersion} to v${newVersion}.\n\nChangelog:\n${changelogMarkdown}\n`, done);
     }, callback);
 }
@@ -239,7 +270,7 @@ function boxUpdateError(eventId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, done) {
+    forEachAdmin({ skip: [] }, function (admin, done) {
         mailer.boxUpdateError(admin.email, errorMessage);
         add(admin.id, eventId, 'Cloudron update failed', `Failed to update Cloudron: ${errorMessage}.`, done);
     }, callback);
@@ -251,7 +282,7 @@ function certificateRenewalError(eventId, vhost, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, callback) {
+    forEachAdmin({ skip: [] }, function (admin, callback) {
         mailer.certificateRenewalError(admin.email, vhost, errorMessage);
         add(admin.id, eventId, `Certificate renewal of ${vhost} failed`, `Failed to new certs of ${vhost}: ${errorMessage}. Renewal will be retried in 12 hours`, callback);
     }, callback);
@@ -263,7 +294,7 @@ function backupFailed(eventId, taskId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, callback) {
+    forEachAdmin({ skip: [] }, function (admin, callback) {
         mailer.backupFailed(admin.email, errorMessage, `${settings.adminOrigin()}/logs.html?taskId=${taskId}`);
         add(admin.id, eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}).`, callback);
     }, callback);
@@ -275,11 +306,10 @@ function alert(id, title, message, callback) {
     assert.strictEqual(typeof message, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`alert: id=${id} title=${title}`);
-
     const acknowledged = !message;
+    debug(`alert: id=${id} title=${title} ack=${acknowledged}`);
 
-    actionForAllAdmins([], function (admin, callback) {
+    forEachAdmin({ skip: [] }, function (admin, callback) {
         const data = {
             userId: admin.id,
             eventId: null,

src/paths.js

@@ -16,8 +16,10 @@ exports = module.exports = {
 
     CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
     INFRA_VERSION_FILE: path.join(baseDir(), 'platformdata/INFRA_VERSION'),
+    DASHBOARD_DIR: constants.TEST ? path.join(__dirname, '../../dashboard/src') : path.join(baseDir(), 'box/dashboard/dist'),
 
     PROVIDER_FILE: '/etc/cloudron/PROVIDER',
+    SETUP_TOKEN_FILE: '/etc/cloudron/SETUP_TOKEN',
 
     PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'),
     APPS_DATA_DIR: path.join(baseDir(), 'appsdata'),
@@ -35,12 +37,14 @@ exports = module.exports = {
     SNAPSHOT_INFO_FILE: path.join(baseDir(), 'platformdata/backup/snapshot-info.json'),
     DYNDNS_INFO_FILE: path.join(baseDir(), 'platformdata/dyndns-info.json'),
     FEATURES_INFO_FILE: path.join(baseDir(), 'platformdata/features-info.json'),
+    PROXY_AUTH_TOKEN_SECRET_FILE: path.join(baseDir(), 'platformdata/proxy-auth-token-secret'),
     VERSION_FILE: path.join(baseDir(), 'platformdata/VERSION'),
 
     // this is not part of appdata because an icon may be set before install
     APP_ICONS_DIR: path.join(baseDir(), 'boxdata/appicons'),
     PROFILE_ICONS_DIR: path.join(baseDir(), 'boxdata/profileicons'),
     MAIL_DATA_DIR: path.join(baseDir(), 'boxdata/mail'),
+    SFTP_KEYS_DIR: path.join(baseDir(), 'boxdata/sftp/ssh'),
     ACME_ACCOUNT_KEY_FILE: path.join(baseDir(), 'boxdata/acme/acme.key'),
     APP_CERTS_DIR: path.join(baseDir(), 'boxdata/certs'),
     CLOUDRON_AVATAR_FILE: path.join(baseDir(), 'boxdata/avatar.png'),
@@ -56,7 +60,9 @@ exports = module.exports = {
 
     GHOST_USER_FILE: path.join(baseDir(), 'platformdata/cloudron_ghost.json'),
 
+    SWAP_RATIO_FILE: path.join(baseDir(), 'platformdata/swap-ratio'),
+
     // this pattern is for the cloudron logs API route to work
     BACKUP_LOG_FILE: path.join(baseDir(), 'platformdata/logs/backup/app.log'),
-    UPDATER_LOG_FILE: path.join(baseDir(), 'platformdata/logs/updater/app.log')
+    UPDATER_LOG_FILE: path.join(baseDir(), 'platformdata/logs/updater/app.log'),
 };

src/platform.js

@@ -1,27 +1,24 @@
 'use strict';
 
 exports = module.exports = {
-    start: start,
-    stopAllTasks: stopAllTasks,
+    start,
+    stopAllTasks,
 
     // exported for testing
     _isReady: false
 };
 
-var addons = require('./addons.js'),
-    apps = require('./apps.js'),
+const apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     debug = require('debug')('box:platform'),
     fs = require('fs'),
-    graphs = require('./graphs.js'),
     infra = require('./infra_version.js'),
     locker = require('./locker.js'),
     paths = require('./paths.js'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
-    settings = require('./settings.js'),
-    sftp = require('./sftp.js'),
+    services = require('./services.js'),
     shell = require('./shell.js'),
     tasks = require('./tasks.js'),
     _ = require('underscore');
@@ -54,11 +51,9 @@ function start(callback) {
     if (error) return callback(error);
 
     async.series([
-        (next) => { if (existingInfra.version !== infra.version) removeAllContainers(existingInfra, next); else next(); },
+        (next) => { if (existingInfra.version !== infra.version) removeAllContainers(next); else next(); },
         markApps.bind(null, existingInfra), // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
-        graphs.startGraphite.bind(null, existingInfra),
-        sftp.startSftp.bind(null, existingInfra),
-        addons.startServices.bind(null, existingInfra),
+        services.startServices.bind(null, existingInfra),
         fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra, null, 4))
     ], function (error) {
         if (error) return callback(error);
@@ -80,7 +75,7 @@ function onPlatformReady(infraChanged) {
     exports._isReady = true;
 
     let tasks = [ apps.schedulePendingTasks ];
-    if (infraChanged) tasks.push(applyPlatformConfig, pruneInfraImages);
+    if (infraChanged) tasks.push(pruneInfraImages);
 
     async.series(async.reflectAll(tasks), function (error, results) {
         results.forEach((result, idx) => {
@@ -89,14 +84,6 @@ function onPlatformReady(infraChanged) {
     });
 }
 
-function applyPlatformConfig(callback) {
-    settings.getPlatformConfig(function (error, platformConfig) {
-        if (error) return callback(error);
-
-        addons.updateServiceConfig(platformConfig, callback);
-    });
-}
-
 function pruneInfraImages(callback) {
     debug('pruneInfraImages: checking existing images');
 
@@ -115,14 +102,14 @@ function pruneInfraImages(callback) {
             debug(`pruneInfraImages: removing unused image of ${image.repo}: tag: ${parts[1]} id: ${parts[0]}`);
 
             let result = safe.child_process.execSync(`docker rmi ${parts[0]}`, { encoding: 'utf8' });
-            if (result === null) debug(`Erroring removing image ${parts[0]}: ${safe.error.mesage}`);
+            if (result === null) debug(`Error removing image ${parts[0]}: ${safe.error.mesage}`);
         }
 
         iteratorCallback();
     }, callback);
 }
 
-function removeAllContainers(existingInfra, callback) {
+function removeAllContainers(callback) {
     debug('removeAllContainers: removing all containers for infra upgrade');
 
     async.series([

src/provision.js

@@ -236,6 +236,7 @@ function getStatus(callback) {
                 cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
                 footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
                 adminFqdn: settings.adminDomain() ? settings.adminFqdn() : null,
+                language: allSettings[settings.LANGUAGE_KEY],
                 activated: activated,
                 provider: settings.provider() // used by setup wizard of marketplace images
             }, gProvisionStatus));

src/proxyauth.js

@@ -0,0 +1,259 @@
+'use strict';
+
+// heavily inspired from https://gock.net/blog/2020/nginx-subrequest-authentication-server/ and https://github.com/andygock/auth-server
+
+exports = module.exports = {
+    start,
+    stop
+};
+
+const apps = require('./apps.js'),
+    assert = require('assert'),
+    basicAuth = require('basic-auth'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:proxyAuth'),
+    ejs = require('ejs'),
+    express = require('express'),
+    fs = require('fs'),
+    hat = require('./hat.js'),
+    http = require('http'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    jwt = require('jsonwebtoken'),
+    middleware = require('./middleware'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    speakeasy = require('speakeasy'),
+    translation = require('./translation.js'),
+    users = require('./users.js');
+
+let gHttpServer = null;
+let TOKEN_SECRET = null;
+const EXPIRY_DAYS = 7;
+
+function jwtVerify(req, res, next) {
+    const token = req.cookies.authToken;
+
+    if (!token) return next();
+
+    jwt.verify(token, TOKEN_SECRET, function (error, decoded) {
+        if (error) {
+            debug('clearing token', error);
+            res.clearCookie('authToken');
+            return next(new HttpError(403, 'Malformed token or bad signature'));
+        }
+
+        req.user = decoded.user || null;
+        next();
+    });
+}
+
+function basicAuthVerify(req, res, next) {
+    const appId = req.headers['x-app-id'] || '';
+    const credentials = basicAuth(req);
+    if (!appId || !credentials) return next();
+
+    const api = credentials.name.indexOf('@') !== -1 ? users.verifyWithEmail : users.verifyWithUsername;
+
+    api(credentials.name, credentials.pass, appId, function (error, user) {
+        if (error) return next(new HttpError(403, 'Invalid username or password' ));
+
+        req.user = user;
+        next();
+    });
+}
+
+function loginPage(req, res, next) {
+    const appId = req.headers['x-app-id'] || '';
+    if (!appId) return next(new HttpError(503, 'Nginx misconfiguration'));
+
+    translation.getTranslations(function (error, translationAssets) {
+        if (error) return next(new HttpError(500, 'No translation found'));
+
+        const raw = safe.fs.readFileSync(path.join(paths.DASHBOARD_DIR, 'templates/proxyauth-login.ejs'), 'utf8');
+        if (raw === null) return next(new HttpError(500, 'Login template not found'));
+
+        const translatedContent = translation.translate(raw, translationAssets.translations || {}, translationAssets.fallback || {});
+        var finalContent = '';
+
+        apps.get(appId, function (error, app) {
+            if (error) return next(new HttpError(503, error.message));
+
+            const title = app.label || app.manifest.title;
+
+            apps.getIconPath(app, {}, function (error, iconPath) {
+                const icon = 'data:image/png;base64,' + safe.fs.readFileSync(iconPath || '', 'base64');
+
+                try {
+                    finalContent = ejs.render(translatedContent, { title, icon });
+                } catch (e) {
+                    debug('Error rendering proxyauth-login.ejs', e);
+                    return next(new HttpError(500, 'Login template error'));
+                }
+
+                res.set('Content-Type', 'text/html');
+                return res.send(finalContent);
+            });
+        });
+    });
+}
+
+// someday this can be more sophisticated and check for a real browser
+function isBrowser(req) {
+    const userAgent = req.get('user-agent');
+    if (!userAgent) return false;
+
+    return !userAgent.toLowerCase().includes('docker-client');
+}
+
+// called by nginx to authorize any protected route. this route must return only 2xx or 401/403 (http://nginx.org/en/docs/http/ngx_http_auth_request_module.html)
+function auth(req, res, next) {
+    if (!req.user) {
+        if (isBrowser(req)) return next(new HttpError(401, 'Unauthorized'));
+
+        // the header has to be generated here and cannot be set in nginx config - https://forum.nginx.org/read.php?2,171461,171469#msg-171469
+        res.set('www-authenticate', 'Basic realm="Cloudron"');
+        return next(new HttpError(401, 'Unauthorized'));
+    }
+
+    // user is already authenticated, refresh cookie
+    const token = jwt.sign({ user: req.user }, TOKEN_SECRET, { expiresIn: `${EXPIRY_DAYS}d` });
+
+    res.cookie('authToken', token, {
+        httpOnly: true,
+        maxAge: EXPIRY_DAYS * 86400 * 1000, // milliseconds
+        secure: true
+    });
+
+    return next(new HttpSuccess(200, {}));
+}
+
+// endpoint called by login page, username and password posted as JSON body
+function passwordAuth(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    const appId = req.headers['x-app-id'] || '';
+    if (!appId) return next(new HttpError(503, 'Nginx misconfiguration'));
+
+    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be non empty string' ));
+    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be non empty string' ));
+    if ('totpToken' in req.body && typeof req.body.totpToken !== 'string') return next(new HttpError(400, 'totpToken must be a string' ));
+
+    const { username, password, totpToken } = req.body;
+
+    const api = username.indexOf('@') !== -1 ? users.verifyWithEmail : users.verifyWithUsername;
+
+    api(username, password, appId, function (error, user) {
+        if (error) return next(new HttpError(403, 'Invalid username or password' ));
+
+        if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
+            if (!totpToken) return next(new HttpError(403, 'A totpToken must be provided'));
+
+            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
+            if (!verified) return next(new HttpError(403, 'Invalid totpToken'));
+        }
+
+        req.user = user;
+        next();
+    });
+}
+
+function authorize(req, res, next) {
+    const appId = req.headers['x-app-id'] || '';
+    if (!appId) return next(new HttpError(503, 'Nginx misconfiguration'));
+
+    apps.get(appId, function (error, app) {
+        if (error) return next(new HttpError(403, 'No such app' ));
+
+        apps.hasAccessTo(app, req.user, function (error, hasAccess) {
+            if (error) return next(new HttpError(403, 'Forbidden' ));
+            if (!hasAccess) return next(new HttpError(403, 'Forbidden' ));
+
+            const token = jwt.sign({ user: users.removePrivateFields(req.user) }, TOKEN_SECRET, { expiresIn: `${EXPIRY_DAYS}d` });
+
+            res.cookie('authToken', token, {
+                httpOnly: true,
+                maxAge: EXPIRY_DAYS * 86400 * 1000, // milliseconds
+                secure: true
+            });
+
+            res.redirect(302, '/');
+        });
+    });
+}
+
+function logoutPage(req, res, next) {
+    const appId = req.headers['x-app-id'] || '';
+    if (!appId) return next(new HttpError(503, 'Nginx misconfiguration'));
+
+    apps.get(appId, function (error, app) {
+        if (error) return next(new HttpError(503, error.message));
+
+        res.clearCookie('authToken');
+
+        // when we have no path, redirect to the login page. we cannot redirect to '/' because browsers will immediately serve up the cached page
+        // if a path is set, we can assume '/' is a public page
+        res.redirect(302, app.manifest.addons.proxyAuth.path ? '/' : '/login');
+    });
+}
+
+function logout(req, res, next) {
+    res.clearCookie('authToken');
+    next(new HttpSuccess(200, {}));
+}
+
+// provides webhooks for the auth wall
+function initializeAuthwallExpressSync() {
+    let app = express();
+    let httpServer = http.createServer(app);
+
+    let QUERY_LIMIT = '1mb'; // max size for json and urlencoded queries
+    let REQUEST_TIMEOUT = 10000; // timeout for all requests
+
+    let json = middleware.json({ strict: true, limit: QUERY_LIMIT }); // application/json
+
+    if (process.env.BOX_ENV !== 'test') app.use(middleware.morgan('proxyauth :method :url :status :response-time ms - :res[content-length]', { immediate: false }));
+
+    var router = new express.Router();
+    router.del = router.delete; // amend router.del for readability further on
+
+    app
+        .use(middleware.timeout(REQUEST_TIMEOUT))
+        .use(middleware.cookieParser())
+        .use(router)
+        .use(middleware.lastMile());
+
+    router.get ('/login',     loginPage);
+    router.get ('/auth',      jwtVerify, basicAuthVerify, auth);
+    router.post('/login',     json, passwordAuth, authorize);
+    router.get ('/logout',    logoutPage);
+    router.post('/logout',    json, logout);
+
+    return httpServer;
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+    assert.strictEqual(gHttpServer, null, 'Authwall is already up and running.');
+
+    if (!fs.existsSync(paths.PROXY_AUTH_TOKEN_SECRET_FILE)) {
+        TOKEN_SECRET = hat(64);
+        fs.writeFileSync(paths.PROXY_AUTH_TOKEN_SECRET_FILE, TOKEN_SECRET, 'utf8');
+    } else {
+        TOKEN_SECRET = fs.readFileSync(paths.PROXY_AUTH_TOKEN_SECRET_FILE, 'utf8').trim();
+    }
+
+    gHttpServer = initializeAuthwallExpressSync();
+
+    gHttpServer.listen(constants.AUTHWALL_PORT, '127.0.0.1', callback);
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!gHttpServer) return callback(null);
+
+    gHttpServer.close(callback);
+    gHttpServer = null;
+}

src/reverseproxy.js

@@ -57,6 +57,14 @@ var acme2 = require('./cert/acme2.js'),
 var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/nginxconfig.ejs', { encoding: 'utf8' }),
     RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh');
 
+function nginxLocation(s) {
+    if (!s.startsWith('!')) return s;
+
+    let re = s.replace(/[\^$\\.*+?()[\]{}|]/g, '\\$&'); // https://github.com/es-shims/regexp.escape/blob/master/implementation.js
+
+    return `~ ^(?!(${re.slice(1)}))`; // negative regex assertion - https://stackoverflow.com/questions/16302897/nginx-location-not-equal-to-regex
+}
+
 function getAcmeApi(domainObject, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -109,7 +117,7 @@ function providerMatchesSync(domainObject, certFilePath, apiOptions) {
     const domain = subject.substr(subject.indexOf('=') + 1).trim(); // subject can be /CN=, CN=, CN = and other forms
     const issuer = subjectAndIssuer.match(/^issuer=(.*)$/m)[1];
     const isWildcardCert = domain.includes('*');
-    const isLetsEncryptProd = issuer.includes('Let\'s Encrypt Authority');
+    const isLetsEncryptProd = issuer.includes('Let\'s Encrypt');
 
     const issuerMismatch = (apiOptions.prod && !isLetsEncryptProd) || (!apiOptions.prod && isLetsEncryptProd);
     // bare domain is not part of wildcard SAN
@@ -117,7 +125,9 @@ function providerMatchesSync(domainObject, certFilePath, apiOptions) {
 
     const mismatch = issuerMismatch || wildcardMismatch;
 
-    debug(`providerMatchesSync: ${certFilePath} subject=${subject} domain=${domain} issuer=${issuer} wildcard=${isWildcardCert}/${apiOptions.wildcard} prod=${isLetsEncryptProd}/${apiOptions.prod} match=${!mismatch}`);
+    debug(`providerMatchesSync: ${certFilePath} subject=${subject} domain=${domain} issuer=${issuer} `
+        + `wildcard=${isWildcardCert}/${apiOptions.wildcard} prod=${isLetsEncryptProd}/${apiOptions.prod} `
+        + `issuerMismatch=${issuerMismatch} wildcardMismatch=${wildcardMismatch} match=${!mismatch}`);
 
     return !mismatch;
 }
@@ -160,7 +170,7 @@ function validateCertificate(location, domainObject, certificate) {
 }
 
 function reload(callback) {
-    if (process.env.BOX_ENV === 'test') return callback();
+    if (constants.TEST) return callback();
 
     shell.sudo('reload', [ RELOAD_NGINX_CMD ], {}, function (error) {
         if (error) return callback(new BoxError(BoxError.NGINX_ERROR, `Error reloading nginx: ${error.message}`));
@@ -247,22 +257,22 @@ function setAppCertificateSync(location, domainObject, certificate) {
     return null;
 }
 
-function getAcmeCertificate(hostname, domainObject, callback) {
-    assert.strictEqual(typeof hostname, 'string');
+function getAcmeCertificate(vhost, domainObject, callback) {
+    assert.strictEqual(typeof vhost, 'string'); // this can contain wildcard domain (for alias domains)
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     let certFilePath, keyFilePath;
 
-    if (hostname !== domainObject.domain && domainObject.tlsConfig.wildcard) { // bare domain is not part of wildcard SAN
-        let certName = domains.makeWildcard(hostname).replace('*.', '_.');
+    if (vhost !== domainObject.domain && domainObject.tlsConfig.wildcard) { // bare domain is not part of wildcard SAN
+        let certName = domains.makeWildcard(vhost).replace('*.', '_.');
         certFilePath = path.join(paths.APP_CERTS_DIR, `${certName}.cert`);
         keyFilePath = path.join(paths.APP_CERTS_DIR, `${certName}.key`);
 
         if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath });
     } else {
-        certFilePath = path.join(paths.APP_CERTS_DIR, `${hostname}.cert`);
-        keyFilePath = path.join(paths.APP_CERTS_DIR, `${hostname}.key`);
+        certFilePath = path.join(paths.APP_CERTS_DIR, `${vhost}.cert`);
+        keyFilePath = path.join(paths.APP_CERTS_DIR, `${vhost}.key`);
 
         if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath });
     }
@@ -334,7 +344,7 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
                     debug(`ensureCertificate: ${vhost} certificate already exists at ${currentBundle.keyFilePath}`);
 
                     if (!isExpiringSync(currentBundle.certFilePath, 24 * 30) && providerMatchesSync(domainObject, currentBundle.certFilePath, apiOptions)) return callback(null, currentBundle, { renewed: false });
-                    debug(`ensureCertificate: ${vhost} cert require renewal`);
+                    debug(`ensureCertificate: ${vhost} cert requires renewal`);
                 } else {
                     debug(`ensureCertificate: ${vhost} cert does not exist`);
                 }
@@ -380,7 +390,8 @@ function writeDashboardNginxConfig(bundle, configFileName, vhost, callback) {
         endpoint: 'admin',
         certFilePath: bundle.certFilePath,
         keyFilePath: bundle.keyFilePath,
-        robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n')
+        robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n'),
+        proxyAuth: { enabled: false, id: null, location: nginxLocation('/') }
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, configFileName);
@@ -427,8 +438,9 @@ function writeDashboardConfig(domain, callback) {
     });
 }
 
-function writeAppNginxConfig(app, bundle, callback) {
+function writeAppNginxConfig(app, fqdn, bundle, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof fqdn, 'string');
     assert.strictEqual(typeof bundle, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -447,20 +459,28 @@ function writeAppNginxConfig(app, bundle, callback) {
     var data = {
         sourceDir: sourceDir,
         adminOrigin: settings.adminOrigin(),
-        vhost: app.fqdn,
+        vhost: fqdn,
         hasIPv6: sysinfo.hasIPv6(),
-        port: app.httpPort,
+        ip: app.containerIp,
+        port: app.manifest.httpPort,
         endpoint: endpoint,
         certFilePath: bundle.certFilePath,
         keyFilePath: bundle.keyFilePath,
         robotsTxtQuoted,
         cspQuoted,
-        hideHeaders
+        hideHeaders,
+        proxyAuth: {
+            enabled: app.sso && app.manifest.addons && app.manifest.addons.proxyAuth,
+            id: app.id,
+            location: nginxLocation(safe.query(app.manifest, 'addons.proxyAuth.path') || '/')
+        },
+        httpPaths: app.manifest.httpPaths || {}
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 
-    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-    debug('writeAppNginxConfig: writing config for "%s" to %s with options %j', app.fqdn, nginxConfigFilename, data);
+    const aliasSuffix = app.fqdn === fqdn ? '' : `-alias-${fqdn.replace('*', '_')}`;
+    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}${aliasSuffix}.conf`);
+    debug('writeAppNginxConfig: writing config for "%s" to %s with options %j', fqdn, nginxConfigFilename, data);
 
     if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
         debug('Error creating nginx config for "%s" : %s', app.fqdn, safe.error.message);
@@ -486,7 +506,8 @@ function writeAppRedirectNginxConfig(app, fqdn, bundle, callback) {
         keyFilePath: bundle.keyFilePath,
         robotsTxtQuoted: null,
         cspQuoted: null,
-        hideHeaders: []
+        hideHeaders: [],
+        proxyAuth: { enabled: false, id: app.id, location: nginxLocation('/') }
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 
@@ -506,21 +527,30 @@ function writeAppConfig(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    getCertificate(app.fqdn, app.domain, function (error, bundle) {
-        if (error) return callback(error);
+    let appDomains = [];
+    appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary' });
 
-        writeAppNginxConfig(app, bundle, function (error) {
-            if (error) return callback(error);
-
-            async.eachSeries(app.alternateDomains, function (alternateDomain, iteratorDone) {
-                getCertificate(alternateDomain.fqdn, alternateDomain.domain, function (error, bundle) {
-                    if (error) return iteratorDone(error);
-
-                    writeAppRedirectNginxConfig(app, alternateDomain.fqdn, bundle, iteratorDone);
-                });
-            }, callback);
-        });
+    app.alternateDomains.forEach(function (alternateDomain) {
+        appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate' });
     });
+
+    app.aliasDomains.forEach(function (aliasDomain) {
+        appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias' });
+    });
+
+    async.eachSeries(appDomains, function (appDomain, iteratorDone) {
+        getCertificate(appDomain.fqdn, appDomain.domain, function (error, bundle) {
+            if (error) return iteratorDone(error);
+
+            if (appDomain.type === 'primary') {
+                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            } else if (appDomain.type === 'alternate') {
+                writeAppRedirectNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            } else if (appDomain.type === 'alias') {
+                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            }
+        });
+    }, callback);
 }
 
 function configureApp(app, auditSource, callback) {
@@ -528,21 +558,30 @@ function configureApp(app, auditSource, callback) {
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    ensureCertificate(app.fqdn, app.domain, auditSource, function (error, bundle) {
-        if (error) return callback(error);
+    let appDomains = [];
+    appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary' });
 
-        writeAppNginxConfig(app, bundle, function (error) {
-            if (error) return callback(error);
-
-            async.eachSeries(app.alternateDomains, function (alternateDomain, iteratorDone) {
-                ensureCertificate(alternateDomain.fqdn, alternateDomain.domain, auditSource, function (error, bundle) {
-                    if (error) return iteratorDone(error);
-
-                    writeAppRedirectNginxConfig(app, alternateDomain.fqdn, bundle, iteratorDone);
-                });
-            }, callback);
-        });
+    app.alternateDomains.forEach(function (alternateDomain) {
+        appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate' });
     });
+
+    app.aliasDomains.forEach(function (aliasDomain) {
+        appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias' });
+    });
+
+    async.eachSeries(appDomains, function (appDomain, iteratorDone) {
+        ensureCertificate(appDomain.fqdn, appDomain.domain, auditSource, function (error, bundle) {
+            if (error) return iteratorDone(error);
+
+            if (appDomain.type === 'primary') {
+                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            } else if (appDomain.type === 'alternate') {
+                writeAppRedirectNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            } else if (appDomain.type === 'alias') {
+                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            }
+        });
+    }, callback);
 }
 
 function unconfigureApp(app, callback) {
@@ -566,7 +605,7 @@ function renewCerts(options, auditSource, progressCallback, callback) {
     apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        var appDomains = [];
+        let appDomains = [];
 
         // add webadmin and mail domain
         if (settings.mailFqdn() === settings.adminFqdn()) {
@@ -576,16 +615,20 @@ function renewCerts(options, auditSource, progressCallback, callback) {
             appDomains.push({ domain: settings.mailDomain(), fqdn: settings.mailFqdn(), type: 'mail' });
         }
 
-        // add app main
         allApps.forEach(function (app) {
             if (app.runState === apps.RSTATE_STOPPED) return; // do not renew certs of stopped apps
 
-            appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'main', app: app, nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf') });
+            appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary', app: app, nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf') });
 
             app.alternateDomains.forEach(function (alternateDomain) {
                 const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-redirect-${alternateDomain.fqdn}.conf`);
                 appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate', app: app, nginxConfigFilename });
             });
+
+            app.aliasDomains.forEach(function (aliasDomain) {
+                const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-alias-${aliasDomain.fqdn.replace('*', '_')}.conf`);
+                appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias', app: app, nginxConfigFilename });
+            });
         });
 
         if (options.domain) appDomains = appDomains.filter(function (appDomain) { return appDomain.domain === options.domain; });
@@ -617,10 +660,12 @@ function renewCerts(options, auditSource, progressCallback, callback) {
                         mail.handleCertChanged,
                         writeDashboardNginxConfig.bind(null, bundle, `${settings.adminFqdn()}.conf`, settings.adminFqdn())
                     ], iteratorCallback);
-                } else if (appDomain.type === 'main') {
-                    return writeAppNginxConfig(appDomain.app, bundle, iteratorCallback);
+                } else if (appDomain.type === 'primary') {
+                    return writeAppNginxConfig(appDomain.app, appDomain.fqdn, bundle, iteratorCallback);
                 } else if (appDomain.type === 'alternate') {
                     return writeAppRedirectNginxConfig(appDomain.app, appDomain.fqdn, bundle, iteratorCallback);
+                } else if (appDomain.type === 'alias') {
+                    return writeAppNginxConfig(appDomain.app, appDomain.fqdn, bundle, iteratorCallback);
                 }
 
                 iteratorCallback(new BoxError(BoxError.INTERNAL_ERROR, `Unknown domain type for ${appDomain.fqdn}. This should never happen`));
@@ -673,7 +718,8 @@ function writeDefaultConfig(options, callback) {
         endpoint: options.activated ? 'ip' : 'setup',
         certFilePath,
         keyFilePath,
-        robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n')
+        robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n'),
+        proxyAuth: { enabled: false, id: null, location: nginxLocation('/') }
     };
     const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
     const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, constants.NGINX_DEFAULT_CONFIG_FILE_NAME);

src/routes/accesscontrol.js

@@ -1,11 +1,11 @@
 'use strict';
 
 exports = module.exports = {
-    passwordAuth: passwordAuth,
-    tokenAuth: tokenAuth,
+    passwordAuth,
+    tokenAuth,
 
-    authorize: authorize,
-    websocketAuth: websocketAuth
+    authorize,
+    websocketAuth
 };
 
 var accesscontrol = require('../accesscontrol.js'),
@@ -21,17 +21,17 @@ function passwordAuth(req, res, next) {
 
     if (!req.body.username || typeof req.body.username !== 'string') return next(new HttpError(400, 'A username must be non-empty string'));
     if (!req.body.password || typeof req.body.password !== 'string') return next(new HttpError(400, 'A password must be non-empty string'));
+    if ('totpToken' in req.body && typeof req.body.totpToken !== 'string') return next(new HttpError(400, 'totpToken must be a string' ));
 
-    const username = req.body.username;
-    const password = req.body.password;
+    const { username, password, totpToken } = req.body;
 
     function check2FA(user) {
         assert.strictEqual(typeof user, 'object');
 
         if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
-            if (!req.body.totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
+            if (!totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
 
-            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
+            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: totpToken, window: 2 });
             if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
         }
 
@@ -99,6 +99,7 @@ function tokenAuth(req, res, next) {
         if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
         if (error) return next(new HttpError(500, error.message));
 
+        req.access_token = token; // used in logout route
         req.user = user;
 
         next();

src/routes/apps.js

@@ -8,6 +8,7 @@ exports = module.exports = {
     uninstall,
     restore,
     importApp,
+    exportApp,
     backup,
     update,
     getLogs,
@@ -43,6 +44,7 @@ exports = module.exports = {
     uploadFile,
     downloadFile,
 
+
     load
 };
 
@@ -138,6 +140,11 @@ function install(req, res, next) {
         if (data.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
     }
 
+    if ('aliasDomains' in data) {
+        if (!Array.isArray(data.aliasDomains)) return next(new HttpError(400, 'aliasDomains must be an array'));
+        if (data.aliasDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'aliasDomains array must contain objects with domain and subdomain strings'));
+    }
+
     if ('env' in data) {
         if (!data.env || typeof data.env !== 'object') return next(new HttpError(400, 'env must be an object'));
         if (Object.keys(data.env).some(function (key) { return typeof data.env[key] !== 'string'; })) return next(new HttpError(400, 'env must contain values as strings'));
@@ -352,6 +359,11 @@ function setLocation(req, res, next) {
         if (req.body.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
     }
 
+    if ('aliasDomains' in req.body) {
+        if (!Array.isArray(req.body.aliasDomains)) return next(new HttpError(400, 'aliasDomains must be an array'));
+        if (req.body.aliasDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'aliasDomains array must contain objects with domain and subdomain strings'));
+    }
+
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
 
     apps.setLocation(req.resource, req.body, auditSource.fromRequest(req), function (error, result) {
@@ -443,6 +455,17 @@ function importApp(req, res, next) {
     });
 }
 
+function exportApp(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.resource, 'object');
+
+    apps.exportApp(req.resource, {}, auditSource.fromRequest(req), function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, { taskId: result.taskId }));
+    });
+}
+
 function clone(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.resource, 'object');
@@ -597,7 +620,7 @@ function getLogs(req, res, next) {
 
         res.writeHead(200, {
             'Content-Type': 'application/x-logs',
-            'Content-Disposition': 'attachment; filename="log.txt"',
+            'Content-Disposition': `attachment; filename="${req.resource.id}.log"`,
             'Cache-Control': 'no-cache',
             'X-Accel-Buffering': 'no' // disable nginx buffering
         });

src/routes/cloudron.js

@@ -20,6 +20,7 @@ exports = module.exports = {
     prepareDashboardDomain,
     renewCerts,
     getServerIp,
+    getLanguages,
     syncExternalLdap
 };
 
@@ -36,6 +37,7 @@ let assert = require('assert'),
     system = require('../system.js'),
     tokendb = require('../tokendb.js'),
     tokens = require('../tokens.js'),
+    translation = require('../translation.js'),
     updater = require('../updater.js'),
     users = require('../users.js'),
     updateChecker = require('../updatechecker.js');
@@ -62,24 +64,11 @@ function login(req, res, next) {
 }
 
 function logout(req, res) {
-    var token;
+    assert.strictEqual(typeof req.access_token, 'string');
 
-    // this determines the priority
-    if (req.body && req.body.access_token) token = req.body.access_token;
-    if (req.query && req.query.access_token) token = req.query.access_token;
-    if (req.headers && req.headers.authorization) {
-        var parts = req.headers.authorization.split(' ');
-        if (parts.length == 2) {
-            var scheme = parts[0];
-            var credentials = parts[1];
+    eventlog.add(eventlog.ACTION_USER_LOGOUT, auditSource.fromRequest(req), { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
-            if (/^Bearer$/i.test(scheme)) token = credentials;
-        }
-    }
-
-    if (!token) return res.redirect('/login.html');
-
-    tokendb.delByAccessToken(token, function () { res.redirect('/login.html'); });
+    tokendb.delByAccessToken(req.access_token, function () { res.redirect('/login.html'); });
 }
 
 function passwordResetRequest(req, res, next) {
@@ -225,7 +214,7 @@ function getLogs(req, res, next) {
 
         res.writeHead(200, {
             'Content-Type': 'application/x-logs',
-            'Content-Disposition': 'attachment; filename="log.txt"',
+            'Content-Disposition': `attachment; filename="${req.params.unit}.log"`,
             'Cache-Control': 'no-cache',
             'X-Accel-Buffering': 'no' // disable nginx buffering
         });
@@ -313,3 +302,11 @@ function getServerIp(req, res, next) {
         next(new HttpSuccess(200, { ip }));
     });
 }
+
+function getLanguages(req, res, next) {
+    translation.getLanguages(function (error, languages) {
+        if (error) return next(new BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { languages }));
+    });
+}

src/routes/domains.js

@@ -99,6 +99,13 @@ function update(req, res, next) {
         if (!req.body.tlsConfig.provider || typeof req.body.tlsConfig.provider !== 'string') return next(new HttpError(400, 'tlsConfig.provider must be a string'));
     }
 
+    if ('wellKnown' in req.body) {
+        if (typeof req.body.wellKnown !== 'object') return next(new HttpError(400, 'wellKnown must be an object'));
+        if (req.body.wellKnown) {
+            if (Object.keys(req.body.wellKnown).some(k => typeof req.body.wellKnown[k] !== 'string')) return next(new HttpError(400, 'wellKnown is a map of strings'));
+        }
+    }
+
     // some DNS providers like DigitalOcean take a really long time to verify credentials (https://github.com/expressjs/timeout/issues/26)
     req.clearTimeout();
 
@@ -107,7 +114,8 @@ function update(req, res, next) {
         provider: req.body.provider,
         config: req.body.config,
         fallbackCertificate: req.body.fallbackCertificate || null,
-        tlsConfig: req.body.tlsConfig || { provider: 'letsencrypt-prod' }
+        tlsConfig: req.body.tlsConfig || { provider: 'letsencrypt-prod' },
+        wellKnown: req.body.wellKnown || null
     };
 
     domains.update(req.params.domain, data, auditSource.fromRequest(req), function (error) {

src/routes/filemanager.js

@@ -4,11 +4,11 @@ exports = module.exports = {
     proxy
 };
 
-var addons = require('../addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     middleware = require('../middleware/index.js'),
     HttpError = require('connect-lastmile').HttpError,
+    services = require('../services.js'),
     url = require('url');
 
 function proxy(req, res, next) {
@@ -18,7 +18,7 @@ function proxy(req, res, next) {
 
     req.clearTimeout();
 
-    addons.getContainerDetails('sftp', 'CLOUDRON_SFTP_TOKEN', function (error, result) {
+    services.getContainerDetails('sftp', 'CLOUDRON_SFTP_TOKEN', function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         let parsedUrl = url.parse(req.url, true /* parseQueryString */);

src/routes/graphs.js

@@ -1,7 +1,7 @@
 'use strict';
 
 exports = module.exports = {
-    getGraphs: getGraphs
+    getGraphs
 };
 
 var middleware = require('../middleware/index.js'),

src/routes/groups.js

@@ -62,6 +62,7 @@ function updateMembers(req, res, next) {
 
     if (!req.body.userIds) return next(new HttpError(404, 'missing or invalid userIds fields'));
     if (!Array.isArray(req.body.userIds)) return next(new HttpError(404, 'userIds must be an array'));
+    if (req.body.userIds.some((u) => typeof u !== 'string')) return next(new HttpError(400, 'userIds array must contain strings'));
 
     groups.setMembers(req.params.groupId, req.body.userIds, function (error) {
         if (error) return next(BoxError.toHttpError(error));

src/routes/index.js

@@ -25,5 +25,6 @@ exports = module.exports = {
     tasks: require('./tasks.js'),
     tokens: require('./tokens.js'),
     users: require('./users.js'),
-    volumes: require('./volumes.js')
+    volumes: require('./volumes.js'),
+    wellknown: require('./wellknown.js')
 };

src/routes/mail.js

@@ -196,9 +196,10 @@ function addMailbox(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
     if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be a string'));
-    if (typeof req.body.userId !== 'string') return next(new HttpError(400, 'userId must be a string'));
+    if (typeof req.body.ownerId !== 'string') return next(new HttpError(400, 'ownerId must be a string'));
+    if (typeof req.body.ownerType !== 'string') return next(new HttpError(400, 'ownerType must be a string'));
 
-    mail.addMailbox(req.body.name, req.params.domain, req.body.userId, auditSource.fromRequest(req), function (error) {
+    mail.addMailbox(req.body.name, req.params.domain, req.body.ownerId, req.body.ownerType, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(201, {}));
@@ -209,9 +210,10 @@ function updateMailbox(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.params.name, 'string');
 
-    if (typeof req.body.userId !== 'string') return next(new HttpError(400, 'userId must be a string'));
+    if (typeof req.body.ownerId !== 'string') return next(new HttpError(400, 'ownerId must be a string'));
+    if (typeof req.body.ownerType !== 'string') return next(new HttpError(400, 'ownerType must be a string'));
 
-    mail.updateMailboxOwner(req.params.name, req.params.domain, req.body.userId, auditSource.fromRequest(req), function (error) {
+    mail.updateMailboxOwner(req.params.name, req.params.domain, req.body.ownerId, req.body.ownerType, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));

src/routes/mailserver.js

@@ -2,21 +2,28 @@
 
 exports = module.exports = {
     proxy,
+    restart,
 
     getLocation,
     setLocation
 };
 
-var addons = require('../addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
+    debug = require('debug')('box:routes/mailserver'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     mail = require('../mail.js'),
     middleware = require('../middleware/index.js'),
+    services = require('../services.js'),
     url = require('url');
 
+function restart(req, res, next) {
+    mail.restartMail((error) => debug('Error restarting mail container', error));
+    next();
+}
+
 function proxy(req, res, next) {
     let parsedUrl = url.parse(req.url, true /* parseQueryString */);
     const pathname = req.path.split('/').pop();
@@ -26,7 +33,7 @@ function proxy(req, res, next) {
     delete req.headers['authorization'];
     delete req.headers['cookies'];
 
-    addons.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
+    services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
         if (error) return next(BoxError.toHttpError(error));
 
         parsedUrl.query['access_token'] = addonDetails.token;
@@ -36,6 +43,7 @@ function proxy(req, res, next) {
         proxyOptions.rejectUnauthorized = false;
         const mailserverProxy = middleware.proxy(proxyOptions);
 
+        req.clearTimeout(); // TODO: add timeout to mail server proxy logic instead of this
         mailserverProxy(req, res, function (error) {
             if (!error) return next();
 

src/routes/provision.js

@@ -1,11 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    providerTokenAuth: providerTokenAuth,
-    setup: setup,
-    activate: activate,
-    restore: restore,
-    getStatus: getStatus
+    providerTokenAuth,
+    setup,
+    activate,
+    restore,
+    getStatus,
+    setupTokenAuth
 };
 
 var assert = require('assert'),
@@ -15,10 +16,24 @@ var assert = require('assert'),
     debug = require('debug')('box:routes/setup'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    paths = require('../paths.js'),
     provision = require('../provision.js'),
     request = require('request'),
+    safe = require('safetydance'),
     settings = require('../settings.js');
 
+function setupTokenAuth(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    const setupToken = safe.fs.readFileSync(paths.SETUP_TOKEN_FILE, 'utf8');
+    if (!setupToken) return next();
+
+    if (!req.body.setupToken) return next(new HttpError(400, 'setup token required'));
+    if (setupToken.trim() !== req.body.setupToken) return next(new HttpError(422, 'setup token does not match'));
+
+    return next();
+}
+
 function providerTokenAuth(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 

src/routes/services.js

@@ -1,24 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    getAll: getAll,
-    get: get,
-    configure: configure,
-    getLogs: getLogs,
-    getLogStream: getLogStream,
-    restart: restart
+    getAll,
+    get,
+    configure,
+    getLogs,
+    getLogStream,
+    restart,
+    rebuild
 };
 
-var addons = require('../addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess;
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    services = require('../services.js');
 
 function getAll(req, res, next) {
-    req.clearTimeout(); // can take a while to get status of all services
-
-    addons.getServices(function (error, result) {
+    services.getServiceIds(function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { services: result }));
@@ -28,7 +27,7 @@ function getAll(req, res, next) {
 function get(req, res, next) {
     assert.strictEqual(typeof req.params.service, 'string');
 
-    addons.getService(req.params.service, function (error, result) {
+    services.getServiceStatus(req.params.service, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { service: result }));
@@ -38,20 +37,18 @@ function get(req, res, next) {
 function configure(req, res, next) {
     assert.strictEqual(typeof req.params.service, 'string');
 
-    if (typeof req.body.memory !== 'number') return next(new HttpError(400, 'memory must be a number'));
-    if (typeof req.body.memorySwap !== 'number') return next(new HttpError(400, 'memorySwap must be a number'));
+    if (typeof req.body.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit must be a number'));
 
     const data = {
-        memory: req.body.memory,
-        memorySwap: req.body.memorySwap
+        memoryLimit: req.body.memoryLimit
     };
 
-    if (req.params.service === 'sftp' && 'requireAdmin' in req.body) {
+    if (req.params.service === 'sftp') {
         if (typeof req.body.requireAdmin !== 'boolean') return next(new HttpError(400, 'requireAdmin must be a boolean'));
         data.requireAdmin = req.body.requireAdmin;
     }
 
-    addons.configureService(req.params.service, data, function (error) {
+    services.configureService(req.params.service, data, function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, {}));
@@ -70,12 +67,12 @@ function getLogs(req, res, next) {
         format: req.query.format || 'json'
     };
 
-    addons.getServiceLogs(req.params.service, options, function (error, logStream) {
+    services.getServiceLogs(req.params.service, options, function (error, logStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.writeHead(200, {
             'Content-Type': 'application/x-logs',
-            'Content-Disposition': 'attachment; filename="log.txt"',
+            'Content-Disposition': `attachment; filename="${req.params.service}.log"`,
             'Cache-Control': 'no-cache',
             'X-Accel-Buffering': 'no' // disable nginx buffering
         });
@@ -100,7 +97,7 @@ function getLogStream(req, res, next) {
         format: 'json'
     };
 
-    addons.getServiceLogs(req.params.service, options, function (error, logStream) {
+    services.getServiceLogs(req.params.service, options, function (error, logStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.writeHead(200, {
@@ -124,7 +121,17 @@ function getLogStream(req, res, next) {
 function restart(req, res, next) {
     assert.strictEqual(typeof req.params.service, 'string');
 
-    addons.restartService(req.params.service, function (error) {
+    services.restartService(req.params.service, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
+function rebuild(req, res, next) {
+    assert.strictEqual(typeof req.params.service, 'string');
+
+    services.rebuildService(req.params.service, function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, {}));

src/routes/settings.js

@@ -116,32 +116,6 @@ function setBackupConfig(req, res, next) {
     });
 }
 
-function getPlatformConfig(req, res, next) {
-    settings.getPlatformConfig(function (error, config) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, config));
-    });
-}
-
-function setPlatformConfig(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    for (let addon of [ 'mysql', 'postgresql', 'mail', 'mongodb' ]) {
-        if (!(addon in req.body)) continue;
-        if (typeof req.body[addon] !== 'object') return next(new HttpError(400, 'addon config must be an object'));
-
-        if (typeof req.body[addon].memory !== 'number') return next(new HttpError(400, 'memory must be a number'));
-        if (typeof req.body[addon].memorySwap !== 'number') return next(new HttpError(400, 'memorySwap must be a number'));
-    }
-
-    settings.setPlatformConfig(req.body, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, {}));
-    });
-}
-
 function getExternalLdapConfig(req, res, next) {
     settings.getExternalLdapConfig(function (error, config) {
         if (error) return next(BoxError.toHttpError(error));
@@ -273,17 +247,37 @@ function setSysinfoConfig(req, res, next) {
     });
 }
 
+function getLanguage(req, res, next) {
+    settings.getLanguage(function (error, language) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { language }));
+    });
+}
+
+function setLanguage(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.language || typeof req.body.language !== 'string') return next(new HttpError(400, 'language is required'));
+
+    settings.setLanguage(req.body.language, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, {}));
+    });
+}
+
 function get(req, res, next) {
     assert.strictEqual(typeof req.params.setting, 'string');
 
     switch (req.params.setting) {
     case settings.DYNAMIC_DNS_KEY: return getDynamicDnsConfig(req, res, next);
     case settings.BACKUP_CONFIG_KEY: return getBackupConfig(req, res, next);
-    case settings.PLATFORM_CONFIG_KEY: return getPlatformConfig(req, res, next);
     case settings.EXTERNAL_LDAP_KEY: return getExternalLdapConfig(req, res, next);
     case settings.UNSTABLE_APPS_KEY: return getUnstableAppsConfig(req, res, next);
     case settings.REGISTRY_CONFIG_KEY: return getRegistryConfig(req, res, next);
     case settings.SYSINFO_CONFIG_KEY: return getSysinfoConfig(req, res, next);
+    case settings.LANGUAGE_KEY: return getLanguage(req, res, next);
 
     case settings.AUTOUPDATE_PATTERN_KEY: return getAutoupdatePattern(req, res, next);
     case settings.TIME_ZONE_KEY: return getTimeZone(req, res, next);
@@ -300,11 +294,11 @@ function set(req, res, next) {
 
     switch (req.params.setting) {
     case settings.DYNAMIC_DNS_KEY: return setDynamicDnsConfig(req, res, next);
-    case settings.PLATFORM_CONFIG_KEY: return setPlatformConfig(req, res, next);
     case settings.EXTERNAL_LDAP_KEY: return setExternalLdapConfig(req, res, next);
     case settings.UNSTABLE_APPS_KEY: return setUnstableAppsConfig(req, res, next);
     case settings.REGISTRY_CONFIG_KEY: return setRegistryConfig(req, res, next);
     case settings.SYSINFO_CONFIG_KEY: return setSysinfoConfig(req, res, next);
+    case settings.LANGUAGE_KEY: return setLanguage(req, res, next);
 
     case settings.AUTOUPDATE_PATTERN_KEY: return setAutoupdatePattern(req, res, next);
     case settings.TIME_ZONE_KEY: return setTimeZone(req, res, next);

src/routes/tasks.js

@@ -70,7 +70,7 @@ function getLogs(req, res, next) {
 
         res.writeHead(200, {
             'Content-Type': 'application/x-logs',
-            'Content-Disposition': 'attachment; filename="log.txt"',
+            'Content-Disposition': `attachment; filename="task-${req.params.taskId}.log"`,
             'Cache-Control': 'no-cache',
             'X-Accel-Buffering': 'no' // disable nginx buffering
         });

src/routes/test/apps-test.js

@@ -37,7 +37,7 @@ const docker = new Docker({ socketPath: '/var/run/docker.sock' });
 
 // Test image information
 var TEST_IMAGE_REPO = 'docker.io/cloudron/io.cloudron.testapp';
-var TEST_IMAGE_TAG = '20200207-233155-725d9e015';
+var TEST_IMAGE_TAG = '20201121-223249-985e86ebb';
 var TEST_IMAGE = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
 
 const DOMAIN_0 = {
@@ -74,39 +74,6 @@ var token_1 = null;
 let KEY, CERT;
 let appstoreIconServer = hock.createHock({ throwOnUnmatched: false });
 
-function checkAddons(appEntry, done) {
-    async.retry({ times: 15, interval: 3000 }, function (callback) {
-        // this was previously written with superagent but it was getting sporadic EPIPE
-        var req = http.get({ hostname: 'localhost', port: appEntry.httpPort, path: '/check_addons?username=' + USERNAME + '&password=' + PASSWORD });
-        req.on('error', callback);
-        req.on('response', function (res) {
-            if (res.statusCode !== 200) return callback('app returned non-200 status : ' + res.statusCode);
-
-            var d = '';
-            res.on('data', function (chunk) { d += chunk.toString('utf8'); });
-            res.on('end', function () {
-                var body = JSON.parse(d);
-
-                delete body.recvmail; // unclear why dovecot mail delivery won't work
-                delete body.stdenv; // cannot access APP_ORIGIN
-                delete body.email; // sieve will fail not sure why yet
-                delete body.docker; // TODO fix this for some reason we cannot connect to the docker proxy on port 3003
-
-                for (var key in body) {
-                    if (body[key] !== 'OK') {
-                        console.log('Not done yet: ' + JSON.stringify(body));
-                        return callback('Not done yet: ' + JSON.stringify(body));
-                    }
-                }
-
-                callback();
-            });
-        });
-
-        req.end();
-    }, done);
-}
-
 function checkRedis(containerId, done) {
     var redisIp, exportedRedisPort;
 
@@ -583,31 +550,7 @@ describe('App API', function () {
             });
         });
 
-        it('http is up and running', function (done) {
-            var tryCount = 20;
-
-            // TODO what does that check for?
-            expect(appResult.httpPort).to.be(undefined);
-
-            (function healthCheck() {
-                superagent.get('http://localhost:' + appEntry.httpPort + appResult.manifest.healthCheckPath)
-                    .end(function (err, res) {
-                        if (err || res.statusCode !== 200) {
-                            if (--tryCount === 0) {
-                                console.log('Unable to curl http://localhost:' + appEntry.httpPort + appResult.manifest.healthCheckPath);
-                                return done(new Error('Timedout'));
-                            }
-                            return setTimeout(healthCheck, 2000);
-                        }
-
-                        expect(!err).to.be.ok();
-                        expect(res.statusCode).to.equal(200);
-                        done();
-                    });
-            })();
-        });
-
-        it('tcp port mapping works', function (done) {
+        xit('tcp port mapping works', function (done) {
             var client = net.connect(7171);
             client.on('data', function (data) {
                 expect(data.toString()).to.eql('ECHO_SERVER_PORT=7171');
@@ -625,30 +568,15 @@ describe('App API', function () {
             });
         });
 
-        it('app responds to http request', function (done) {
-            superagent.get('http://localhost:' + appEntry.httpPort).end(function (err, res) {
-                expect(!err).to.be.ok();
-                expect(res.statusCode).to.equal(200);
+        xit('app responds to http request', function (done) {
+            console.log(`talking to http://${appEntry.containerIp}:7777`);
+            superagent.get(`http://${appEntry.containerIp}:7777`).end(function (error, result) {
+                console.dir(error);
+                expect(result.statusCode).to.equal(200);
                 done();
             });
         });
 
-        it('installation - app can populate addons', function (done) {
-            superagent.get(`http://localhost:${appEntry.httpPort}/populate_addons`).end(function (error, res) {
-                expect(!error).to.be.ok();
-                expect(res.statusCode).to.equal(200);
-                for (var key in res.body) {
-                    expect(res.body[key]).to.be('OK');
-                }
-                done();
-            });
-        });
-
-        it('installation - app can check addons', function (done) {
-            console.log('This test can take a while as it waits for scheduler addon to tick 3');
-            checkAddons(appEntry, done);
-        });
-
         it('installation - redis addon created', function (done) {
             checkRedis('redis-' + APP_ID, done);
         });
@@ -1154,7 +1082,7 @@ describe('App API', function () {
             });
         });
 
-        it('port mapping works after reconfiguration', function (done) {
+        xit('port mapping works after reconfiguration', function (done) {
             setTimeout(function () {
                 var client = net.connect(7172);
                 client.on('data', function (data) {
@@ -1164,16 +1092,6 @@ describe('App API', function () {
                 client.on('error', done);
             }, 4000);
         });
-
-        it('app can check addons', function (done) {
-            console.log('This test can take a while as it waits for scheduler addon to tick 4');
-
-            apps.get(APP_ID, function (error, app) {
-                if (error) return done(error);
-
-                checkAddons(app, done);
-            });
-        });
     });
 
     describe('configure debug mode', function () {
@@ -1443,16 +1361,6 @@ describe('App API', function () {
             waitForTask(taskId, done);
         });
 
-        it('app can check addons', function (done) {
-            console.log('This test can take a while as it waits for scheduler addon to tick 4');
-
-            apps.get(APP_ID, function (error, app) {
-                if (error) return done(error);
-
-                checkAddons(app, done);
-            });
-        });
-
         it('can reset data dir', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/data_dir')
                 .query({ access_token: token })
@@ -1468,15 +1376,6 @@ describe('App API', function () {
             waitForTask(taskId, done);
         });
 
-        it('app can check addons', function (done) {
-            console.log('This test can take a while as it waits for scheduler addon to tick 4');
-
-            apps.get(APP_ID, function (error, app) {
-                if (error) return done(error);
-
-                checkAddons(app, done);
-            });
-        });
     });
 
     describe('start/stop', function () {
@@ -1503,11 +1402,11 @@ describe('App API', function () {
             waitForTask(taskId, done);
         });
 
-        it('did stop the app', function (done) {
+        xit('did stop the app', function (done) {
             apps.get(APP_ID, function (error, app) {
                 if (error) return done(error);
 
-                superagent.get('http://localhost:' + app.httpPort + APP_MANIFEST.healthCheckPath).end(function (err) {
+                superagent.get(`http://${app.containerIp}:7777` + APP_MANIFEST.healthCheckPath).end(function (err) {
                     if (!err || err.code !== 'ECONNREFUSED') return done(new Error('App has not died'));
 
                     // wait for app status to be updated
@@ -1529,7 +1428,7 @@ describe('App API', function () {
                 });
         });
 
-        it('can start app', function (done) {
+        xit('can start app', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/start')
                 .query({ access_token: token })
                 .end(function (err, res) {
@@ -1543,7 +1442,7 @@ describe('App API', function () {
             waitForTask(taskId, function () { setTimeout(done, 5000); }); // give app 5 seconds to start
         });
 
-        it('did start the app', function (done) {
+        xit('did start the app', function (done) {
             apps.get(APP_ID, function (error, app) {
                 if (error) return done(error);
 
@@ -1569,7 +1468,7 @@ describe('App API', function () {
             waitForTask(taskId, function () { setTimeout(done, 12000); }); // give app 12 seconds (to die and start)
         });
 
-        it('did restart the app', function (done) {
+        xit('did restart the app', function (done) {
             apps.get(APP_ID, function (error, app) {
                 if (error) return done(error);
 
@@ -1670,47 +1569,6 @@ describe('App API', function () {
         });
     });
 
-    describe('not sure what this is', function () {
-        it('app install succeeds again', function (done) {
-            var fake1 = nock(settings.apiServerOrigin()).get('/api/v1/apps/' + APP_STORE_ID).reply(200, { manifest: APP_MANIFEST });
-            var fake2 = nock(settings.apiServerOrigin()).post(function (uri) { return uri.indexOf('/api/v1/cloudronapps') >= 0; }, (body) => body.appstoreId === APP_STORE_ID && body.manifestId === APP_MANIFEST.id && body.appId).reply(201, { });
-
-            superagent.post(SERVER_URL + '/api/v1/apps/install')
-                .query({ access_token: token })
-                .send({ appStoreId: APP_STORE_ID, location: APP_LOCATION_2, domain: DOMAIN_0.domain, portBindings: null, accessRestriction: null })
-                .end(function (err, res) {
-                    expect(res.statusCode).to.equal(202);
-                    expect(res.body.id).to.be.a('string');
-                    APP_ID = res.body.id;
-                    expect(fake1.isDone()).to.be.ok();
-                    expect(fake2.isDone()).to.be.ok();
-                    done();
-                });
-        });
-
-        it('app install fails with developer token', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                .send({ username: USERNAME, password: PASSWORD })
-                .end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(200);
-                    expect(new Date(result.body.expires).toString()).to.not.be('Invalid Date');
-                    expect(result.body.accessToken).to.be.a('string');
-
-                    // overwrite non dev token
-                    token = result.body.accessToken;
-
-                    superagent.post(SERVER_URL + '/api/v1/apps/install')
-                        .query({ access_token: token })
-                        .send({ manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, domain: DOMAIN_0.domain, portBindings: null, accessRestriction: null })
-                        .end(function (err, res) {
-                            expect(res.statusCode).to.equal(424); // appstore purchase external error
-                            done();
-                        });
-                });
-        });
-    });
-
     describe('the end', function () {
         // this is here so we can debug things if tests fail
         it('can stop box', stopBox);

src/routes/test/cloudron-test.js

@@ -558,5 +558,17 @@ describe('Cloudron API', function () {
                     });
             });
         });
+
+        describe('languages', function () {
+            it('succeeds', function (done) {
+                superagent.get(SERVER_URL + '/api/v1/cloudron/languages')
+                    .end(function (error, result) {
+                        expect(result.statusCode).to.equal(200);
+                        expect(result.body.languages).to.be.an('array');
+                        expect(result.body.languages.indexOf('en')).to.not.equal(-1);
+                        done();
+                    });
+            });
+        });
     });
 });

src/routes/test/groups-test.js

@@ -145,6 +145,16 @@ describe('Groups API', function () {
             });
     });
 
+    it('cannot set duplicate groups for a user', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
+            .query({ access_token: token })
+            .send({ groupIds: [ groupObject.id, group1Object.id, groupObject.id ]})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(409);
+                done();
+            });
+    });
+
     it('can set users of a group', function (done) {
         superagent.put(SERVER_URL + '/api/v1/groups/' + groupObject.id + '/members')
             .query({ access_token: token })
@@ -155,6 +165,17 @@ describe('Groups API', function () {
             });
     });
 
+    it('cannot set duplicate users of a group', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/groups/' + groupObject.id + '/members')
+            .query({ access_token: token })
+            .send({ userIds: [ userId, userId_1, userId ]})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(409);
+                done();
+            });
+    });
+
+
     it('cannot get non-existing group', function (done) {
         superagent.get(SERVER_URL + '/api/v1/groups/nope')
             .query({ access_token: token })
@@ -180,8 +201,8 @@ describe('Groups API', function () {
                 expect(result.statusCode).to.equal(200);
                 expect(result.body.name).to.be(groupObject.name);
                 expect(result.body.userIds.length).to.be(2);
-                expect(result.body.userIds[0]).to.be(userId);
-                expect(result.body.userIds[1]).to.be(userId_1);
+                expect(result.body.userIds).to.contain(userId);
+                expect(result.body.userIds).to.contain(userId_1);
                 done();
             });
     });

src/routes/test/mail-test.js

@@ -565,7 +565,7 @@ describe('Mail API', function () {
     describe('mailboxes', function () {
         it('add succeeds', function (done) {
             superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes')
-                .send({ name: MAILBOX_NAME, userId: userId })
+                .send({ name: MAILBOX_NAME, ownerId: userId, ownerType: 'user' })
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(201);
@@ -575,7 +575,7 @@ describe('Mail API', function () {
 
         it('cannot add again', function (done) {
             superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes')
-                .send({ name: MAILBOX_NAME, userId: userId })
+                .send({ name: MAILBOX_NAME, ownerId: userId, ownerType: 'user' })
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(409);
@@ -600,6 +600,7 @@ describe('Mail API', function () {
                     expect(res.body.mailbox).to.be.an('object');
                     expect(res.body.mailbox.name).to.equal(MAILBOX_NAME);
                     expect(res.body.mailbox.ownerId).to.equal(userId);
+                    expect(res.body.mailbox.ownerType).to.equal('user');
                     expect(res.body.mailbox.aliasName).to.equal(null);
                     expect(res.body.mailbox.aliasDomain).to.equal(null);
                     expect(res.body.mailbox.domain).to.equal(DOMAIN_0.domain);
@@ -616,8 +617,8 @@ describe('Mail API', function () {
                     expect(res.body.mailboxes[0]).to.be.an('object');
                     expect(res.body.mailboxes[0].name).to.equal(MAILBOX_NAME);
                     expect(res.body.mailboxes[0].ownerId).to.equal(userId);
-                    expect(res.body.mailboxes[0].aliasName).to.equal(null);
-                    expect(res.body.mailboxes[0].aliasDomain).to.equal(null);
+                    expect(res.body.mailboxes[0].ownerType).to.equal('user');
+                    expect(res.body.mailboxes[0].aliases).to.eql([]);
                     expect(res.body.mailboxes[0].domain).to.equal(DOMAIN_0.domain);
                     done();
                 });
@@ -659,7 +660,7 @@ describe('Mail API', function () {
 
         it('add the mailbox', function (done) {
             superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes')
-                .send({ name: MAILBOX_NAME, userId: userId })
+                .send({ name: MAILBOX_NAME, ownerId: userId, ownerType: 'user' })
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(201);

src/routes/test/settings-test.js

@@ -382,4 +382,56 @@ describe('Settings API', function () {
                 });
         });
     });
+
+    describe('language', function () {
+        it('can get default language', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/settings/language')
+                .query({ access_token: token })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(200);
+                    expect(res.body.language).to.equal('en');
+                    done();
+                });
+        });
+
+        it('cannot set language with missing language', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/settings/language')
+                .query({ access_token: token })
+                .send({ foo: 'bar' })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('cannot set language with invalid language', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/settings/language')
+                .query({ access_token: token })
+                .send({ language: 'doesnotexist' })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(404);
+                    done();
+                });
+        });
+
+        it('can set language', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/settings/language')
+                .query({ access_token: token })
+                .send({ language: 'de' })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(200);
+                    done();
+                });
+        });
+
+        it('can get language', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/settings/language')
+                .query({ access_token: token })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(200);
+                    expect(res.body.language).to.equal('de');
+                    done();
+                });
+        });
+    });
 });

src/routes/test/users-test.js

@@ -932,5 +932,44 @@ describe('Users API', function () {
                 });
         });
     });
+
+    describe('transfer ownership', function () {
+
+        before(function (done) {
+            superagent.post(SERVER_URL + '/api/v1/users')
+                .query({ access_token: token })
+                .send({ username: USERNAME_1, email: EMAIL_1 })
+                .end(function (error, result) {
+                    expect(error).to.not.be.ok();
+                    expect(result.statusCode).to.equal(201);
+
+                    user_1 = result.body;
+                    token_1 = hat(8 * 32);
+
+                    // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+                    tokendb.add({ id: 'tid-3', accessToken: token_1, identifier: user_1.id, clientId: 'test-client-id', expires: Date.now() + 10000, scope: 'unused', name: 'fromtest' }, done);
+                });
+        });
+
+        it('succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/users/' + user_1.id + '/make_owner')
+                .query({ access_token: token })
+                .send({})
+                .end(function (error, result) {
+                    expect(error).to.not.be.ok();
+                    expect(result.statusCode).to.equal(204);
+
+                    superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
+                        .query({ access_token: token_1 })
+                        .end(function (error, result) {
+                            expect(error).to.not.be.ok();
+                            expect(result.statusCode).to.equal(200);
+                            expect(result.body.role).to.equal('user');
+
+                            done();
+                    });
+            });
+        });
+    });
 });
 

src/routes/test/volumes-test.js

@@ -55,7 +55,7 @@ describe('Volumes API', function () {
     it('cannot create volume with bad name', function (done) {
         superagent.post(SERVER_URL + '/api/v1/volumes')
             .query({ access_token: token })
-            .send({ name: 'music#/ ', hostPath: '/media/music' })
+            .send({ name: 'music#/ ', hostPath: '/media/cloudron-test-music' })
             .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -75,7 +75,7 @@ describe('Volumes API', function () {
     it('can create volume', function (done) {
         superagent.post(SERVER_URL + '/api/v1/volumes')
             .query({ access_token: token })
-            .send({ name: 'music', hostPath: '/media/music' })
+            .send({ name: 'music', hostPath: '/media/cloudron-test-music' })
             .end(function (err, res) {
                 expect(res.statusCode).to.equal(201);
                 expect(res.body.id).to.be.a('string');
@@ -91,7 +91,7 @@ describe('Volumes API', function () {
                 expect(res.statusCode).to.equal(200);
                 expect(res.body.volumes.length).to.be(1);
                 expect(res.body.volumes[0].id).to.be(volumeId);
-                expect(res.body.volumes[0].hostPath).to.be('/media/music');
+                expect(res.body.volumes[0].hostPath).to.be('/media/cloudron-test-music');
                 done();
             });
     });
@@ -111,7 +111,7 @@ describe('Volumes API', function () {
             .end(function (err, res) {
                 expect(res.statusCode).to.equal(200);
                 expect(res.body.id).to.be(volumeId);
-                expect(res.body.hostPath).to.be('/media/music');
+                expect(res.body.hostPath).to.be('/media/cloudron-test-music');
                 done();
             });
     });

src/routes/users.js

@@ -13,6 +13,7 @@ exports = module.exports = {
     setGroups,
     setAvatar,
     clearAvatar,
+    makeOwner,
 
     load
 };
@@ -216,3 +217,19 @@ function clearAvatar(req, res, next) {
         next(new HttpSuccess(202, {}));
     });
 }
+
+// This route transfers ownership from token user to user specified in path param
+function makeOwner(req, res, next) {
+    assert.strictEqual(typeof req.resource, 'object');
+
+    // first make new one owner, then devote current one
+    users.update(req.resource, { role: users.ROLE_OWNER }, auditSource.fromRequest(req), function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        users.update(req.user, { role: users.ROLE_USER }, auditSource.fromRequest(req), function (error) {
+            if (error) return next(BoxError.toHttpError(error));
+
+            next(new HttpSuccess(204));
+        });
+    });
+}

src/routes/wellknown.js

@@ -0,0 +1,21 @@
+'use strict';
+
+exports = module.exports = {
+    get
+};
+
+const domains = require('../domains.js'),
+    HttpError = require('connect-lastmile').HttpError;
+
+function get(req, res, next) {
+    const host = req.headers['host'];
+
+    domains.get(host, function (error, domain) {
+        if (error) return next(new HttpError(404, error.message));
+
+        const location = req.params[0];
+        if (!domain.wellKnown || !(location in domain.wellKnown)) return next(new HttpError(404, 'No custom well-known config'));
+
+        res.status(200).send(domain.wellKnown[location]);
+    });
+}

src/scheduler.js

@@ -1,7 +1,9 @@
 'use strict';
 
 exports = module.exports = {
-    sync: sync
+    sync,
+    suspendJobs,
+    resumeJobs
 };
 
 let apps = require('./apps.js'),
@@ -15,8 +17,19 @@ let apps = require('./apps.js'),
     _ = require('underscore');
 
 // appId -> { containerId, schedulerConfig (manifest), cronjobs }
-var gState = { };
+let gState = { };
+let gSuspendedAppIds = new Set(); // suspended because some apptask is running
 
+// TODO: this should probably also stop existing jobs to completely prevent race but the code is not re-entrant
+function suspendJobs(appId) {
+    debug(`suspendJobs: ${appId}`);
+    gSuspendedAppIds.add(appId);
+}
+
+function resumeJobs(appId) {
+    debug(`resumeJobs: ${appId}`);
+    gSuspendedAppIds.delete(appId);
+}
 
 function runTask(appId, taskName, callback) {
     assert.strictEqual(typeof appId, 'string');
@@ -26,6 +39,8 @@ function runTask(appId, taskName, callback) {
     const JOB_MAX_TIME = 30 * 60 * 1000; // 30 minutes
     const containerName = `${appId}-${taskName}`;
 
+    if (gSuspendedAppIds.has(appId)) return callback();
+
     apps.get(appId, function (error, app) {
         if (error) return callback(error);
 
@@ -61,10 +76,10 @@ function createJobs(app, schedulerConfig, callback) {
         // stopJobs only deletes jobs since previous run. This means that when box code restarts, none of the containers
         // are removed. The deleteContainer here ensures we re-create the cron containers with the latest config
         docker.deleteContainer(containerName, function ( /* ignoredError */) {
-            docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], { } /* options */, function (error, container) {
+            docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], { } /* options */, function (error) {
                 if (error && error.reason !== BoxError.ALREADY_EXISTS) return iteratorDone(error);
 
-                debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${container.id}`);
+                debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`);
 
                 var cronJob = new CronJob({
                     cronTime: cronTime, // at this point, the pattern has been validated

src/server.js

@@ -20,7 +20,7 @@ let assert = require('assert'),
     users = require('./users.js'),
     ws = require('ws');
 
-var gHttpServer = null;
+let gHttpServer = null;
 
 function initializeExpressSync() {
     var app = express();
@@ -86,18 +86,19 @@ function initializeExpressSync() {
     const authorizeUserManager = routes.accesscontrol.authorize(users.ROLE_USER_MANAGER);
 
     // public routes
-    router.post('/api/v1/cloudron/setup',    json, routes.provision.providerTokenAuth, routes.provision.setup);    // only available until no-domain
-    router.post('/api/v1/cloudron/restore',  json, routes.provision.restore);    // only available until activated
-    router.post('/api/v1/cloudron/activate', json, routes.provision.activate);
+    router.post('/api/v1/cloudron/setup',    json, routes.provision.setupTokenAuth, routes.provision.providerTokenAuth, routes.provision.setup);    // only available until no-domain
+    router.post('/api/v1/cloudron/restore',  json, routes.provision.setupTokenAuth, routes.provision.restore);    // only available until activated
+    router.post('/api/v1/cloudron/activate', json, routes.provision.setupTokenAuth, routes.provision.activate);
     router.get ('/api/v1/cloudron/status',         routes.provision.getStatus);
+    router.get ('/api/v1/cloudron/languages',      routes.cloudron.getLanguages);
     router.get ('/api/v1/cloudron/avatar',         routes.branding.getCloudronAvatar); // this is a public alias for /api/v1/branding/cloudron_avatar
 
     // login/logout routes
     router.post('/api/v1/cloudron/login',                  json, password, routes.cloudron.login);
-    router.get ('/api/v1/cloudron/logout',                       routes.cloudron.logout); // this will invalidate the token if any and redirect to /login.html always
-    router.post('/api/v1/cloudron/password_reset_request', json, routes.cloudron.passwordResetRequest);
-    router.post('/api/v1/cloudron/password_reset',         json, routes.cloudron.passwordReset);
-    router.post('/api/v1/cloudron/setup_account',          json, routes.cloudron.setupAccount);
+    router.get ('/api/v1/cloudron/logout',                       token,    routes.cloudron.logout); // this will invalidate the token if any and redirect to /login.html always
+    router.post('/api/v1/cloudron/password_reset_request', json,           routes.cloudron.passwordResetRequest);
+    router.post('/api/v1/cloudron/password_reset',         json,           routes.cloudron.passwordReset);
+    router.post('/api/v1/cloudron/setup_account',          json,           routes.cloudron.setupAccount);
 
     // developer routes
     router.post('/api/v1/developer/login', json, password, routes.cloudron.login); // DEPRECATED we should use the regular /api/v1/cloudron/login
@@ -173,6 +174,7 @@ function initializeExpressSync() {
     router.post('/api/v1/users/:userId',               json, token, authorizeUserManager, routes.users.load, routes.users.update);
     router.post('/api/v1/users/:userId/password',      json, token, authorizeUserManager, routes.users.load, routes.users.changePassword);
     router.put ('/api/v1/users/:userId/groups',        json, token, authorizeUserManager, routes.users.load, routes.users.setGroups);
+    router.post('/api/v1/users/:userId/make_owner',    json, token, authorizeOwner,       routes.users.load, routes.users.makeOwner);
     router.post('/api/v1/users/:userId/send_invite',   json, token, authorizeUserManager, routes.users.load, routes.users.sendInvite);
     router.post('/api/v1/users/:userId/create_invite', json, token, authorizeUserManager, routes.users.load, routes.users.createInvite);
     router.post('/api/v1/users/:userId/avatar',        json, token, authorizeUserManager, routes.users.load, multipart, routes.users.setAvatar);
@@ -220,6 +222,7 @@ function initializeExpressSync() {
     router.post('/api/v1/apps/:id/update',                       json, token, authorizeAdmin, routes.apps.load, routes.apps.update);
     router.post('/api/v1/apps/:id/restore',                      json, token, authorizeAdmin, routes.apps.load, routes.apps.restore);
     router.post('/api/v1/apps/:id/import',                       json, token, authorizeAdmin, routes.apps.load, routes.apps.importApp);
+    router.post('/api/v1/apps/:id/export',                       json, token, authorizeAdmin, routes.apps.load, routes.apps.exportApp);
     router.post('/api/v1/apps/:id/backup',                       json, token, authorizeAdmin, routes.apps.load, routes.apps.backup);
     router.get ('/api/v1/apps/:id/backups',                            token, authorizeAdmin, routes.apps.load, routes.apps.listBackups);
     router.post('/api/v1/apps/:id/start',                        json, token, authorizeAdmin, routes.apps.load, routes.apps.start);
@@ -232,6 +235,7 @@ function initializeExpressSync() {
     router.post('/api/v1/apps/:id/upload',                       json, token, authorizeAdmin, multipart, routes.apps.load, routes.apps.uploadFile);
     router.use ('/api/v1/apps/:id/files/*',                            token, authorizeAdmin, routes.filemanager.proxy);
     router.get ('/api/v1/apps/:id/exec',                               token, authorizeAdmin, routes.apps.load, routes.apps.exec);
+
     // websocket cannot do bearer authentication
     router.get ('/api/v1/apps/:id/execws',    routes.accesscontrol.websocketAuth.bind(null, users.ROLE_ADMIN), routes.apps.load, routes.apps.execWebSocket);
 
@@ -262,6 +266,8 @@ function initializeExpressSync() {
     router.post('/api/v1/mailserver/spam_acl',                token, authorizeAdmin, routes.mailserver.proxy);
     router.get ('/api/v1/mailserver/spam_custom_config',      token, authorizeAdmin, routes.mailserver.proxy);
     router.post('/api/v1/mailserver/spam_custom_config',      token, authorizeAdmin, routes.mailserver.proxy);
+    router.get ('/api/v1/mailserver/solr_config',             token, authorizeAdmin, routes.mailserver.proxy);
+    router.post('/api/v1/mailserver/solr_config',             token, authorizeAdmin, routes.mailserver.proxy, routes.mailserver.restart);
 
     router.get ('/api/v1/mail/:domain',                               token, authorizeAdmin, routes.mail.getDomain);
     router.get ('/api/v1/mail/:domain/status',                        token, authorizeAdmin, routes.mail.getStatus);
@@ -292,12 +298,12 @@ function initializeExpressSync() {
     router.post('/api/v1/support/remote_support', json, token, authorizeAdmin, routes.support.canEnableRemoteSupport, routes.support.enableRemoteSupport);
 
     // domain routes
-    router.post('/api/v1/domains',                   json, token, authorizeAdmin, routes.domains.add);
-    router.get ('/api/v1/domains',                         token,                 routes.domains.getAll);
-    router.get ('/api/v1/domains/:domain',                 token, authorizeAdmin, routes.domains.get);  // this is manage scope because it returns non-restricted fields
-    router.put ('/api/v1/domains/:domain',           json, token, authorizeAdmin, routes.domains.update);
-    router.del ('/api/v1/domains/:domain',                 token, authorizeAdmin, routes.domains.del);
-    router.get ('/api/v1/domains/:domain/dns_check',       token, authorizeAdmin, routes.domains.checkDnsRecords);
+    router.post('/api/v1/domains',                    json, token, authorizeAdmin, routes.domains.add);
+    router.get ('/api/v1/domains',                          token,                 routes.domains.getAll);
+    router.get ('/api/v1/domains/:domain',                  token, authorizeAdmin, routes.domains.get);  // this is manage scope because it returns non-restricted fields
+    router.put ('/api/v1/domains/:domain',            json, token, authorizeAdmin, routes.domains.update);
+    router.del ('/api/v1/domains/:domain',                  token, authorizeAdmin, routes.domains.del);
+    router.get ('/api/v1/domains/:domain/dns_check',        token, authorizeAdmin, routes.domains.checkDnsRecords);
 
     // volume routes
     router.post('/api/v1/volumes',                   json, token, authorizeAdmin, routes.volumes.add);
@@ -306,13 +312,17 @@ function initializeExpressSync() {
     router.del ('/api/v1/volumes/:id',                     token, authorizeAdmin, routes.volumes.load, routes.volumes.del);
     router.use ('/api/v1/volumes/:id/files/*',             token, authorizeAdmin, routes.filemanager.proxy);
 
-    // addon routes
+    // service routes
     router.get ('/api/v1/services',                          token, authorizeAdmin, routes.services.getAll);
     router.get ('/api/v1/services/:service',                 token, authorizeAdmin, routes.services.get);
     router.post('/api/v1/services/:service',           json, token, authorizeAdmin, routes.services.configure);
     router.get ('/api/v1/services/:service/logs',            token, authorizeAdmin, routes.services.getLogs);
     router.get ('/api/v1/services/:service/logstream',       token, authorizeAdmin, routes.services.getLogStream);
     router.post('/api/v1/services/:service/restart',   json, token, authorizeAdmin, routes.services.restart);
+    router.post('/api/v1/services/:service/rebuild',   json, token, authorizeAdmin, routes.services.rebuild);
+
+    // well known
+    router.get ('/well-known-handler/*', routes.wellknown.get);
 
     // disable server socket "idle" timeout. we use the timeout middleware to handle timeouts on a route level
     // we rely on nginx for timeouts on the TCP level (see client_header_timeout)
@@ -372,7 +382,7 @@ function stop(callback) {
     async.series([
         cloudron.uninitialize,
         database.uninitialize,
-        gHttpServer.close.bind(gHttpServer),
+        gHttpServer.close.bind(gHttpServer)
     ], function (error) {
         if (error) return callback(error);