data dir: allow sameness of old and new dir

this makes it easy to migrate to a new volume setup

(cherry picked from commit a32166bc9d)

CHANGES

@@ -2502,71 +2502,3 @@
 * volumes: Ensure long volume names do not overflow the table
 * Move all appstore filter to the left
 * app data: allow sameness of old and new dir
-
-[7.2.5]
-* Fix storage volume migration
-* Fix issue where only 25 group members were returned
-* Fix eventlog display
-
-[7.3.0]
-* Proxied apps
-* Applinks - app bookmarks in dashboard
-* backups: optional encryption of backup file names
-* eventlog: add event for impersonated user login
-* ldap & user directory: Remove virtual user and admin groups
-* Randomize certificate generation cronjob to lighten load on Let's Encrypt servers
-* mail: catch all address can be any domain
-* mail: accept only STARTTLS servers for relay
-* graphs: cgroup v2 support
-* mail: fix issue where signature was appended to text attachments
-* redis: restart button will now rebuild if the container is missing
-* backups: allow space in label name
-* mail: fix crash when solr is enabled on Ubuntu 22 (cgroup v2 detection fix)
-* mail: fix issue where certificate renewal did not restart the mail container properly
-* notification: Fix crash when backupId is null
-* IPv6: initial support for ipv6 only server
-* User directory: Cloudron connector uses 2FA auth
-* port bindings: add read only flag
-* mail: add storage quota support
-* mail: allow aliases to have wildcard
-* proxyAuth: add supportsBearerAuth flag
-* backups: Fix precondition check which was not erroring if mount is missing
-* mail: add queue management API and UI
-* graphs: show app disk usage graphs
-* UI: fix issue where mailbox display name was not init correctly
-* wasabi: add singapore and sydney regions
-* filemanager: add split view
-* nginx: fix zero length certs when out of disk space
-* read only API tokens
-
-[7.3.1]
-* Add cloudlare R2
-* app proxy: fixes to https proxying
-* app links: fix icons
-
-[7.3.2]
-* support: require owner permissions
-* postgresql: fix issue when restoring large dumps
-* graphs: add cpu/disk/network usage
-* graphs: new disk usage UI
-* relay: add office 365
-
-[7.3.3]
-* Fix oom detection in tasks
-* ldap: memberof is a DN and not just group name
-* mail relay: office365 provider
-* If we can't fetch applink upstreamUri, just stop icon and title detection
-* manifest: add runtimeDirs
-* remove external df module
-* Show remaining disk space in usage graph
-* Make users and groups available for the new app link dialog
-* Show swaps in disk graphs
-* disk usage: run once a day
-* mail: fix 100% cpu use with unreachable servers
-* security: do not password reset mail to cloudron owned mail domain
-* logrotate: only keep 14 days of logs
-* mail: fix dnsbl count when all servers are removed
-* applink: make users and groups available for the new app link dialog
-* Show app disk usage in storage tab
-* Make volume read-only checkbox a dropdown
-

box.js

@@ -9,7 +9,7 @@ const fs = require('fs'),
     safe = require('safetydance'),
     server = require('./src/server.js'),
     settings = require('./src/settings.js'),
-    directoryServer = require('./src/directoryserver.js');
+    userdirectory = require('./src/userdirectory.js');
 
 let logFd;
 
@@ -38,8 +38,8 @@ async function startServers() {
     await proxyAuth.start();
     await ldap.start();
 
-    const conf = await settings.getDirectoryServerConfig();
-    if (conf.enabled) await directoryServer.start();
+    const conf = await settings.getUserDirectoryConfig();
+    if (conf.enabled) await userdirectory.start();
 }
 
 async function main() {
@@ -49,18 +49,12 @@ async function main() {
     // require this here so that logging handler is already setup
     const debug = require('debug')('box:box');
 
-    process.on('SIGHUP', async function () {
-        debug('Received SIGHUP. Re-reading configs.');
-        const conf = await settings.getDirectoryServerConfig();
-        if (conf.enabled) await directoryServer.checkCertificate();
-    });
-
     process.on('SIGINT', async function () {
         debug('Received SIGINT. Shutting down.');
 
         await proxyAuth.stop();
         await server.stop();
-        await directoryServer.stop();
+        await userdirectory.stop();
         await ldap.stop();
         setTimeout(process.exit.bind(process), 3000);
     });
@@ -70,7 +64,7 @@ async function main() {
 
         await proxyAuth.stop();
         await server.stop();
-        await directoryServer.stop();
+        await userdirectory.stop();
         await ldap.stop();
         setTimeout(process.exit.bind(process), 3000);
     });

migrations/20220602050517-apps-add-storageVolumeId.js

@@ -13,16 +13,14 @@ function getMountPoint(dataDir) {
 }
 
 exports.up = async function(db) {
-    // use safe() here because this migration failed midway in 7.2.4
-    await safe(db.runSql('ALTER TABLE apps ADD storageVolumeId VARCHAR(128), ADD FOREIGN KEY(storageVolumeId) REFERENCES volumes(id)'));
-    await safe(db.runSql('ALTER TABLE apps ADD storageVolumePrefix VARCHAR(128)'));
-    await safe(db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_storageVolume UNIQUE (storageVolumeId, storageVolumePrefix)'));
+    await db.runSql('ALTER TABLE apps ADD storageVolumeId VARCHAR(128), ADD FOREIGN KEY(storageVolumeId) REFERENCES volumes(id)');
+    await db.runSql('ALTER TABLE apps ADD storageVolumePrefix VARCHAR(128)');
+    await db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_storageVolume UNIQUE (storageVolumeId, storageVolumePrefix)');
 
     const apps = await db.runSql('SELECT * FROM apps WHERE dataDir IS NOT NULL');
+    const allVolumes = await db.runSql('SELECT * FROM volumes');
 
     for (const app of apps) {
-        const allVolumes = await db.runSql('SELECT * FROM volumes');
-
         console.log(`data-dir (${app.id}): migrating data dir ${app.dataDir}`);
 
         const mountPoint = getMountPoint(app.dataDir);
@@ -38,7 +36,7 @@ exports.up = async function(db) {
         }
 
         const id = uuid.v4().replace(/-/g, ''); // to make systemd mount file names more readable
-        const name = `appdata-${id}`;
+        const name = `app-${app.id}`;
         const type = app.dataDir === mountPoint ? 'filesystem' : 'mountpoint';
 
         console.log(`data-dir (${app.id}): creating new volume ${id}`);

migrations/20220607204734-apps-add-upstreamUri.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = async function (db) {
-    await db.runSql('ALTER TABLE apps ADD COLUMN upstreamUri VARCHAR(256) DEFAULT ""');
-};
-
-exports.down = async function (db) {
-    await db.runSql('ALTER TABLE apps DROP COLUMN upstreamUri');
-};

migrations/20220626163116-backupConfig-default-encryptedFilenames.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = async function(db) {
-    const result = await db.runSql('SELECT * FROM settings WHERE name=?', [ 'backup_config' ]);
-    if (!result.length) return;
-
-    const backupConfig = JSON.parse(result[0].value);
-
-    if (backupConfig.encryption && backupConfig.format === 'rsync') backupConfig.encryptedFilenames = true;
-
-    await db.runSql('UPDATE settings SET value=? WHERE name=?', [ JSON.stringify(backupConfig), 'backup_config',  ]);
-};
-
-exports.down = async function(/* db */) {
-};

migrations/20220706170039-applinks-create-table.js

@@ -1,22 +0,0 @@
-'use strict';
-
-exports.up = async function (db) {
-    var cmd = 'CREATE TABLE applinks(' +
-                'id VARCHAR(128) NOT NULL UNIQUE,' +
-                'accessRestrictionJson TEXT,' +
-                'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
-                'updateTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
-                'ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,' +
-                'label VARCHAR(128),' +
-                'tagsJson VARCHAR(2048),' +
-                'icon MEDIUMBLOB,' +
-                'upstreamUri VARCHAR(256) DEFAULT "",' +
-                'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin';
-
-    await db.runSql(cmd);
-};
-
-exports.down = async function (db) {
-    await db.runSql('DROP TABLE applinks');
-};
-

migrations/20220817211634-mailboxes-add-quota.js

@@ -1,17 +0,0 @@
-'use strict';
-
-const async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN storageQuota BIGINT DEFAULT 0'),
-        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN messagesQuota BIGINT DEFAULT 0'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE mailboxes DROP COLUMN storageQuota'),
-        db.runSql.bind(db, 'ALTER TABLE mailboxes DROP COLUMN messagesQuota')
-    ], callback);
-};

migrations/20220911090713-mail-catchAll-make-absolute.js

@@ -1,18 +0,0 @@
-'use strict';
-
-const safe = require('safetydance');
-
-exports.up = async function (db) {
-    const mailDomains = await db.runSql('SELECT * FROM mail', []);
-
-    for (const mailDomain of mailDomains) {
-        let catchAll = safe.JSON.parse(mailDomain.catchAllJson) || [];
-        if (catchAll.length === 0) continue;
-
-        catchAll = catchAll.map(a => `${a}@${mailDomain.domain}`);
-        await db.runSql('UPDATE mail SET catchAllJson = ? WHERE domain = ?', [ JSON.stringify(catchAll), mailDomain.domain ]);
-    }
-};
-
-exports.down = async function( /* db */) {
-};

migrations/20220922194751-tokens-add-scopeJson.js

@@ -1,13 +0,0 @@
-'use strict';
-
-exports.up = async function (db) {
-    await db.runSql('ALTER TABLE tokens DROP COLUMN scope');
-    await db.runSql('ALTER TABLE tokens ADD COLUMN scopeJson TEXT');
-
-    await db.runSql('UPDATE tokens SET scopeJson = ?', [ JSON.stringify({'*':'rw'})]);
-};
-
-exports.down = async function (db) {
-    await db.runSql('ALTER TABLE tokens ADD COLUMN scope VARCHAR(512) NOT NULL DEFAULT ""');
-    await db.runSql('ALTER TABLE tokens DROP COLUMN scopeJson');
-};

migrations/schema.sql

@@ -58,7 +58,7 @@ CREATE TABLE IF NOT EXISTS tokens(
     accessToken VARCHAR(128) NOT NULL UNIQUE,
     identifier VARCHAR(128) NOT NULL, // resourceId: app id or user id
     clientId VARCHAR(128),
-    scopeJson TEXT,
+    scope VARCHAR(512) NOT NULL,
     expires BIGINT NOT NULL, // FIXME: make this a timestamp
     lastUsedTime TIMESTAMP NULL,
     PRIMARY KEY(accessToken));
@@ -102,7 +102,6 @@ CREATE TABLE IF NOT EXISTS apps(
     appStoreIcon MEDIUMBLOB,
     icon MEDIUMBLOB,
     crontab TEXT,
-    upstreamUri VARCHAR(256) DEFAULT "",
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
@@ -217,8 +216,6 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     domain VARCHAR(128),
     active BOOLEAN DEFAULT 1,
     enablePop3 BOOLEAN DEFAULT 0,
-    storageQuota BIGINT DEFAULT 0,
-    messagesQuota BIGINT DEFAULT 0,
 
     FOREIGN KEY(domain) REFERENCES mail(domain),
     FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
@@ -300,17 +297,4 @@ CREATE TABLE IF NOT EXISTS blobs(
     value MEDIUMBLOB,
     PRIMARY KEY(id));
 
-CREATE TABLE IF NOT EXISTS appLinks(
-    id VARCHAR(128) NOT NULL UNIQUE,
-    accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
-    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, // when the app was installed
-    updateTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,   // when the last app update was done
-    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, // when this db record was updated (useful for UI caching)
-    label VARCHAR(128),     // display name
-    tagsJson VARCHAR(2048), // array of tags
-    icon MEDIUMBLOB,
-    upstreamUri VARCHAR(256) DEFAULT "",
-
-    PRIMARY KEY(id));
-
 CHARACTER SET utf8 COLLATE utf8_bin;

package.json

@@ -12,12 +12,13 @@
   },
   "dependencies": {
     "@google-cloud/dns": "^2.2.4",
-    "@google-cloud/storage": "^5.20.5",
-    "async": "^3.2.4",
-    "aws-sdk": "^2.1248.0",
+    "@google-cloud/storage": "^5.19.2",
+    "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
+    "async": "^3.2.3",
+    "aws-sdk": "^2.1115.0",
     "basic-auth": "^2.0.1",
-    "body-parser": "^1.20.1",
-    "cloudron-manifestformat": "^5.19.0",
+    "body-parser": "^1.20.0",
+    "cloudron-manifestformat": "^5.16.0",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.1.1",
     "connect-timeout": "^1.9.0",
@@ -27,33 +28,39 @@
     "db-migrate": "^0.11.13",
     "db-migrate-mysql": "^2.2.0",
     "debug": "^4.3.4",
-    "dockerode": "^3.3.4",
-    "ejs": "^3.1.8",
-    "express": "^4.18.2",
+    "dockerode": "^3.3.1",
+    "ejs": "^3.1.6",
+    "ejs-cli": "^2.2.3",
+    "express": "^4.17.3",
     "ipaddr.js": "^2.0.1",
-    "jsdom": "^20.0.2",
+    "js-yaml": "^4.1.0",
+    "json": "^11.0.0",
     "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.3.3",
+    "ldapjs": "^2.3.2",
     "lodash": "^4.17.21",
-    "moment": "^2.29.4",
-    "moment-timezone": "^0.5.38",
+    "moment": "^2.29.2",
+    "moment-timezone": "^0.5.34",
     "morgan": "^1.10.0",
     "multiparty": "^4.2.3",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.8.0",
-    "qrcode": "^1.5.1",
+    "nodemailer": "^6.7.3",
+    "nodemailer-smtp-transport": "^2.7.4",
+    "progress-stream": "^2.0.0",
+    "qrcode": "^1.5.0",
     "readdirp": "^3.6.0",
     "safetydance": "^2.2.0",
-    "semver": "^7.3.8",
+    "semver": "^7.3.7",
     "speakeasy": "^2.0.0",
-    "superagent": "^7.1.5",
+    "split": "^1.0.1",
+    "superagent": "^7.1.1",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
+    "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
-    "ua-parser-js": "^1.0.32",
-    "underscore": "^1.13.6",
+    "ua-parser-js": "^1.0.2",
+    "underscore": "^1.13.2",
     "uuid": "^8.3.2",
     "validator": "^13.7.0",
-    "ws": "^8.10.0",
+    "ws": "^8.5.0",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
@@ -62,9 +69,15 @@
     "js2xmlparser": "^4.0.2",
     "mocha": "^9.2.2",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^13.2.9"
+    "nock": "^13.2.4",
+    "node-sass": "^7.0.1",
+    "nyc": "^15.1.0"
   },
   "scripts": {
-    "test": "./run-tests"
+    "test": "./runTests",
+    "postmerge": "/bin/true",
+    "precommit": "/bin/true",
+    "prepush": "npm test",
+    "dashboard": "node_modules/.bin/gulp"
   }
 }

runTests

@@ -6,7 +6,7 @@ readonly source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly DATA_DIR="${HOME}/.cloudron_test"
 readonly DEFAULT_TESTS="./src/test/*-test.js ./src/routes/test/*-test.js"
 
-! "${source_dir}/src/test/check-install" && exit 1
+! "${source_dir}/src/test/checkInstall" && exit 1
 
 # cleanup old data dirs some of those docker container data requires sudo to be removed
 echo "=> Provide root password to purge any leftover data in ${DATA_DIR} and load apparmor profile:"
@@ -23,7 +23,7 @@ mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
 mkdir -p boxdata/box boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
-mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications/dashboard platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update
+mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update
 sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
 
 # translations
@@ -39,7 +39,7 @@ if [[ -z ${FAST+x} ]]; then
     echo "=> Delete all docker containers first"
     docker ps -qa --filter "label=isCloudronManaged" | xargs --no-run-if-empty docker rm -f
     docker rm -f mysql-server
-    echo "==> To skip this run with: FAST=1 ./run-tests"
+    echo "==> To skip this run with: FAST=1 ./runTests"
 else
     echo "==> WARNING!! Skipping docker container cleanup, the database might not be pristine!"
 fi
@@ -84,5 +84,10 @@ if [[ $# -gt 0 ]]; then
     TESTS="$*"
 fi
 
-echo "=> Run tests with mocha"
-BOX_ENV=test ./node_modules/.bin/mocha --bail --no-timeouts --exit -R spec ${TESTS}
+if [[ -z ${COVERAGE+x} ]]; then
+    echo "=> Run tests with mocha"
+    BOX_ENV=test ./node_modules/.bin/mocha --bail --no-timeouts --exit -R spec ${TESTS}
+else
+    echo "=> Run tests with mocha and coverage"
+    BOX_ENV=test ./node_modules/.bin/nyc --reporter=html  ./node_modules/.bin/mocha --no-timeouts --exit -R spec ${TESTS}
+fi

scripts/cloudron-setup

@@ -11,7 +11,7 @@ trap exitHandler EXIT
 # change this to a hash when we make a upgrade release
 readonly LOG_FILE="/var/log/cloudron-setup.log"
 readonly MINIMUM_DISK_SIZE_GB="18" # this is the size of "/" and required to fit in docker images 18 is a safe bet for different reporting on 20GB min
-readonly MINIMUM_MEMORY="960"      # this is mostly reported for 1GB main memory (DO 992, EC2 967, Linode 989, Serverdiscounter.com 974)
+readonly MINIMUM_MEMORY="974"      # this is mostly reported for 1GB main memory (DO 992, EC2 990, Linode 989, Serverdiscounter.com 974)
 
 readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"
 
@@ -236,31 +236,20 @@ while true; do
     sleep 10
 done
 
-ip4=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
-ip6=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv6.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
-
-url4=""
-url6=""
-fallbackUrl=""
-if [[ -z "${setupToken}" ]]; then
-    [[ -n "${ip4}" ]] && url4="https://${ip4}"
-    [[ -n "${ip6}" ]] && url6="https://[${ip6}]"
-    [[ -z "${ip4}" && -z "${ip6}" ]] && fallbackUrl="https://<IP>"
-else
-    [[ -n "${ip4}" ]] && url4="https://${ip4}/?setupToken=${setupToken}"
-    [[ -n "${ip6}" ]] && url6="https://[${ip6}]/?setupToken=${setupToken}"
-    [[ -z "${ip4}" && -z "${ip6}" ]] && fallbackUrl="https://<IP>?setupToken=${setupToken}"
+if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+    ip='<IP>'
 fi
-echo -e "\n\n${GREEN}After reboot, visit one of the following URLs and accept the self-signed certificate to finish setup.${DONE}\n"
-[[ -n "${url4}" ]] && echo -e "  * ${GREEN}${url4}${DONE}"
-[[ -n "${url6}" ]] && echo -e "  * ${GREEN}${url6}${DONE}"
-[[ -n "${fallbackUrl}" ]] && echo -e "  * ${GREEN}${fallbackUrl}${DONE}"
+if [[ -z "${setupToken}" ]]; then
+    url="https://${ip}"
+else
+    url="https://${ip}/?setupToken=${setupToken}"
+fi
+echo -e "\n\n${GREEN}After reboot, visit ${url} and accept the self-signed certificate to finish setup.${DONE}\n"
 
 if [[ "${rebootServer}" == "true" ]]; then
     systemctl stop box mysql # sometimes mysql ends up having corrupt privilege tables
 
-    # https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#ANSI_002dC-Quoting
-    read -p $'\n'"The server has to be rebooted to apply all the settings. Reboot now ? [Y/n] " yn
+    read -p "The server has to be rebooted to apply all the settings. Reboot now ? [Y/n] " yn
     yn=${yn:-y}
     case $yn in
         [Yy]* ) exitHandler; systemctl reboot;;

scripts/create-release-tarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v16.18.1" ]]; then
-    echo "This script requires node 16.18.1"
+if [[ "$(node --version)" != "v16.13.1" ]]; then
+    echo "This script requires node 16.13.1"
     exit 1
 fi
 

scripts/init-ubuntu.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 # This script is run on the base ubuntu. Put things here which are managed by ubuntu
-# This script is also run after ubuntu upgrade
 
 set -euv -o pipefail
 
@@ -121,13 +120,13 @@ else
     if ! apt-get install -y --no-install-recommends collectd collectd-utils; then
         # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
         echo "Failed to install collectd, continuing anyway. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
+        sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
     fi
 
     if [[ "${ubuntu_version}" == "20.04" ]]; then
         echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
     fi
 fi
-sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
 
 # some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
 echo "==> Configuring host"
@@ -180,7 +179,7 @@ systemctl disable systemd-resolved || true
 ufw disable || true
 
 # we need unbound to work as this is required for installer.sh to do any DNS requests
-echo -e "server:\n\tinterface: 127.0.0.1\n" > /etc/unbound/unbound.conf.d/cloudron-network.conf
+echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: no" > /etc/unbound/unbound.conf.d/cloudron-network.conf
 systemctl restart unbound
 
 # Ubuntu 22 has private home directories by default (https://discourse.ubuntu.com/t/private-home-directories-for-ubuntu-21-04-onwards/)

scripts/installer.sh

@@ -69,11 +69,10 @@ readonly ubuntu_codename=$(lsb_release -cs)
 
 readonly is_update=$(systemctl is-active -q box && echo "yes" || echo "no")
 
-log "Updating from $(cat $box_src_dir/VERSION 2>/dev/null) to $(cat $box_src_tmp_dir/VERSION 2>/dev/null)"
+log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION)"
 
 # https://docs.docker.com/engine/installation/linux/ubuntulinux/
-readonly docker_version=20.10.21
-readonly containerd_version=1.6.10-1
+readonly docker_version=20.10.14
 if ! which docker 2>/dev/null || [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
     log "installing/updating docker"
 
@@ -81,8 +80,8 @@ if ! which docker 2>/dev/null || [[ $(docker version --format {{.Client.Version}
     mkdir -p /etc/systemd/system/docker.service.d
     echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
 
-    # there are 3 packages for docker - containerd, CLI and the daemon (https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/)
-    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_${containerd_version}_amd64.deb" -o /tmp/containerd.deb
+    # there are 3 packages for docker - containerd, CLI and the daemon
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.5.11-1_amd64.deb" -o /tmp/containerd.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
@@ -115,14 +114,14 @@ elif [[ "${ubuntu_version}" == "18.04" ]]; then
     fi
 fi
 
-readonly node_version=16.18.1
+readonly node_version=16.14.2
 if ! which node 2>/dev/null || [[ "$(node --version)" != "v${node_version}" ]]; then
     log "installing/updating node ${node_version}"
     mkdir -p /usr/local/node-${node_version}
     $curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-${node_version}
     ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
     ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-16.14.2
+    rm -rf /usr/local/node-16.13.1
 fi
 
 # note that rebuild requires the above node
@@ -146,18 +145,12 @@ log "downloading new addon images"
 images=$(node -e "let i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
 log "\tPulling docker images: ${images}"
-if ! curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip; then
-    docker_registry=registry.ipv6.docker.com
-else
-    docker_registry=registry-1.docker.io
-fi
-
 for image in ${images}; do
-    while ! docker pull "${docker_registry}/${image}"; do           # this pulls the image using the sha256
+    while ! docker pull "${image}"; do           # this pulls the image using the sha256
         log "Could not pull ${image}"
         sleep 5
     done
-    while ! docker pull "${docker_registry}/${image%@sha256:*}"; do  # this will tag the image for readability
+    while ! docker pull "${image%@sha256:*}"; do  # this will tag the image for readability
         log "Could not pull ${image%@sha256:*}"
         sleep 5
    done
@@ -170,8 +163,7 @@ CLOUDRON_SYSLOG_VERSION="1.1.0"
 while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLOUDRON_SYSLOG_VERSION} ]]; do
     rm -rf "${CLOUDRON_SYSLOG_DIR}"
     mkdir -p "${CLOUDRON_SYSLOG_DIR}"
-    # verbatim is not needed in node 18 since that is the default there. in node 16, ipv4 is preferred and this breaks on ipv6 only servers
-    if NODE_OPTIONS="--dns-result-order=verbatim" npm install --unsafe-perm -g --prefix "${CLOUDRON_SYSLOG_DIR}" cloudron-syslog@${CLOUDRON_SYSLOG_VERSION}; then break; fi
+    if npm install --unsafe-perm -g --prefix "${CLOUDRON_SYSLOG_DIR}" cloudron-syslog@${CLOUDRON_SYSLOG_VERSION}; then break; fi
     log "Failed to install cloudron-syslog, trying again"
     sleep 5
 done

scripts/recreate-containers

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly logfile="/home/yellowtent/platformdata/logs/box.log"
-
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-echo "This will re-create all the containers. Services will go down for a bit."
-
-read -p "Do you want to proceed? (y/N) " -n 1 -r choice
-echo
-
-if [[ ! $choice =~ ^[Yy]$ ]]; then
-    exit 1
-fi
-
-echo -n "Re-creating addon containers (this takes a while) ."
-line_count=$(cat /home/yellowtent/platformdata/logs/box.log | wc -l)
-sed -e 's/"version": ".*",/"version":"48.0.0",/' -i /home/yellowtent/platformdata/INFRA_VERSION
-systemctl restart box
-
-while ! tail -n "+${line_count}" "${logfile}" | grep -q "platform is ready"; do
-    echo -n "."
-    sleep 2
-done
-
-echo -e "\nDone.\nThe Cloudron dashboard will say 'Configuring (Queued)' for each app. The apps will come up in a short while."
-

setup/start.sh

@@ -20,6 +20,7 @@ readonly BOX_DATA_DIR="${HOME_DIR}/boxdata/box"
 readonly MAIL_DATA_DIR="${HOME_DIR}/boxdata/mail"
 
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly json="$(realpath ${script_dir}/../node_modules/.bin/json)"
 readonly ubuntu_version=$(lsb_release -rs)
 
 cp -f "${script_dir}/../scripts/cloudron-support" /usr/bin/cloudron-support
@@ -56,10 +57,9 @@ mkdir -p "${PLATFORM_DATA_DIR}/mysql"
 mkdir -p "${PLATFORM_DATA_DIR}/postgresql"
 mkdir -p "${PLATFORM_DATA_DIR}/mongodb"
 mkdir -p "${PLATFORM_DATA_DIR}/redis"
-mkdir -p "${PLATFORM_DATA_DIR}/tls"
 mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner" \
          "${PLATFORM_DATA_DIR}/addons/mail/dkim"
-mkdir -p "${PLATFORM_DATA_DIR}/collectd"
+mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
 mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
 mkdir -p "${PLATFORM_DATA_DIR}/acme"
 mkdir -p "${PLATFORM_DATA_DIR}/backup"
@@ -107,6 +107,8 @@ unbound-anchor -a /var/lib/unbound/root.key
 
 log "Adding systemd services"
 cp -r "${script_dir}/start/systemd/." /etc/systemd/system/
+[[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/MemoryMax/MemoryLimit/g' -i /etc/systemd/system/box.service
+[[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/Type=notify/Type=simple/g' -i /etc/systemd/system/unbound.service
 systemctl daemon-reload
 systemctl enable --now cloudron-syslog
 systemctl enable unbound
@@ -131,7 +133,7 @@ rm -f /etc/sudoers.d/${USER} /etc/sudoers.d/cloudron
 cp "${script_dir}/start/sudoers" /etc/sudoers.d/cloudron
 
 log "Configuring collectd"
-rm -rf /etc/collectd /var/log/collectd.log "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
+rm -rf /etc/collectd /var/log/collectd.log
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
 systemctl restart collectd
@@ -161,7 +163,7 @@ log "Configuring nginx"
 # link nginx config to system config
 unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
 ln -s "${PLATFORM_DATA_DIR}/nginx" /etc/nginx
-mkdir -p "${PLATFORM_DATA_DIR}/nginx/applications/dashboard"
+mkdir -p "${PLATFORM_DATA_DIR}/nginx/applications"
 mkdir -p "${PLATFORM_DATA_DIR}/nginx/cert"
 cp "${script_dir}/start/nginx/nginx.conf" "${PLATFORM_DATA_DIR}/nginx/nginx.conf"
 cp "${script_dir}/start/nginx/mime.types" "${PLATFORM_DATA_DIR}/nginx/mime.types"
@@ -229,7 +231,7 @@ log "Changing ownership"
 # note, change ownership after db migrate. this allow db migrate to move files around as root and then we can fix it up here
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
 chown -R "${USER}" /etc/cloudron
-chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall" "${PLATFORM_DATA_DIR}/sshfs" "${PLATFORM_DATA_DIR}/cifs" "${PLATFORM_DATA_DIR}/tls"
+chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall" "${PLATFORM_DATA_DIR}/sshfs" "${PLATFORM_DATA_DIR}/cifs"
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 chown "${USER}:${USER}" "${APPS_DATA_DIR}"

setup/start/cloudron-motd

@@ -4,51 +4,30 @@
 
 printf "**********************************************************************\n\n"
 
-readonly cache_file4="/var/cache/cloudron-motd-cache4"
-readonly cache_file6="/var/cache/cloudron-motd-cache6"
+cache_file="/var/cache/cloudron-motd-cache"
 
-url4=""
-url6=""
-fallbackUrl=""
-
-function detectIp() {
-    if [[ ! -f "${cache_file4}" ]]; then
-        ip4=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
-        [[ -n "${ip4}" ]] && echo "${ip4}" > "${cache_file4}"
-    else
-        ip4=$(cat "${cache_file4}")
+if [[ -z "$(ls -A /home/yellowtent/platformdata/addons/mail/dkim)" ]]; then
+    if [[ ! -f "${cache_file}" ]]; then
+        curl --fail --connect-timeout 2 --max-time 2 -q https://ipv4.api.cloudron.io/api/v1/helper/public_ip --output "${cache_file}" || true
     fi
-
-    if [[ ! -f "${cache_file6}" ]]; then
-        ip6=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv6.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
-        [[ -n "${ip6}" ]] && echo "${ip6}" > "${cache_file6}"
+    if [[ -f "${cache_file}" ]]; then
+        ip=$(sed -n -e 's/.*"ip": "\(.*\)"/\1/p' /var/cache/cloudron-motd-cache)
     else
-        ip6=$(cat "${cache_file6}")
+        ip='<IP>'
     fi
 
     if [[ ! -f /etc/cloudron/SETUP_TOKEN ]]; then
-        [[ -n "${ip4}" ]] && url4="https://${ip4}"
-        [[ -n "${ip6}" ]] && url6="https://[${ip6}]"
-        [[ -z "${ip4}" && -z "${ip6}" ]] && fallbackUrl="https://<IP>"
+        url="https://${ip}"
     else
         setupToken="$(cat /etc/cloudron/SETUP_TOKEN)"
-        [[ -n "${ip4}" ]] && url4="https://${ip4}/?setupToken=${setupToken}"
-        [[ -n "${ip6}" ]] && url6="https://[${ip6}]/?setupToken=${setupToken}"
-        [[ -z "${ip4}" && -z "${ip6}" ]] && fallbackUrl="https://<IP>?setupToken=${setupToken}"
+        url="https://${ip}/?setupToken=${setupToken}"
     fi
-}
-
-if [[ -z "$(ls -A /home/yellowtent/platformdata/addons/mail/dkim)" ]]; then
-    detectIp
 
     printf "\t\t\tWELCOME TO CLOUDRON\n"
     printf "\t\t\t-------------------\n"
 
-    printf '\n\e[1;32m%-6s\e[m\n' "Visit one of the following URLs on your browser and accept the self-signed certificate to finish setup."
-    [[ -n "${url4}" ]] && printf '\e[1;32m%-6s\e[m\n' "  * ${url4}"
-    [[ -n "${url6}" ]] && printf '\e[1;32m%-6s\e[m\n' "  * ${url6}"
-    [[ -n "${fallbackUrl}" ]] && printf '\e[1;32m%-6s\e[m\n' "  * ${fallbackUrl}"
-    printf "\nCloudron overview - https://docs.cloudron.io/ \n"
+    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit ${url} on your browser and accept the self-signed certificate to finish setup."
+    printf "Cloudron overview - https://docs.cloudron.io/ \n"
     printf "Cloudron setup - https://docs.cloudron.io/installation/#setup \n"
 else
     printf "\t\t\tNOTE TO CLOUDRON ADMINS\n"

setup/start/collectd/collectd.conf

@@ -187,9 +187,9 @@ LoadPlugin swap
 
        CalculateNum false
        CalculateSum true
-       CalculateAverage false
+       CalculateAverage true
        CalculateMinimum false
-       CalculateMaximum false
+       CalculateMaximum true
        CalculateStddev false
    </Aggregation>
 </Plugin>
@@ -211,7 +211,23 @@ LoadPlugin swap
     Interactive false
 
     Import "df"
-    Import "docker-stats"
+
+    Import "du"
+    <Module du>
+        <Path>
+            Instance maildata
+            Dir "/home/yellowtent/boxdata/mail"
+        </Path>
+        <Path>
+            Instance boxdata
+            Dir "/home/yellowtent/boxdata"
+            Exclude "mail"
+        </Path>
+        <Path>
+            Instance platformdata
+            Dir "/home/yellowtent/platformdata"
+        </Path>
+    </Module>
 </Plugin>
 
 <Plugin write_graphite>
@@ -227,3 +243,6 @@ LoadPlugin swap
    </Node>
 </Plugin>
 
+<Include "/etc/collectd/collectd.conf.d">
+    Filter "*.conf"
+</Include>

setup/start/collectd/docker-stats.py

@@ -1,64 +0,0 @@
-import collectd,os,subprocess,json,re
-
-# https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/
-
-def parseSiSize(size):
-    units = {"B": 1, "KB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12}
-    number, unit, _ = re.split('([a-zA-Z]+)', size.upper())
-    return int(float(number)*units[unit])
-
-def parseBinarySize(size):
-    units = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "TIB": 2**40}
-    number, unit, _ = re.split('([a-zA-Z]+)', size.upper())
-    return int(float(number)*units[unit])
-
-def init():
-    collectd.info('custom docker-status plugin initialized')
-
-def read():
-    try:
-        lines = subprocess.check_output('docker stats --format  "{{ json . }}" --no-stream --no-trunc', shell=True).decode('utf-8').strip().split("\n")
-    except Exception as e:
-        collectd.info('\terror getting docker stats: %s' % (str(e)))
-        return 0
-
-    # Sample line
-    # {"BlockIO":"430kB / 676kB","CPUPerc":"0.00%","Container":"7eae5e6f4f11","ID":"7eae5e6f4f11","MemPerc":"59.15%","MemUsage":"45.55MiB / 77MiB","Name":"1062eef3-ec96-4d81-9f02-15b7dd81ccb9","NetIO":"1.5MB / 3.48MB","PIDs":"5"}
-
-    for line in lines:
-        stat = json.loads(line)
-        containerName = stat["Name"] # same as app id
-        networkData = stat["NetIO"].split("/")
-        networkRead = parseSiSize(networkData[0].strip())
-        networkWrite = parseSiSize(networkData[1].strip())
-
-        blockData = stat["BlockIO"].split("/")
-        blockRead = parseSiSize(blockData[0].strip())
-        blockWrite = parseSiSize(blockData[1].strip())
-
-        memUsageData = stat["MemUsage"].split("/")
-        memUsed = parseBinarySize(memUsageData[0].strip())
-        memMax = parseBinarySize(memUsageData[1].strip())
-
-        cpuPercData = stat["CPUPerc"].strip("%")
-        cpuPerc = float(cpuPercData)
-
-        # type comes from https://github.com/collectd/collectd/blob/master/src/types.db and https://collectd.org/wiki/index.php/Data_source
-        val = collectd.Values(type='gauge', plugin='docker-stats', plugin_instance=containerName)
-        val.dispatch(values=[networkRead], type_instance='network-read')
-        val.dispatch(values=[networkWrite], type_instance='network-write')
-        val.dispatch(values=[blockRead], type_instance='blockio-read')
-        val.dispatch(values=[blockWrite], type_instance='blockio-write')
-        val.dispatch(values=[memUsed], type_instance='mem-used')
-        val.dispatch(values=[memMax], type_instance='mem-max')
-        val.dispatch(values=[cpuPerc], type_instance='cpu-perc')
-
-        val = collectd.Values(type='counter', plugin='docker-stats', plugin_instance=containerName)
-        val.dispatch(values=[networkRead], type_instance='network-read')
-        val.dispatch(values=[networkWrite], type_instance='network-write')
-        val.dispatch(values=[blockRead], type_instance='blockio-read')
-        val.dispatch(values=[blockWrite], type_instance='blockio-write')
-
-collectd.register_init(init)
-# see Interval setting in collectd.conf for polling interval
-collectd.register_read(read)

setup/start/collectd/du.py

@@ -0,0 +1,105 @@
+import collectd,os,subprocess,sys,re,time
+
+# https://www.programcreek.com/python/example/106897/collectd.register_read
+
+PATHS = [] # { name, dir, exclude }
+# there is a pattern in carbon/storage-schemas.conf which stores values every 12h for a year
+INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this
+
+# we used to pass the INTERVAL as a parameter to register_read. however, collectd write_graphite
+# takes a bit to load (tcp connection) and drops the du data. this then means that we have to wait
+# for INTERVAL secs for du data. instead, we just cache the value for INTERVAL instead
+CACHE = dict()
+CACHE_TIME = 0
+
+def du(pathinfo):
+    # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
+    dirname = pathinfo['dir']
+    cmd = 'timeout 1800 du -DsB1 "{}"'.format(dirname)
+    if pathinfo['exclude'] != '':
+        cmd += ' --exclude "{}"'.format(pathinfo['exclude'])
+
+    collectd.info('computing size with command: %s' % cmd);
+    try:
+        size = subprocess.check_output(cmd, shell=True).split()[0].decode('utf-8')
+        collectd.info('\tsize of %s is %s (time: %i)' % (dirname, size, int(time.time())))
+        return size
+    except Exception as e:
+        collectd.info('\terror getting the size of %s: %s' % (dirname, str(e)))
+        return 0
+
+def parseSize(size):
+    units = {"B": 1, "KB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12}
+    number, unit, _ = re.split('([a-zA-Z]+)', size.upper())
+    return int(float(number)*units[unit])
+
+def dockerSize():
+    # use --format '{{json .}}' to dump the string. '{{if eq .Type "Images"}}{{.Size}}{{end}}' still creates newlines
+    # https://godoc.org/github.com/docker/go-units#HumanSize is used. so it's 1000 (KB) and not 1024 (KiB)
+    cmd = 'timeout 1800 docker system df --format "{{.Size}}" | head -n1'
+    try:
+        size = subprocess.check_output(cmd, shell=True).strip().decode('utf-8')
+        collectd.info('size of docker images is %s (%s) (time: %i)' % (size, parseSize(size), int(time.time())))
+        return parseSize(size)
+    except Exception as e:
+        collectd.info('error getting docker images size : %s' % str(e))
+        return 0
+
+# configure is called for each module block. this is called before init
+def configure(config):
+    global PATHS
+
+    for child in config.children:
+        if child.key != 'Path':
+            collectd.info('du plugin: Unknown config key "%s"' % key)
+            continue
+
+        pathinfo = { 'name': '', 'dir': '', 'exclude': '' }
+        for node in child.children:
+            if node.key == 'Instance':
+                pathinfo['name'] = node.values[0]
+            elif node.key == 'Dir':
+                pathinfo['dir'] = node.values[0]
+            elif node.key == 'Exclude':
+                pathinfo['exclude'] = node.values[0]
+
+        PATHS.append(pathinfo);
+        collectd.info('du plugin: monitoring %s' % pathinfo['dir']);
+
+def init():
+    global PATHS
+    collectd.info('custom du plugin initialized with %s %s' % (PATHS, sys.version))
+
+def read():
+    global CACHE, CACHE_TIME
+
+    # read from cache if < 12 hours
+    read_cache = (time.time() - CACHE_TIME) < INTERVAL
+
+    if not read_cache:
+        CACHE_TIME = time.time()
+
+    for pathinfo in PATHS:
+        dirname = pathinfo['dir']
+        if read_cache and dirname in CACHE:
+            size = CACHE[dirname]
+        else:
+            size = du(pathinfo)
+            CACHE[dirname] = size
+
+        # type comes from https://github.com/collectd/collectd/blob/master/src/types.db
+        val = collectd.Values(type='capacity', plugin='du', plugin_instance=pathinfo['name'])
+        val.dispatch(values=[size], type_instance='usage')
+
+    if read_cache and 'docker' in CACHE:
+        size = CACHE['docker']
+    else:
+        size = dockerSize()
+        CACHE['docker'] = size
+
+    val = collectd.Values(type='capacity', plugin='du', plugin_instance='docker')
+    val.dispatch(values=[size], type_instance='usage')
+
+collectd.register_init(init)
+collectd.register_config(configure)
+collectd.register_read(read)

setup/start/logrotate/box

@@ -1,11 +1,9 @@
 # logrotate config for box logs
 
-# we rotate weekly, unless 10M was hit. Keep only up to 5 rotated files. Also, delete if > 14 days old
+# keep upto 5 logs of size 10M each
 /home/yellowtent/platformdata/logs/box.log {
     rotate 5
-    weekly
-    maxage 14
-    maxsize 10M
+    size 10M
     # we never compress so we can simply tail the files
     nocompress
     copytruncate

setup/start/logrotate/platform

@@ -14,9 +14,7 @@
 /home/yellowtent/platformdata/logs/updater/*.log {
     # only keep one rotated file, we currently do not send that over the api
     rotate 1
-    weekly
-    maxage 14
-    maxsize 10M
+    size 10M
     missingok
     # we never compress so we can simply tail the files
     nocompress
@@ -25,7 +23,7 @@
 }
 
 # keep task logs for a week. the 'nocreate' option ensures empty log files are not
-# created post rotation. task logs are kept for 7 days
+# created post rotation
 /home/yellowtent/platformdata/logs/tasks/*.log {
     minage 7
     daily

setup/start/nginx/nginx.conf

@@ -39,5 +39,4 @@ http {
     limit_req_zone $binary_remote_addr zone=admin_login:10m rate=10r/s; # 10 request a second
 
     include applications/*.conf;
-    include applications/*/*.conf;
 }

setup/start/sudoers

@@ -19,6 +19,9 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmaddondir.sh
 Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 
+Defaults!/home/yellowtent/box/src/scripts/configurecollectd.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/configurecollectd.sh
+
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 
@@ -65,7 +68,5 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmount.sh
 Defaults!/home/yellowtent/box/src/scripts/remountmount.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/remountmount.sh
 
-Defaults!/home/yellowtent/box/src/scripts/du.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/du.sh
-
 cloudron-support ALL=(ALL) NOPASSWD: ALL
+

setup/start/systemd/box.service

@@ -13,7 +13,6 @@ Type=idle
 WorkingDirectory=/home/yellowtent/box
 Restart=always
 ExecStart=/home/yellowtent/box/box.js
-ExecReload=/usr/bin/kill -HUP $MAINPID
 ; we run commands like df which will parse properly only with correct locale
 Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C"
 ; kill apptask processes as well

setup/start/unbound.conf

@@ -6,7 +6,7 @@ server:
         interface: 127.0.0.1
         interface: 172.18.0.1
         ip-freebind: yes
-        do-ip6: yes
+        do-ip6: no
         access-control: 127.0.0.1 allow
         access-control: 172.18.0.1/16 allow
         cache-max-negative-ttl: 30

src/accesscontrol.js

@@ -0,0 +1,26 @@
+'use strict';
+
+exports = module.exports = {
+    verifyToken
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    safe = require('safetydance'),
+    tokens = require('./tokens.js'),
+    users = require('./users.js');
+
+async function verifyToken(accessToken) {
+    assert.strictEqual(typeof accessToken, 'string');
+
+    const token = await tokens.getByAccessToken(accessToken);
+    if (!token) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'No such token');
+
+    const user = await users.get(token.identifier);
+    if (!user) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not found');
+    if (!user.active) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not active');
+
+    await safe(tokens.update(token.id, { lastUsedTime: new Date() })); // ignore any error
+
+    return user;
+}

src/acme2.js

@@ -17,11 +17,9 @@ const assert = require('assert'),
     fs = require('fs'),
     os = require('os'),
     path = require('path'),
-    paths = require('./paths.js'),
     promiseRetry = require('./promise-retry.js'),
     superagent = require('superagent'),
     safe = require('safetydance'),
-    users = require('./users.js'),
     _ = require('underscore');
 
 const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
@@ -31,26 +29,16 @@ const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
 // https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
 // https://community.letsencrypt.org/t/list-of-client-implementations/2103
 
-function Acme2(fqdn, domainObject, email) {
-    assert.strictEqual(typeof fqdn, 'string');
-    assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof email, 'string');
+function Acme2(options) {
+    assert.strictEqual(typeof options, 'object');
 
-    this.fqdn = fqdn;
-    this.accountKey = null;
-    this.email = email;
+    this.accountKeyPem = null; // Buffer .
+    this.email = options.email;
     this.keyId = null;
-    const prod = domainObject.tlsConfig.provider.match(/.*-prod/) !== null; // matches 'le-prod' or 'letsencrypt-prod'
-    this.caDirectory = prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
+    this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
     this.directory = {};
-    this.performHttpAuthorization = domainObject.provider.match(/noop|manual|wildcard/) !== null;
-    this.wildcard = !!domainObject.tlsConfig.wildcard;
-    this.domain = domainObject.domain;
-
-    this.cn = fqdn !== this.domain && this.wildcard ? dns.makeWildcard(fqdn) : fqdn; // bare domain is not part of wildcard SAN
-    this.certName = this.cn.replace('*.', '_.');
-
-    debug(`Acme2: will get cert for fqdn: ${this.fqdn} cn: ${this.cn} certName: ${this.certName} wildcard: ${this.wildcard} http: ${this.performHttpAuthorization}`);
+    this.performHttpAuthorization = !!options.performHttpAuthorization;
+    this.wildcard = !!options.wildcard;
 }
 
 // urlsafe base64 encoding (jose)
@@ -64,7 +52,7 @@ function b64(str) {
 }
 
 function getModulus(pem) {
-    assert.strictEqual(typeof pem, 'string');
+    assert(Buffer.isBuffer(pem));
 
     const stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
     if (!stdout) return null;
@@ -76,7 +64,8 @@ function getModulus(pem) {
 Acme2.prototype.sendSignedRequest = async function (url, payload) {
     assert.strictEqual(typeof url, 'string');
     assert.strictEqual(typeof payload, 'string');
-    assert.strictEqual(typeof this.accountKey, 'string');
+
+    assert(Buffer.isBuffer(this.accountKeyPem));
 
     const that = this;
     let header = {
@@ -91,7 +80,7 @@ Acme2.prototype.sendSignedRequest = async function (url, payload) {
         header.jwk = {
             e: b64(Buffer.from([0x01, 0x00, 0x01])), // exponent - 65537
             kty: 'RSA',
-            n: b64(getModulus(this.accountKey))
+            n: b64(getModulus(this.accountKeyPem))
         };
     }
 
@@ -110,7 +99,7 @@ Acme2.prototype.sendSignedRequest = async function (url, payload) {
 
     const signer = crypto.createSign('RSA-SHA256');
     signer.update(protected64 + '.' + payload64, 'utf8');
-    const signature64 = urlBase64Encode(signer.sign(that.accountKey, 'base64'));
+    const signature64 = urlBase64Encode(signer.sign(that.accountKeyPem, 'base64'));
 
     const data = {
         protected: protected64,
@@ -146,7 +135,7 @@ Acme2.prototype.updateContact = async function (registrationUri) {
 };
 
 async function generateAccountKey() {
-    const acmeAccountKey = safe.child_process.execSync('openssl genrsa 4096', { encoding: 'utf8' });
+    const acmeAccountKey = safe.child_process.execSync('openssl genrsa 4096');
     if (!acmeAccountKey) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate acme account key: ${safe.error.message}`);
     return acmeAccountKey;
 }
@@ -158,18 +147,18 @@ Acme2.prototype.ensureAccount = async function () {
 
     debug('ensureAccount: registering user');
 
-    this.accountKey = await blobs.getString(blobs.ACME_ACCOUNT_KEY);
-    if (!this.accountKey) {
+    this.accountKeyPem = await blobs.get(blobs.ACME_ACCOUNT_KEY);
+    if (!this.accountKeyPem) {
         debug('ensureAccount: generating new account keys');
-        this.accountKey = await generateAccountKey();
-        await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey);
+        this.accountKeyPem = await generateAccountKey();
+        await blobs.set(blobs.ACME_ACCOUNT_KEY, this.accountKeyPem);
     }
 
     let result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
     if (result.status === 403 && result.body.type === 'urn:ietf:params:acme:error:unauthorized') {
         debug(`ensureAccount: key was revoked. ${result.status} ${JSON.stringify(result.body)}. generating new account key`);
-        this.accountKey = await generateAccountKey();
-        await blobs.setString(blobs.ACME_ACCOUNT_KEY, this.accountKey);
+        this.accountKeyPem = await generateAccountKey();
+        await blobs.set(blobs.ACME_ACCOUNT_KEY, this.accountKeyPem);
         result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
     }
 
@@ -183,21 +172,23 @@ Acme2.prototype.ensureAccount = async function () {
     await this.updateContact(result.headers.location);
 };
 
-Acme2.prototype.newOrder = async function () {
+Acme2.prototype.newOrder = async function (domain) {
+    assert.strictEqual(typeof domain, 'string');
+
     const payload = {
         identifiers: [{
             type: 'dns',
-            value: this.cn
+            value: domain
         }]
     };
 
-    debug(`newOrder: ${this.cn}`);
+    debug(`newOrder: ${domain}`);
 
     const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload));
     if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`);
     if (result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`);
 
-    debug(`newOrder: created order ${this.cn} %j`, result.body);
+    debug('newOrder: created order %s %j', domain, result.body);
 
     const order = result.body, orderUrl = result.headers.location;
 
@@ -231,12 +222,12 @@ Acme2.prototype.waitForOrder = async function (orderUrl) {
 };
 
 Acme2.prototype.getKeyAuthorization = function (token) {
-    assert(typeof this.accountKey, 'string');
+    assert(Buffer.isBuffer(this.accountKeyPem));
 
     let jwk = {
         e: b64(Buffer.from([0x01, 0x00, 0x01])), // Exponent - 65537
         kty: 'RSA',
-        n: b64(getModulus(this.accountKey))
+        n: b64(getModulus(this.accountKeyPem))
     };
 
     let shasum = crypto.createHash('sha256');
@@ -284,12 +275,10 @@ Acme2.prototype.waitForChallenge = async function (challenge) {
 };
 
 // https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
-Acme2.prototype.signCertificate = async function (finalizationUrl, csrPem) {
+Acme2.prototype.signCertificate = async function (domain, finalizationUrl, csrDer) {
+    assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof finalizationUrl, 'string');
-    assert.strictEqual(typeof csrPem, 'string');
-
-    const csrDer = safe.child_process.execSync('openssl req -inform pem -outform der', { input: csrPem });
-    if (!csrDer) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+    assert(Buffer.isBuffer(csrDer));
 
     const payload = {
         csr: b64(csrDer)
@@ -302,28 +291,22 @@ Acme2.prototype.signCertificate = async function (finalizationUrl, csrPem) {
     if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to sign certificate. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
 };
 
-Acme2.prototype.ensureKey = async function () {
-    const key = await blobs.getString(`${blobs.CERT_PREFIX}-${this.certName}.key`);
-    if (key) {
-        debug(`ensureKey: reuse existing key for ${this.cn}`);
-        return key;
+Acme2.prototype.createKeyAndCsr = async function (hostname, keyFilePath, csrFilePath) {
+    assert.strictEqual(typeof hostname, 'string');
+
+    if (safe.fs.existsSync(keyFilePath)) {
+        debug('createKeyAndCsr: reuse the key for renewal at %s', keyFilePath);
+    } else {
+        let key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
+        if (!key) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+        if (!safe.fs.writeFileSync(keyFilePath, key)) throw new BoxError(BoxError.FS_ERROR, safe.error);
+
+        debug('createKeyAndCsr: key file saved at %s', keyFilePath);
     }
 
-    debug(`ensureKey: generating new key for ${this.cn}`);
-    const newKey = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1', { encoding: 'utf8' }); // openssl ecparam -list_curves
-    if (!newKey) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
-    return newKey;
-};
-
-Acme2.prototype.createCsr = async function (key) {
-    assert.strictEqual(typeof key, 'string');
-
     const [error, tmpdir] = await safe(fs.promises.mkdtemp(path.join(os.tmpdir(), 'acme-')));
     if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating temporary directory for openssl config: ${error.message}`);
 
-    const keyFilePath = path.join(tmpdir, 'key');
-    if (!safe.fs.writeFileSync(keyFilePath, key)) throw new BoxError(BoxError.FS_ERROR, `Failed to write key file: ${safe.error.message}`);
-
     // OCSP must-staple is currently disabled because nginx does not provide staple on the first request (https://forum.cloudron.io/topic/4917/ocsp-stapling-for-tls-ssl/)
     // ' -addext "tlsfeature = status_request"'; // this adds OCSP must-staple
     // we used to use -addext to the CLI to add these but that arg doesn't work on Ubuntu 16.04
@@ -331,37 +314,47 @@ Acme2.prototype.createCsr = async function (key) {
     const conf = '[req]\nreq_extensions = v3_req\ndistinguished_name = req_distinguished_name\n'
         + '[req_distinguished_name]\n\n'
         + '[v3_req]\nbasicConstraints = CA:FALSE\nkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\nsubjectAltName = @alt_names\n'
-        + `[alt_names]\nDNS.1 = ${this.cn}\n`;
+        + `[alt_names]\nDNS.1 = ${hostname}\n`;
 
     const opensslConfigFile = path.join(tmpdir, 'openssl.conf');
     if (!safe.fs.writeFileSync(opensslConfigFile, conf)) throw new BoxError(BoxError.FS_ERROR, `Failed to write openssl config: ${safe.error.message}`);
 
     // while we pass the CN anyways, subjectAltName takes precedence
-    const csrPem = safe.child_process.execSync(`openssl req -new -key ${keyFilePath} -outform PEM -subj /CN=${this.cn} -config ${opensslConfigFile}`, { encoding: 'utf8' });
-    if (!csrPem) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+    const csrDer = safe.child_process.execSync(`openssl req -new -key ${keyFilePath} -outform DER -subj /CN=${hostname} -config ${opensslConfigFile}`);
+    if (!csrDer) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+    if (!safe.fs.writeFileSync(csrFilePath, csrDer)) throw new BoxError(BoxError.FS_ERROR, safe.error); // bookkeeping. inspect with openssl req -text -noout -in hostname.csr -inform der
 
     await safe(fs.promises.rm(tmpdir, { recursive: true, force: true }));
-    debug(`createCsr: csr file created for ${this.cn}`);
-    return csrPem; // inspect with openssl req -text -noout -in hostname.csr -inform pem
+
+    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFilePath);
+
+    return csrDer;
 };
 
-Acme2.prototype.downloadCertificate = async function (certUrl) {
+Acme2.prototype.downloadCertificate = async function (hostname, certUrl, certFilePath) {
+    assert.strictEqual(typeof hostname, 'string');
     assert.strictEqual(typeof certUrl, 'string');
 
-    return await promiseRetry({ times: 5, interval: 20000, debug }, async () => {
-        debug(`downloadCertificate: downloading certificate of ${this.cn}`);
+    await promiseRetry({ times: 5, interval: 20000, debug }, async () => {
+        debug(`downloadCertificate: downloading certificate of ${hostname}`);
 
         const result = await this.postAsGet(certUrl);
         if (result.statusCode === 202) throw new BoxError(BoxError.ACME_ERROR, 'Retry downloading certificate');
         if (result.statusCode !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
 
-        const fullChainPem = result.body.toString('utf8'); // buffer
-        return fullChainPem;
+        const fullChainPem = result.body; // buffer
+
+        if (!safe.fs.writeFileSync(certFilePath, fullChainPem)) throw new BoxError(BoxError.FS_ERROR, safe.error);
+
+        debug(`downloadCertificate: cert file for ${hostname} saved at ${certFilePath}`);
     });
 };
 
-Acme2.prototype.prepareHttpChallenge = async function (authorization) {
+Acme2.prototype.prepareHttpChallenge = async function (hostname, domain, authorization, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof authorization, 'object');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
 
     debug('prepareHttpChallenge: challenges: %j', authorization);
     let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
@@ -372,39 +365,44 @@ Acme2.prototype.prepareHttpChallenge = async function (authorization) {
 
     let keyAuthorization = this.getKeyAuthorization(challenge.token);
 
-    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
+    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(acmeChallengesDir, challenge.token));
 
-    if (!safe.fs.writeFileSync(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), keyAuthorization)) throw new BoxError(BoxError.FS_ERROR, `Error writing challenge: ${safe.error.message}`);
+    if (!safe.fs.writeFileSync(path.join(acmeChallengesDir, challenge.token), keyAuthorization)) throw new BoxError(BoxError.FS_ERROR, `Error writing challenge: ${safe.error.message}`);
 
     return challenge;
 };
 
-Acme2.prototype.cleanupHttpChallenge = async function (challenge) {
+Acme2.prototype.cleanupHttpChallenge = async function (hostname, domain, challenge, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
 
-    debug('cleanupHttpChallenge: unlinking %s', path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
+    debug('cleanupHttpChallenge: unlinking %s', path.join(acmeChallengesDir, challenge.token));
 
-    if (!safe.fs.unlinkSync(path.join(paths.ACME_CHALLENGES_DIR, challenge.token))) throw new BoxError(BoxError.FS_ERROR, `Error unlinking challenge: ${safe.error.message}`);
+    if (!safe.fs.unlinkSync(path.join(acmeChallengesDir, challenge.token))) throw new BoxError(BoxError.FS_ERROR, `Error unlinking challenge: ${safe.error.message}`);
 };
 
-function getChallengeSubdomain(cn, domain) {
+function getChallengeSubdomain(hostname, domain) {
     let challengeSubdomain;
 
-    if (cn === domain) {
+    if (hostname === domain) {
         challengeSubdomain = '_acme-challenge';
-    } else if (cn.includes('*')) { // wildcard
-        let subdomain = cn.slice(0, -domain.length - 1);
+    } else if (hostname.includes('*')) { // wildcard
+        let subdomain = hostname.slice(0, -domain.length - 1);
         challengeSubdomain = subdomain ? subdomain.replace('*', '_acme-challenge') : '_acme-challenge';
     } else {
-        challengeSubdomain = '_acme-challenge.' + cn.slice(0, -domain.length - 1);
+        challengeSubdomain = '_acme-challenge.' + hostname.slice(0, -domain.length - 1);
     }
 
-    debug(`getChallengeSubdomain: challenge subdomain for cn ${cn} at domain ${domain} is ${challengeSubdomain}`);
+    debug(`getChallengeSubdomain: challenge subdomain for hostname ${hostname} at domain ${domain} is ${challengeSubdomain}`);
 
     return challengeSubdomain;
 }
 
-Acme2.prototype.prepareDnsChallenge = async function (authorization) {
+Acme2.prototype.prepareDnsChallenge = async function (hostname, domain, authorization) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof authorization, 'object');
 
     debug('prepareDnsChallenge: challenges: %j', authorization);
@@ -417,34 +415,39 @@ Acme2.prototype.prepareDnsChallenge = async function (authorization) {
     shasum.update(keyAuthorization);
 
     const txtValue = urlBase64Encode(shasum.digest('base64'));
-    const challengeSubdomain = getChallengeSubdomain(this.cn, this.domain);
+    const challengeSubdomain = getChallengeSubdomain(hostname, domain);
 
     debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
 
-    await dns.upsertDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]);
+    await dns.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
 
-    await dns.waitForDnsRecord(challengeSubdomain, this.domain, 'TXT', txtValue, { times: 200 });
+    await dns.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 });
 
     return challenge;
 };
 
-Acme2.prototype.cleanupDnsChallenge = async function (challenge) {
+Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challenge) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof challenge, 'object');
 
     const keyAuthorization = this.getKeyAuthorization(challenge.token);
-    const shasum = crypto.createHash('sha256');
+    let shasum = crypto.createHash('sha256');
     shasum.update(keyAuthorization);
 
     const txtValue = urlBase64Encode(shasum.digest('base64'));
-    const challengeSubdomain = getChallengeSubdomain(this.cn, this.domain);
+    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
 
     debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
 
-    await dns.removeDnsRecords(challengeSubdomain, this.domain, 'TXT', [ `"${txtValue}"` ]);
+    await dns.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
 };
 
-Acme2.prototype.prepareChallenge = async function (authorizationUrl) {
+Acme2.prototype.prepareChallenge = async function (hostname, domain, authorizationUrl, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof authorizationUrl, 'string');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
 
     debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
 
@@ -454,49 +457,55 @@ Acme2.prototype.prepareChallenge = async function (authorizationUrl) {
     const authorization = response.body;
 
     if (this.performHttpAuthorization) {
-        return await this.prepareHttpChallenge(authorization);
+        return await this.prepareHttpChallenge(hostname, domain, authorization, acmeChallengesDir);
     } else {
-        return await this.prepareDnsChallenge(authorization);
+        return await this.prepareDnsChallenge(hostname, domain, authorization);
     }
 };
 
-Acme2.prototype.cleanupChallenge = async function (challenge) {
+Acme2.prototype.cleanupChallenge = async function (hostname, domain, challenge, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
 
     debug(`cleanupChallenge: http: ${this.performHttpAuthorization}`);
 
     if (this.performHttpAuthorization) {
-        await this.cleanupHttpChallenge(challenge);
+        await this.cleanupHttpChallenge(hostname, domain, challenge, acmeChallengesDir);
     } else {
-        await this.cleanupDnsChallenge(challenge);
+        await this.cleanupDnsChallenge(hostname, domain, challenge);
     }
 };
 
-Acme2.prototype.acmeFlow = async function () {
-    await this.ensureAccount();
-    const { order, orderUrl } = await this.newOrder();
+Acme2.prototype.acmeFlow = async function (hostname, domain, paths) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof paths, 'object');
 
-    const certificates = [];
+    const { certFilePath, keyFilePath, csrFilePath, acmeChallengesDir } = paths;
+
+    await this.ensureAccount();
+    const { order, orderUrl } = await this.newOrder(hostname);
 
     for (let i = 0; i < order.authorizations.length; i++) {
         const authorizationUrl = order.authorizations[i];
         debug(`acmeFlow: authorizing ${authorizationUrl}`);
 
-        const challenge = await this.prepareChallenge(authorizationUrl);
+        const challenge = await this.prepareChallenge(hostname, domain, authorizationUrl, acmeChallengesDir);
         await this.notifyChallengeReady(challenge);
         await this.waitForChallenge(challenge);
-        const key = await this.ensureKey();
-        const csr = await this.createCsr(key);
-        await this.signCertificate(order.finalize, csr);
+        const csrDer = await this.createKeyAndCsr(hostname, keyFilePath, csrFilePath);
+        await this.signCertificate(hostname, order.finalize, csrDer);
         const certUrl = await this.waitForOrder(orderUrl);
-        const cert = await this.downloadCertificate(certUrl);
+        await this.downloadCertificate(hostname, certUrl, certFilePath);
 
-        await safe(this.cleanupChallenge(challenge), { debug });
-
-        certificates.push({ cert, key, csr });
+        try {
+            await this.cleanupChallenge(hostname, domain, challenge, acmeChallengesDir);
+        } catch (cleanupError) {
+            debug('acmeFlow: ignoring error when cleaning up challenge:', cleanupError);
+        }
     }
-
-    return certificates;
 };
 
 Acme2.prototype.loadDirectory = async function () {
@@ -513,36 +522,32 @@ Acme2.prototype.loadDirectory = async function () {
     });
 };
 
-Acme2.prototype.getCertificate = async function () {
-    debug(`getCertificate: start acme flow for ${this.cn} from ${this.caDirectory}`);
+Acme2.prototype.getCertificate = async function (vhost, domain, paths) {
+    assert.strictEqual(typeof vhost, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof paths, 'object');
+
+    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
+
+    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
+        vhost = dns.makeWildcard(vhost);
+        debug(`getCertificate: will get wildcard cert for ${vhost}`);
+    }
 
     await this.loadDirectory();
-    const result = await this.acmeFlow();
-
-    debug(`getCertificate: acme flow completed for ${this.cn}. result: ${result.length}`);
-
-    await blobs.setString(`${blobs.CERT_PREFIX}-${this.certName}.key`, result[0].key);
-    await blobs.setString(`${blobs.CERT_PREFIX}-${this.certName}.cert`, result[0].cert);
-    await blobs.setString(`${blobs.CERT_PREFIX}-${this.certName}.csr`, result[0].csr);
-
-    return result[0];
+    await this.acmeFlow(vhost, domain, paths);
 };
 
-async function getCertificate(fqdn, domainObject) {
-    assert.strictEqual(typeof fqdn, 'string'); // this can also be a wildcard domain (for alias domains)
-    assert.strictEqual(typeof domainObject, 'object');
+async function getCertificate(vhost, domain, paths, options) {
+    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof paths, 'object');
+    assert.strictEqual(typeof options, 'object');
 
-    // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
-    // we cannot use admin@fqdn because the user might not have set it up.
-    // we simply update the account with the latest email we have each time when getting letsencrypt certs
-    // https://github.com/ietf-wg-acme/acme/issues/30
-    const owner = await users.getOwner();
-    const email = owner?.email || 'webmaster@cloudron.io'; // can error if not activated yet
+    await promiseRetry({ times: 3, interval: 0, debug }, async function () {
+        debug(`getCertificate: for vhost ${vhost} and domain ${domain}`);
 
-    return await promiseRetry({ times: 3, interval: 0, debug }, async function () {
-        debug(`getCertificate: for fqdn ${fqdn} and domain ${domainObject.domain}`);
-
-        const acme = new Acme2(fqdn, domainObject, email);
-        return await acme.getCertificate();
+        const acme = new Acme2(options || { });
+        return await acme.getCertificate(vhost, domain, paths);
     });
 }

src/apphealthmonitor.js

@@ -70,36 +70,25 @@ async function checkAppHealth(app, options) {
 
     const manifest = app.manifest;
 
-    let healthCheckUrl, host;
-    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) {
-        healthCheckUrl = app.upstreamUri;
-        host = new URL(app.upstreamUri).host; // includes port
-    } else {
-        const [error, data] = await safe(docker.inspect(app.containerId));
-        if (error || !data || !data.State) return await setHealth(app, apps.HEALTH_ERROR);
-        if (data.State.Running !== true) return await setHealth(app, apps.HEALTH_DEAD);
+    const [error, data] = await safe(docker.inspect(app.containerId));
+    if (error || !data || !data.State) return await setHealth(app, apps.HEALTH_ERROR);
+    if (data.State.Running !== true) return await setHealth(app, apps.HEALTH_DEAD);
 
-        // non-appstore apps may not have healthCheckPath
-        if (!manifest.healthCheckPath) return await setHealth(app, apps.HEALTH_HEALTHY);
-
-        healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
-        host = app.fqdn;
-    }
+    // non-appstore apps may not have healthCheckPath
+    if (!manifest.healthCheckPath) return await setHealth(app, apps.HEALTH_HEALTHY);
 
+    const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
     const [healthCheckError, response] = await safe(superagent
         .get(healthCheckUrl)
-        .disableTLSCerts() // for app proxy
-        .set('Host', host) // required for some apache configs with rewrite rules
+        .set('Host', app.fqdn) // required for some apache configs with rewrite rules
         .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
         .redirects(0)
         .ok(() => true)
         .timeout(options.timeout * 1000));
 
     if (healthCheckError) {
-        await apps.appendLogLine(app, `=> Healtheck error: ${healthCheckError}`);
         await setHealth(app, apps.HEALTH_UNHEALTHY);
     } else if (response.status > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
-        await apps.appendLogLine(app, `=> Healtheck error got response status ${response.status}`);
         await setHealth(app, apps.HEALTH_UNHEALTHY);
     } else {
         await setHealth(app, apps.HEALTH_HEALTHY);
@@ -140,7 +129,9 @@ async function processDockerEvents(options) {
         const [error, info] = await safe(getContainerInfo(containerId));
         const program = error ? containerId : (info.addonName || info.app.fqdn);
         const now = Date.now();
-        const notifyUser = !info?.app?.debugMode && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
+
+        // do not send mails for dev apps
+        const notifyUser = !(info.app && info.app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
 
         debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
 
@@ -172,10 +163,10 @@ async function processApp(options) {
 
     await Promise.allSettled(healthChecks); // wait for all promises to finish
 
-    const stopped = allApps.filter(app => app.runState === apps.RSTATE_STOPPED);
-    const running = allApps.filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
+    const alive = allApps
+        .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
 
-    debug(`app health: ${running.length} running / ${stopped.length} stopped / ${allApps.length - running.length - stopped.length} unresponsive`);
+    debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.`);
 }
 
 async function run(intervalSecs) {

src/applinks.js

@@ -1,213 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    list,
-    listByUser,
-    add,
-    get,
-    update,
-    remove,
-    getIcon
-};
-
-const assert = require('assert'),
-    apps = require('./apps.js'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js'),
-    debug = require('debug')('box:applinks'),
-    jsdom = require('jsdom'),
-    safe = require('safetydance'),
-    superagent = require('superagent'),
-    uuid = require('uuid'),
-    validator = require('validator');
-
-const APPLINKS_FIELDS= [ 'id', 'accessRestrictionJson', 'creationTime', 'updateTime', 'ts', 'label', 'tagsJson', 'icon', 'upstreamUri' ].join(',');
-
-function postProcess(result) {
-    assert.strictEqual(typeof result, 'object');
-
-    assert(result.tagsJson === null || typeof result.tagsJson === 'string');
-    result.tags = safe.JSON.parse(result.tagsJson) || [];
-    delete result.tagsJson;
-
-    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
-    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
-    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
-    delete result.accessRestrictionJson;
-
-    result.ts = new Date(result.ts).getTime();
-
-    result.icon = result.icon ? result.icon : null;
-
-}
-
-function validateUpstreamUri(upstreamUri) {
-    assert.strictEqual(typeof upstreamUri, 'string');
-
-    if (!upstreamUri) return new BoxError(BoxError.BAD_FIELD, 'upstreamUri cannot be empty');
-
-    if (!upstreamUri.includes('://')) return new BoxError(BoxError.BAD_FIELD, 'upstreamUri has no schema');
-
-    const uri = safe(() => new URL(upstreamUri));
-    if (!uri) return new BoxError(BoxError.BAD_FIELD, `upstreamUri is invalid: ${safe.error.message}`);
-    if (uri.protocol !== 'http:' && uri.protocol !== 'https:') return new BoxError(BoxError.BAD_FIELD, 'upstreamUri has an unsupported scheme');
-
-    return null;
-}
-
-async function list() {
-    const results = await database.query(`SELECT ${APPLINKS_FIELDS} FROM applinks ORDER BY upstreamUri`);
-
-    results.forEach(postProcess);
-
-    return results;
-}
-
-async function listByUser(user) {
-    assert.strictEqual(typeof user, 'object');
-
-    const result = await list();
-    return result.filter((app) => apps.canAccess(app, user));
-}
-
-async function detectMetaInfo(applink) {
-    assert.strictEqual(typeof applink, 'object');
-
-    const [error, response] = await safe(superagent.get(applink.upstreamUri).set('User-Agent', 'Mozilla'));
-    if (error || !response.text) {
-        debug('detectMetaInfo: Unable to fetch upstreamUri to detect icon and title', error.statusCode);
-        return;
-    }
-
-    if (applink.favicon && applink.label) return;
-
-    // set redirected URI if any for favicon url
-    const redirectUri = (response.redirects && response.redirects.length) ? response.redirects[0] : null;
-
-    const dom = new jsdom.JSDOM(response.text);
-    if (!applink.icon) {
-        let favicon = '';
-        if (dom.window.document.querySelector('link[rel="apple-touch-icon"]')) favicon = dom.window.document.querySelector('link[rel="apple-touch-icon"]').href ;
-        if (!favicon.endsWith('.png') && dom.window.document.querySelector('meta[name="msapplication-TileImage"]')) favicon = dom.window.document.querySelector('meta[name="msapplication-TileImage"]').content ;
-        if (!favicon.endsWith('.png') && dom.window.document.querySelector('link[rel="shortcut icon"]')) favicon = dom.window.document.querySelector('link[rel="shortcut icon"]').href ;
-        if (!favicon.endsWith('.png') && dom.window.document.querySelector('link[rel="icon"]')) {
-            let iconElements = dom.window.document.querySelectorAll('link[rel="icon"]');
-            if (iconElements.length) {
-                favicon = iconElements[0].href; // choose first one for a start
-
-                // check if we have sizes attributes and then choose the largest one
-                iconElements = Array.from(iconElements).filter(function (e) {
-                    return e.attributes.getNamedItem('sizes') && e.attributes.getNamedItem('sizes').value;
-                }).sort(function (a, b) {
-                    return parseInt(b.attributes.getNamedItem('sizes').value.split('x')[0]) - parseInt(a.attributes.getNamedItem('sizes').value.split('x')[0]);
-                });
-                if (iconElements.length) favicon = iconElements[0].href;
-            }
-        }
-        if (!favicon.endsWith('.png') && dom.window.document.querySelector('meta[itemprop="image"]')) favicon = dom.window.document.querySelector('meta[itemprop="image"]').content;
-
-        if (favicon) {
-            if (favicon.startsWith('/')) favicon = (redirectUri || applink.upstreamUri) + favicon;
-
-            debug(`detectMetaInfo: found icon: ${favicon}`);
-
-            const [error, response] = await safe(superagent.get(favicon));
-            if (error) console.error(`Failed to fetch icon ${favicon}: `, error);
-            else if (response.ok && response.headers['content-type'] === 'image/png') applink.icon = response.body;
-            else console.error(`Failed to fetch icon ${favicon}: statusCode=${response.status}`);
-        } else {
-            console.error(`Unable to find a suitable icon for ${applink.upstreamUri}`);
-        }
-    }
-
-    if (!applink.label) {
-        if (dom.window.document.querySelector('meta[property="og:title"]')) applink.label = dom.window.document.querySelector('meta[property="og:title"]').content;
-        else if (dom.window.document.querySelector('meta[property="og:site_name"]')) applink.label = dom.window.document.querySelector('meta[property="og:site_name"]').content;
-        else if (dom.window.document.title) applink.label = dom.window.document.title;
-    }
-}
-
-async function add(applink) {
-    assert.strictEqual(typeof applink, 'object');
-    assert.strictEqual(typeof applink.upstreamUri, 'string');
-
-    debug(`add: ${applink.upstreamUri}`, applink);
-
-    let error = validateUpstreamUri(applink.upstreamUri);
-    if (error) throw error;
-
-    if (applink.icon) {
-        if (!validator.isBase64(applink.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
-        applink.icon = Buffer.from(applink.icon, 'base64');
-    }
-
-    await detectMetaInfo(applink);
-
-    const data = {
-        id: uuid.v4(),
-        accessRestrictionJson: applink.accessRestriction ? JSON.stringify(applink.accessRestriction) : null,
-        label: applink.label || '',
-        tagsJson: applink.tags ? JSON.stringify(applink.tags) : null,
-        icon: applink.icon || null,
-        upstreamUri: applink.upstreamUri
-    };
-
-    const query = 'INSERT INTO applinks (id, accessRestrictionJson, label, tagsJson, icon, upstreamUri) VALUES (?, ?, ?, ?, ?, ?)';
-    const args = [ data.id, data.accessRestrictionJson, data.label, data.tagsJson, data.icon, data.upstreamUri ];
-
-    [error] = await safe(database.query(query, args));
-    if (error) throw error;
-
-    return data.id;
-}
-
-async function get(applinkId) {
-    assert.strictEqual(typeof applinkId, 'string');
-
-    const result = await database.query(`SELECT ${APPLINKS_FIELDS} FROM applinks WHERE id = ?`, [ applinkId ]);
-    if (result.length === 0) return null;
-
-    postProcess(result[0]);
-
-    return result[0];
-}
-
-async function update(applinkId, applink) {
-    assert.strictEqual(typeof applinkId, 'string');
-    assert.strictEqual(typeof applink, 'object');
-    assert.strictEqual(typeof applink.upstreamUri, 'string');
-
-    debug(`update: ${applink.upstreamUri}`, applink);
-
-    let error = validateUpstreamUri(applink.upstreamUri);
-    if (error) throw error;
-
-    if (applink.icon) {
-        if (!validator.isBase64(applink.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
-        applink.icon = Buffer.from(applink.icon, 'base64');
-    }
-
-    await detectMetaInfo(applink);
-
-    const query = 'UPDATE applinks SET label=?, icon=?, upstreamUri=?, tagsJson=?, accessRestrictionJson=? WHERE id = ?';
-    const args = [ applink.label, applink.icon || null, applink.upstreamUri, applink.tags ? JSON.stringify(applink.tags) : null, applink.accessRestriction ? JSON.stringify(applink.accessRestriction) : null, applinkId ];
-
-    const result = await database.query(query, args);
-    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
-}
-
-async function remove(applinkId) {
-    assert.strictEqual(typeof applinkId, 'string');
-
-    const result = await database.query('DELETE FROM applinks WHERE id = ?', [ applinkId ]);
-    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
-}
-
-async function getIcon(applinkId) {
-    assert.strictEqual(typeof applinkId, 'string');
-
-    const applink = await get(applinkId);
-    if (!applink) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
-
-    return applink.icon;
-}

src/apps.js

@@ -27,7 +27,6 @@ exports = module.exports = {
     setAccessRestriction,
     setOperators,
     setCrontab,
-    setUpstreamUri,
     setLabel,
     setIcon,
     setTags,
@@ -56,13 +55,12 @@ exports = module.exports = {
     backup,
     listBackups,
     updateBackup,
-    getBackupDownloadStream,
 
     getTask,
     getLogPaths,
     getLogs,
 
-    appendLogLine,
+    getCertificate,
 
     start,
     stop,
@@ -136,19 +134,9 @@ exports = module.exports = {
     LOCATION_TYPE_REDIRECT: 'redirect',
     LOCATION_TYPE_ALIAS: 'alias',
 
-    // should probably be in table as well
-    LOCATION_TYPE_DASHBOARD: 'dashboard',
-    LOCATION_TYPE_MAIL: 'mail',
-    LOCATION_TYPE_DIRECTORY_SERVER: 'directoryserver',
-
-    // respositories, match with appstore
-    REPOSITORY_CORE: 'core',
-    REPOSITORY_COMMUNITY: 'community',
-
     // exported for testing
     _validatePortBindings: validatePortBindings,
     _validateAccessRestriction: validateAccessRestriction,
-    _validateUpstreamUri: validateUpstreamUri,
     _translatePortBindings: translatePortBindings,
     _parseCrontab: parseCrontab,
     _clear: clear
@@ -168,11 +156,9 @@ const appstore = require('./appstore.js'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
-    LogStream = require('./log-stream.js'),
     mail = require('./mail.js'),
     manifestFormat = require('cloudron-manifestformat'),
     mounts = require('./mounts.js'),
-    notifications = require('./notifications.js'),
     once = require('./once.js'),
     os = require('os'),
     path = require('path'),
@@ -184,11 +170,10 @@ const appstore = require('./appstore.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     spawn = require('child_process').spawn,
-    storage = require('./storage.js'),
+    split = require('split'),
     superagent = require('superagent'),
     system = require('./system.js'),
     tasks = require('./tasks.js'),
-    tgz = require('./backupformat/tgz.js'),
     TransformStream = require('stream').Transform,
     users = require('./users.js'),
     util = require('util'),
@@ -201,12 +186,11 @@ const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationS
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson', 'apps.operatorsJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', 'apps.crontab',
-    'apps.creationTime', 'apps.updateTime', 'apps.enableAutomaticUpdate', 'apps.upstreamUri',
+    'apps.creationTime', 'apps.updateTime', 'apps.enableAutomaticUpdate',
     'apps.enableMailbox', 'apps.mailboxDisplayName', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain',
     'apps.storageVolumeId', 'apps.storageVolumePrefix', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
 
 // const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
-const LOCATION_FIELDS = [ 'appId', 'subdomain', 'domain', 'type', 'certificateJson' ];
 
 const CHECKVOLUME_CMD = path.join(__dirname, 'scripts/checkvolume.sh');
 
@@ -230,6 +214,7 @@ function validatePortBindings(portBindings, manifest) {
         993, /* imaps */
         995, /* pop3s */
         2003, /* graphite (lo) */
+        2004, /* graphite (lo) */
         2514, /* cloudron-syslog (lo) */
         constants.PORT, /* app server (lo) */
         constants.AUTHWALL_PORT, /* protected sites */
@@ -240,6 +225,7 @@ function validatePortBindings(portBindings, manifest) {
         4190, /* managesieve */
         5349, /* turn,stun TLS */
         8000, /* ESXi monitoring */
+        8417, /* graphite (lo) */
     ];
 
     const RESERVED_PORT_RANGES = [
@@ -253,10 +239,7 @@ function validatePortBindings(portBindings, manifest) {
 
     if (!portBindings) return null;
 
-    const tcpPorts = manifest.tcpPorts || { };
-    const udpPorts = manifest.udpPorts || { };
-
-    for (const portName in portBindings) {
+    for (let portName in portBindings) {
         if (!/^[a-zA-Z0-9_]+$/.test(portName)) return new BoxError(BoxError.BAD_FIELD, `${portName} is not a valid environment variable in portBindings`);
 
         const hostPort = portBindings[portName];
@@ -264,11 +247,14 @@ function validatePortBindings(portBindings, manifest) {
         if (RESERVED_PORTS.indexOf(hostPort) !== -1) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} for ${portName} is reserved in portBindings`);
         if (RESERVED_PORT_RANGES.find(range => (hostPort >= range[0] && hostPort <= range[1]))) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} for ${portName} is reserved in portBindings`);
         if (ALLOWED_PORTS.indexOf(hostPort) === -1 && (hostPort <= 1023 || hostPort > 65535)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} for ${portName} is not in permitted range in portBindings`);
+    }
 
-        // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies the service is disabled
-        const portSpec = tcpPorts[portName] || udpPorts[portName];
-        if (!portSpec) return new BoxError(BoxError.BAD_FIELD, `Invalid portBinding ${portName}`);
-        if (portSpec.readOnly && portSpec.defaultValue !== hostPort) return new BoxError(BoxError.BAD_FIELD, `portBinding ${portName} is readOnly and cannot have a different value that the default`);
+    // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
+    // that the user wants the service disabled
+    const tcpPorts = manifest.tcpPorts || { };
+    const udpPorts = manifest.udpPorts || { };
+    for (let portName in portBindings) {
+        if (!(portName in tcpPorts) && !(portName in udpPorts)) return new BoxError(BoxError.BAD_FIELD, `Invalid portBindings ${portName}`);
     }
 
     return null;
@@ -468,23 +454,6 @@ function validateBackupFormat(format) {
     return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format');
 }
 
-function validateUpstreamUri(upstreamUri) {
-    assert.strictEqual(typeof upstreamUri, 'string');
-
-    if (!upstreamUri) return new BoxError(BoxError.BAD_FIELD, 'upstreamUri cannot be empty');
-
-    const uri = safe(() => new URL(upstreamUri));
-    if (!uri) return new BoxError(BoxError.BAD_FIELD, `upstreamUri is invalid: ${safe.error.message}`);
-    if (uri.protocol !== 'http:' && uri.protocol !== 'https:') return new BoxError(BoxError.BAD_FIELD, 'upstreamUri has an unsupported scheme');
-    if (uri.search || uri.hash) return new BoxError(BoxError.BAD_FIELD, 'upstreamUri cannot have search or hash');
-    if (uri.pathname !== '/') return new BoxError(BoxError.BAD_FIELD, 'upstreamUri cannot have a path');
-
-    // we use the uri in a named location @wellknown-upstream. nginx does not support having paths in it
-    if (upstreamUri.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, 'upstreamUri cannot have a path');
-
-    return null;
-}
-
 function validateLabel(label) {
     if (label === null) return null;
 
@@ -539,9 +508,10 @@ async function checkStorage(app, volumeId, prefix) {
     return null;
 }
 
-function getDuplicateErrorDetails(errorMessage, locations, portBindings) {
+function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, portBindings) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert(Array.isArray(locations));
+    assert.strictEqual(typeof domainObjectMap, 'object');
     assert.strictEqual(typeof portBindings, 'object');
 
     const match = errorMessage.match(/ER_DUP_ENTRY: Duplicate entry '(.*)' for key '(.*)'/);
@@ -556,7 +526,7 @@ function getDuplicateErrorDetails(errorMessage, locations, portBindings) {
             const { subdomain, domain, type } = locations[i];
             if  (match[1] !== `${subdomain}-${domain}`) continue;
 
-            return new BoxError(BoxError.ALREADY_EXISTS, `${type} location '${dns.fqdn(subdomain, domain)}' is in use`);
+            return new BoxError(BoxError.ALREADY_EXISTS, `${type} location '${dns.fqdn(subdomain, domainObjectMap[domain])}' is in use`);
         }
     }
 
@@ -581,36 +551,21 @@ async function getStorageDir(app) {
     return path.join(volume.hostPath, app.storageVolumePrefix);
 }
 
-function removeCertificateKeys(app) {
-    if (app.certificate) delete app.certificate.key;
-    app.secondaryDomains.forEach(sd => { if (sd.certificate) delete sd.certificate.key; });
-    app.aliasDomains.forEach(ad => { if (ad.certificate) delete ad.certificate.key; });
-    app.redirectDomains.forEach(rd => { if (rd.certificate) delete rd.certificate.key; });
-}
-
 function removeInternalFields(app) {
-    const result = _.pick(app,
+    return _.pick(app,
         'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId',
-        'subdomain', 'domain', 'fqdn', 'certificate', 'crontab', 'upstreamUri',
+        'subdomain', 'domain', 'fqdn', 'crontab',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'operators',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate',
-        'storageVolumeId', 'storageVolumePrefix', 'mounts', 'repository',
+        'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'storageVolumeId', 'storageVolumePrefix', 'mounts',
         'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain');
-
-    removeCertificateKeys(result);
-    return result;
 }
 
 // non-admins can only see these
 function removeRestrictedFields(app) {
-    const result = _.pick(app,
-        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'accessRestriction', 'repository',
-        'secondaryDomains', 'redirectDomains', 'aliasDomains', 'sso', 'subdomain', 'domain', 'fqdn', 'certificate',
-        'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup', 'upstreamUri');
-
-    removeCertificateKeys(result);
-    return result;
+    return _.pick(app,
+        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'accessRestriction', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'sso',
+        'subdomain', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup');
 }
 
 async function getIcon(app, options) {
@@ -708,35 +663,30 @@ function postProcess(result) {
     const subdomains = JSON.parse(result.subdomains),
         domains = JSON.parse(result.domains),
         subdomainTypes = JSON.parse(result.subdomainTypes),
-        subdomainEnvironmentVariables = JSON.parse(result.subdomainEnvironmentVariables),
-        subdomainCertificateJsons = JSON.parse(result.subdomainCertificateJsons);
+        subdomainEnvironmentVariables = JSON.parse(result.subdomainEnvironmentVariables);
 
     delete result.subdomains;
     delete result.domains;
     delete result.subdomainTypes;
     delete result.subdomainEnvironmentVariables;
-    delete result.subdomainCertificateJsons;
 
     result.secondaryDomains = [];
     result.redirectDomains = [];
     result.aliasDomains = [];
     for (let i = 0; i < subdomainTypes.length; i++) {
-        const subdomain = subdomains[i], domain = domains[i], certificate = safe.JSON.parse(subdomainCertificateJsons[i]);
-
         if (subdomainTypes[i] === exports.LOCATION_TYPE_PRIMARY) {
-            result.subdomain = subdomain;
-            result.domain = domain;
-            result.certificate = certificate;
+            result.subdomain = subdomains[i];
+            result.domain = domains[i];
         } else if (subdomainTypes[i] === exports.LOCATION_TYPE_SECONDARY) {
-            result.secondaryDomains.push({ domain, subdomain, certificate, environmentVariable: subdomainEnvironmentVariables[i] });
+            result.secondaryDomains.push({ domain: domains[i], subdomain: subdomains[i], environmentVariable: subdomainEnvironmentVariables[i] });
         } else if (subdomainTypes[i] === exports.LOCATION_TYPE_REDIRECT) {
-            result.redirectDomains.push({ domain, subdomain, certificate });
+            result.redirectDomains.push({ domain: domains[i], subdomain: subdomains[i] });
         } else if (subdomainTypes[i] === exports.LOCATION_TYPE_ALIAS) {
-            result.aliasDomains.push({ domain, subdomain, certificate });
+            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
         }
     }
 
-    const envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
+    let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
     delete result.envNames;
     delete result.envValues;
     result.env = {};
@@ -744,7 +694,7 @@ function postProcess(result) {
         if (envNames[i]) result.env[envNames[i]] = envValues[i];
     }
 
-    const volumeIds = JSON.parse(result.volumeIds);
+    let volumeIds = JSON.parse(result.volumeIds);
     delete result.volumeIds;
     let volumeReadOnlys = JSON.parse(result.volumeReadOnlys);
     delete result.volumeReadOnlys;
@@ -755,11 +705,6 @@ function postProcess(result) {
     delete result.errorJson;
 
     result.taskId = result.taskId ? String(result.taskId) : null;
-
-    // package repository is currently determined by dockerImage
-    if (!result.manifest.dockerImage) result.repository = '';
-    else if (result.manifest.dockerImage.startsWith('cloudron/')) result.repository = exports.REPOSITORY_CORE;
-    else result.repository = exports.REPOSITORY_COMMUNITY;
 }
 
 function attachProperties(app, domainObjectMap) {
@@ -772,10 +717,10 @@ function attachProperties(app, domainObjectMap) {
     }
     app.portBindings = result;
     app.iconUrl = app.hasIcon || app.hasAppStoreIcon  ? `/api/v1/apps/${app.id}/icon` : null;
-    app.fqdn = dns.fqdn(app.subdomain, app.domain);
-    app.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
-    app.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
-    app.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
+    app.fqdn = dns.fqdn(app.subdomain, domainObjectMap[app.domain]);
+    app.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    app.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    app.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 }
 
 function isAdmin(user) {
@@ -785,7 +730,7 @@ function isAdmin(user) {
 }
 
 function isOperator(app, user) {
-    assert.strictEqual(typeof app, 'object');  // IMPORTANT: can also be applink
+    assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof user, 'object');
 
     if (!app.operators) return isAdmin(user);
@@ -798,7 +743,7 @@ function isOperator(app, user) {
 }
 
 function canAccess(app, user) {
-    assert.strictEqual(typeof app, 'object'); // IMPORTANT: can also be applink
+    assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof user, 'object');
 
     if (app.accessRestriction === null) return true;
@@ -846,7 +791,6 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
         reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null,
         servicesConfigJson = data.servicesConfig ? JSON.stringify(data.servicesConfig) : null,
         enableMailbox = data.enableMailbox || false,
-        upstreamUri = data.upstreamUri || '',
         icon = data.icon || null;
 
     const queries = [];
@@ -854,11 +798,11 @@ async function add(id, appStoreId, manifest, subdomain, domain, portBindings, da
     queries.push({
         query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
             + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, '
-            + 'enableMailbox, mailboxDisplayName, upstreamUri) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+            + 'enableMailbox, mailboxDisplayName) '
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
         args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
             sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon,
-            enableMailbox, mailboxDisplayName, upstreamUri ]
+            enableMailbox, mailboxDisplayName ]
     });
 
     queries.push({
@@ -1071,12 +1015,19 @@ async function clear() {
     await database.query('DELETE FROM apps');
 }
 
+async function getDomainObjectMap() {
+    const domainObjects = await domains.list();
+    let domainObjectMap = {};
+    for (let d of domainObjects) { domainObjectMap[d.domain] = d; }
+    return domainObjectMap;
+}
+
 // each query simply join apps table with another table by id. we then join the full result together
 const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
 const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
-const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(locations.subdomain) AS subdomains, JSON_ARRAYAGG(locations.domain) AS domains, JSON_ARRAYAGG(locations.type) AS subdomainTypes, JSON_ARRAYAGG(locations.environmentVariable) AS subdomainEnvironmentVariables, JSON_ARRAYAGG(locations.certificateJson) AS subdomainCertificateJsons FROM apps LEFT JOIN locations ON apps.id = locations.appId GROUP BY apps.id';
+const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(locations.subdomain) AS subdomains, JSON_ARRAYAGG(locations.domain) AS domains, JSON_ARRAYAGG(locations.type) AS subdomainTypes, JSON_ARRAYAGG(locations.environmentVariable) AS subdomainEnvironmentVariables FROM apps LEFT JOIN locations ON apps.id = locations.appId GROUP BY apps.id';
 const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
-const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, subdomainEnvironmentVariables, subdomainCertificateJsons, volumeIds, volumeReadOnlys FROM apps`
+const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, subdomainEnvironmentVariables, volumeIds, volumeReadOnlys FROM apps`
     + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
     + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
     + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
@@ -1085,7 +1036,7 @@ const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariab
 async function get(id) {
     assert.strictEqual(typeof id, 'string');
 
-    const domainObjectMap = await domains.getDomainObjectMap();
+    const domainObjectMap = await getDomainObjectMap();
 
     const result = await database.query(`${APPS_QUERY} WHERE apps.id = ?`, [ id ]);
     if (result.length === 0) return null;
@@ -1100,7 +1051,7 @@ async function get(id) {
 async function getByIpAddress(ip) {
     assert.strictEqual(typeof ip, 'string');
 
-    const domainObjectMap = await domains.getDomainObjectMap();
+    const domainObjectMap = await getDomainObjectMap();
 
     const result = await database.query(`${APPS_QUERY} WHERE apps.containerIp = ?`, [ ip ]);
     if (result.length === 0) return null;
@@ -1111,7 +1062,7 @@ async function getByIpAddress(ip) {
 }
 
 async function list() {
-    const domainObjectMap = await domains.getDomainObjectMap();
+    const domainObjectMap = await getDomainObjectMap();
 
     const results = await database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ]);
     results.forEach(postProcess);
@@ -1287,10 +1238,10 @@ function checkAppState(app, state) {
 async function validateLocations(locations) {
     assert(Array.isArray(locations));
 
-    const domainObjectMap = await domains.getDomainObjectMap();
+    const domainObjectMap = await getDomainObjectMap();
 
-    for (const location of locations) {
-        if (!(location.domain in domainObjectMap)) return new BoxError(BoxError.BAD_FIELD, `No such domain in ${location.type} location`);
+    for (let location of locations) {
+        if (!(location.domain in domainObjectMap)) throw new BoxError(BoxError.BAD_FIELD, `No such domain in ${location.type} location`);
 
         let subdomain = location.subdomain;
         if (location.type === exports.LOCATION_TYPE_ALIAS && subdomain.startsWith('*')) {
@@ -1298,11 +1249,11 @@ async function validateLocations(locations) {
             subdomain = subdomain.replace(/^\*\./, ''); // remove *.
         }
 
-        const error = dns.validateHostname(subdomain, location.domain);
-        if (error) return new BoxError(BoxError.BAD_FIELD, `Bad ${location.type} location: ${error.message}`);
+        const error = dns.validateHostname(subdomain, domainObjectMap[location.domain]);
+        if (error) throw new BoxError(BoxError.BAD_FIELD, `Bad ${location.type} location: ${error.message}`);
     }
 
-    return null;
+    return domainObjectMap;
 }
 
 async function getCount() {
@@ -1332,7 +1283,6 @@ async function install(data, auditSource) {
         overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false,
         skipDnsSetup = 'skipDnsSetup' in data ? data.skipDnsSetup : false,
         appStoreId = data.appStoreId,
-        upstreamUri = data.upstreamUri || '',
         manifest = data.manifest;
 
     let error = manifestFormat.parse(manifest);
@@ -1356,9 +1306,6 @@ async function install(data, auditSource) {
     error = validateLabel(label);
     if (error) throw error;
 
-    if ('upstreamUri' in data) error = validateUpstreamUri(upstreamUri);
-    if (error) throw error;
-
     error = validateTags(tags);
     if (error) throw error;
 
@@ -1387,13 +1334,12 @@ async function install(data, auditSource) {
         icon = Buffer.from(icon, 'base64');
     }
 
-    const locations = [{ subdomain, domain, type: exports.LOCATION_TYPE_PRIMARY }]
+    const locations = [{ subdomain: subdomain, domain, type: exports.LOCATION_TYPE_PRIMARY }]
         .concat(secondaryDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_SECONDARY })))
         .concat(redirectDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_REDIRECT })))
         .concat(aliasDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_ALIAS })));
 
-    error = await validateLocations(locations);
-    if (error) throw error;
+    const domainObjectMap = await validateLocations(locations);
 
     if (settings.isDemo() && (await getCount() >= constants.DEMO_APP_LIMIT)) throw new BoxError(BoxError.BAD_STATE, 'Too many installed apps, please uninstall a few and try again');
 
@@ -1417,13 +1363,12 @@ async function install(data, auditSource) {
         tags,
         icon,
         enableMailbox,
-        upstreamUri,
         runState: exports.RSTATE_RUNNING,
         installationState: exports.ISTATE_PENDING_INSTALL
     };
 
     const [addError] = await safe(add(appId, appStoreId, manifest, subdomain, domain, translatePortBindings(portBindings, manifest), app));
-    if (addError && addError.reason === BoxError.ALREADY_EXISTS) throw getDuplicateErrorDetails(addError.message, locations, portBindings);
+    if (addError && addError.reason === BoxError.ALREADY_EXISTS) throw getDuplicateErrorDetails(addError.message, locations, domainObjectMap, portBindings);
     if (addError) throw addError;
 
     await purchaseApp({ appId, appstoreId: appStoreId, manifestId: manifest.id || 'customapp' });
@@ -1437,10 +1382,10 @@ async function install(data, auditSource) {
     const taskId = await addTask(appId, app.installationState, task, auditSource);
 
     const newApp = _.extend({}, _.omit(app, 'icon'), { appStoreId, manifest, subdomain, domain, portBindings });
-    newApp.fqdn = dns.fqdn(newApp.subdomain, newApp.domain);
-    newApp.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
-    newApp.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
-    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
+    newApp.fqdn = dns.fqdn(newApp.subdomain, domainObjectMap[newApp.domain]);
+    newApp.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    newApp.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
     await eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId, app: newApp, taskId });
 
@@ -1485,22 +1430,6 @@ async function setCrontab(app, crontab, auditSource) {
     await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, crontab });
 }
 
-async function setUpstreamUri(app, upstreamUri, auditSource) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof upstreamUri, 'string');
-    assert.strictEqual(typeof auditSource, 'object');
-
-    const appId = app.id;
-    const error = validateUpstreamUri(upstreamUri);
-    if (error) throw error;
-
-    await reverseProxy.writeAppConfigs(_.extend({}, app, { upstreamUri }));
-
-    await update(appId, { upstreamUri });
-
-    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, upstreamUri });
-}
-
 async function setLabel(app, label, auditSource) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof label, 'string');
@@ -1774,7 +1703,7 @@ async function setReverseProxyConfig(app, reverseProxyConfig, auditSource) {
     error = validateRobotsTxt(reverseProxyConfig.robotsTxt);
     if (error) throw error;
 
-    await reverseProxy.writeAppConfigs(_.extend({}, app, { reverseProxyConfig }));
+    await reverseProxy.writeAppConfig(_.extend({}, app, { reverseProxyConfig }));
 
     await update(appId, { reverseProxyConfig });
 
@@ -1786,35 +1715,18 @@ async function setCertificate(app, data, auditSource) {
     assert(data && typeof data === 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
-    const { subdomain, domain, cert, key } = data;
+    const appId = app.id;
+    const { location, domain, cert, key } = data;
 
     const domainObject = await domains.get(domain);
-    if (domainObject === null) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
 
     if (cert && key) {
-        const error = reverseProxy.validateCertificate(subdomain, domain, { cert, key });
+        const error = reverseProxy.validateCertificate(location, domainObject, { cert, key });
         if (error) throw error;
     }
 
-    const certificate = cert && key ? { cert, key } : null;
-    const result = await database.query('UPDATE locations SET certificateJson=? WHERE location=? AND domain=?', [ certificate ? JSON.stringify(certificate) : null, subdomain, domain ]);
-    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Location not found');
-
-    const location = await getLocation(subdomain, domain); // fresh location object
-    await reverseProxy.setUserCertificate(app, location);
-    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: app.id, app, subdomain, domain, cert });
-}
-
-async function getLocation(subdomain, domain) {
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof domain, 'string');
-
-    const result = await database.query(`SELECT ${LOCATION_FIELDS} FROM locations WHERE subdomain=? AND domain=?`, [ subdomain, domain ]);
-    if (result.length === 0) return null;
-
-    result[0].certificate = safe.JSON.parse(result[0].certificateJson);
-    result[0].fqdn = dns.fqdn(subdomain, domain);
-    return result[0];
+    await reverseProxy.setAppCertificate(location, domainObject, { cert, key });
+    await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, cert, key });
 }
 
 async function setLocation(app, data, auditSource) {
@@ -1866,8 +1778,7 @@ async function setLocation(app, data, auditSource) {
         .concat(values.redirectDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_REDIRECT })))
         .concat(values.aliasDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_ALIAS })));
 
-    error = await validateLocations(locations);
-    if (error) throw error;
+    const domainObjectMap = await validateLocations(locations);
 
     const task = {
         args: {
@@ -1878,13 +1789,13 @@ async function setLocation(app, data, auditSource) {
         values
     };
     let [taskError, taskId] = await safe(addTask(appId, exports.ISTATE_PENDING_LOCATION_CHANGE, task, auditSource));
-    if (taskError && taskError.reason === BoxError.ALREADY_EXISTS) taskError = getDuplicateErrorDetails(taskError.message, locations, data.portBindings);
+    if (taskError && taskError.reason === BoxError.ALREADY_EXISTS) taskError = getDuplicateErrorDetails(taskError.message, locations, domainObjectMap, data.portBindings);
     if (taskError) throw taskError;
 
-    values.fqdn = dns.fqdn(values.subdomain, values.domain);
-    values.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
-    values.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
-    values.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
+    values.fqdn = dns.fqdn(values.subdomain, domainObjectMap[values.domain]);
+    values.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    values.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    values.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
     await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, _.extend({ appId, app, taskId }, values));
 
@@ -2046,22 +1957,38 @@ async function getLogs(app, options) {
     const logPaths = await getLogPaths(app);
     const cp = spawn('/usr/bin/tail', args.concat(logPaths));
 
-    const logStream = new LogStream({ format, source: appId });
-    logStream.close = cp.kill.bind(cp, 'SIGKILL'); // hook for caller. closing stream kills the child process
+    const transformStream = split(function mapper(line) {
+        if (format !== 'json') return line + '\n';
 
-    cp.stdout.pipe(logStream);
+        const data = line.split(' '); // logs are <ISOtimestamp> <msg>
+        let timestamp = (new Date(data[0])).getTime();
+        if (isNaN(timestamp)) timestamp = 0;
+        const message = line.slice(data[0].length+1);
 
-    return logStream;
+        // ignore faulty empty logs
+        if (!timestamp && !message) return;
+
+        return JSON.stringify({
+            realtimeTimestamp: timestamp * 1000,
+            message: message,
+            source: appId
+        }) + '\n';
+    });
+
+    transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process
+
+    cp.stdout.pipe(transformStream);
+
+    return transformStream;
 }
 
-// never fails just prints error
-async function appendLogLine(app, line) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof line, 'string');
+async function getCertificate(subdomain, domain) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domain, 'string');
 
-    const logFilePath = path.join(paths.LOG_DIR, app.id, 'app.log');
-
-    if (!safe.fs.appendFileSync(logFilePath, line)) console.error(`Could not append log line for app ${app.id} at ${logFilePath}: ${safe.error.message}`);
+    const result = await database.query('SELECT certificateJson FROM locations WHERE subdomain=? AND domain=?', [ subdomain, domain ]);
+    if (result.length === 0) return null;
+    return JSON.parse(result[0].certificateJson);
 }
 
 // does a re-configure when called from most states. for install/clone errors, it re-installs with an optional manifest
@@ -2169,18 +2096,20 @@ async function importApp(app, data, auditSource) {
 
     const appId = app.id;
 
+    // all fields are optional
+    data.remotePath = data.remotePath || null;
+    data.backupFormat = data.backupFormat || null;
+    data.backupConfig = data.backupConfig || null;
     const { remotePath, backupFormat, backupConfig } = data;
 
-    let error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
+    let error = backupFormat ? validateBackupFormat(backupFormat) : null;
     if (error) throw error;
 
-    let restoreConfig;
+    error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
+    if (error) throw error;
 
-    if (data.remotePath) { // if not provided, we import in-place
-        error = validateBackupFormat(backupFormat);
-        if (error) throw error;
-
-        // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
+    // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend
+    if (backupConfig) {
         if (mounts.isManagedProvider(backupConfig.provider)) {
             error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions);
             if (error) throw error;
@@ -2196,19 +2125,19 @@ async function importApp(app, data, auditSource) {
         }
         error = await backups.testProviderConfig(backupConfig);
         if (error) throw error;
+    }
 
+    if (backupConfig) {
         if ('password' in backupConfig) {
             backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.password);
             delete backupConfig.password;
         } else {
             backupConfig.encryption = null;
         }
-
-        restoreConfig = { remotePath, backupFormat, backupConfig };
-    } else {
-        restoreConfig = { remotePath: null };
     }
 
+    const restoreConfig = { remotePath, backupFormat, backupConfig };
+
     const task = {
         args: {
             restoreConfig,
@@ -2286,11 +2215,10 @@ async function clone(app, data, user, auditSource) {
     if (error) throw error;
     const secondaryDomains = translateSecondaryDomains(data.secondaryDomains || {});
 
-    const locations = [{ subdomain, domain, type: exports.LOCATION_TYPE_PRIMARY }]
+    const locations = [{ subdomain: subdomain, domain, type: exports.LOCATION_TYPE_PRIMARY }]
         .concat(secondaryDomains.map(ad => _.extend(ad, { type: exports.LOCATION_TYPE_SECONDARY })));
 
-    error = await validateLocations(locations);
-    if (error) throw error;
+    const domainObjectMap = await validateLocations(locations);
 
     // re-validate because this new box version may not accept old configs
     error = checkManifestConstraints(manifest);
@@ -2332,7 +2260,7 @@ async function clone(app, data, user, auditSource) {
     };
 
     const [addError] = await safe(add(newAppId, appStoreId, manifest, subdomain, domain, translatePortBindings(portBindings, manifest), obj));
-    if (addError && addError.reason === BoxError.ALREADY_EXISTS) throw getDuplicateErrorDetails(addError.message, locations, portBindings);
+    if (addError && addError.reason === BoxError.ALREADY_EXISTS) throw getDuplicateErrorDetails(addError.message, locations, domainObjectMap, portBindings);
     if (addError) throw addError;
 
     await purchaseApp({ appId: newAppId, appstoreId: app.appStoreId, manifestId: manifest.id || 'customapp' });
@@ -2346,10 +2274,10 @@ async function clone(app, data, user, auditSource) {
     const taskId = await addTask(newAppId, exports.ISTATE_PENDING_CLONE, task, auditSource);
 
     const newApp = _.extend({}, _.omit(obj, 'icon'), { appStoreId, manifest, subdomain, domain, portBindings });
-    newApp.fqdn = dns.fqdn(newApp.subdomain, newApp.domain);
-    newApp.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
-    newApp.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
-    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, ad.domain); });
+    newApp.fqdn = dns.fqdn(newApp.subdomain, domainObjectMap[newApp.domain]);
+    newApp.secondaryDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    newApp.redirectDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = dns.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
     await eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId, remotePath: backupInfo.remotePath, oldApp: app, newApp, taskId });
 
@@ -2473,9 +2401,6 @@ async function createExec(app, options) {
         Cmd: cmd
     };
 
-    // currently the webterminal and cli sets C.UTF-8
-    if (options.lang) createOptions.Env = [ 'LANG=' + options.lang ];
-
     return await docker.createExec(app.containerId, createOptions);
 }
 
@@ -2565,7 +2490,6 @@ async function autoupdateApps(updateInfo, auditSource) { // updateInfo is { appI
 
         if (!canAutoupdateApp(app, updateInfo[appId])) {
             debug(`app ${app.fqdn} requires manual update`);
-            notifications.alert(notifications.ALERT_MANUAL_APP_UPDATE, `${app.manifest.title} at ${app.fqdn} requires manual update to version ${updateInfo[appId].manifest.version}`, `Changelog:\n${updateInfo[appId].manifest.changelog}\n`);
             continue;
         }
 
@@ -2618,25 +2542,6 @@ async function updateBackup(app, backupId, data) {
     await backups.update(backupId, data);
 }
 
-async function getBackupDownloadStream(app, backupId) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof backupId, 'string');
-
-    const backup = await backups.get(backupId);
-    if (!backup) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
-    if (backup.identifier !== app.id) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); // some other app's backup
-    if (backup.format !== 'tgz') throw new BoxError(BoxError.BAD_STATE, 'only tgz backups can be downloaded');
-
-    const backupConfig = await settings.getBackupConfig();
-
-    return new Promise((resolve, reject) => {
-        storage.api(backupConfig.provider).download(backupConfig, tgz.getBackupFilePath(backupConfig, backup.remotePath), function (error, sourceStream) {
-            if (error) return reject(error);
-            resolve(sourceStream);
-        });
-    });
-}
-
 async function restoreInstalledApps(options, auditSource) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');

src/appstore.js

@@ -394,14 +394,14 @@ async function createTicket(info, auditSource) {
     return { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` };
 }
 
-async function getApps(repository = 'core') {
+async function getApps() {
     const token = await settings.getAppstoreApiToken();
     if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
     const unstable = await settings.getUnstableAppsConfig();
 
     const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/apps`)
-        .query({ accessToken: token, boxVersion: constants.VERSION, unstable, repository })
+        .query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable })
         .timeout(30 * 1000)
         .ok(() => true));
 

src/apptask.js

@@ -17,9 +17,10 @@ const apps = require('./apps.js'),
     AuditSource = require('./auditsource.js'),
     backuptask = require('./backuptask.js'),
     BoxError = require('./boxerror.js'),
+    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:apptask'),
-    df = require('./df.js'),
+    df = require('@sindresorhus/df'),
     dns = require('./dns.js'),
     docker = require('./docker.js'),
     ejs = require('ejs'),
@@ -44,8 +45,12 @@ const MV_VOLUME_CMD = path.join(__dirname, 'scripts/mvvolume.sh'),
     LOGROTATE_CONFIG_EJS = fs.readFileSync(__dirname + '/logrotate.ejs', { encoding: 'utf8' }),
     CONFIGURE_LOGROTATE_CMD = path.join(__dirname, 'scripts/configurelogrotate.sh');
 
+// https://rootlesscontaine.rs/getting-started/common/cgroup2/#checking-whether-cgroup-v2-is-already-enabled
+const CGROUP_VERSION = fs.existsSync('/sys/fs/cgroup/cgroup.controllers') ? '2' : '1';
+const COLLECTD_CONFIG_EJS = fs.readFileSync(`${__dirname}/collectd/app_cgroup_v${CGROUP_VERSION}.ejs`, { encoding: 'utf8' });
+
 function makeTaskError(error, app) {
-    assert(error instanceof BoxError);
+    assert.strictEqual(typeof error, 'object');
     assert.strictEqual(typeof app, 'object');
 
     // track a few variables which helps 'repair' restart the task (see also scheduleTask in apps.js)
@@ -69,8 +74,6 @@ async function updateApp(app, values) {
 async function allocateContainerIp(app) {
     assert.strictEqual(typeof app, 'object');
 
-    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;
-
     await promiseRetry({ times: 10, interval: 0, debug }, async function () {
         const iprange = iputils.intFromIp('172.18.20.255') - iputils.intFromIp('172.18.16.1');
         let rnd = Math.floor(Math.random() * iprange);
@@ -83,8 +86,6 @@ async function createContainer(app) {
     assert.strictEqual(typeof app, 'object');
     assert(!app.containerId); // otherwise, it will trigger volumeFrom
 
-    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;
-
     debug('createContainer: creating container');
 
     const container = await docker.createContainer(app);
@@ -93,6 +94,7 @@ async function createContainer(app) {
 
     // re-generate configs that rely on container id
     await addLogrotateConfig(app);
+    await addCollectdProfile(app);
 }
 
 async function deleteContainers(app, options) {
@@ -102,6 +104,7 @@ async function deleteContainers(app, options) {
     debug('deleteContainer: deleting app containers (app, scheduler)');
 
     // remove configs that rely on container id
+    await removeCollectdProfile(app);
     await removeLogrotateConfig(app);
     await docker.stopContainers(app.id);
     await docker.deleteContainers(app.id, options);
@@ -154,6 +157,20 @@ async function deleteAppDir(app, options) {
     }
 }
 
+async function addCollectdProfile(app) {
+    assert.strictEqual(typeof app, 'object');
+
+    const appDataDir = await apps.getStorageDir(app);
+    const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir });
+    await collectd.addProfile(app.id, collectdConf);
+}
+
+async function removeCollectdProfile(app) {
+    assert.strictEqual(typeof app, 'object');
+
+    await collectd.removeProfile(app.id);
+}
+
 async function addLogrotateConfig(app) {
     assert.strictEqual(typeof app, 'object');
 
@@ -273,9 +290,6 @@ async function moveDataDir(app, targetVolumeId, targetVolumePrefix) {
 async function downloadImage(manifest) {
     assert.strictEqual(typeof manifest, 'object');
 
-    // skip for relay app
-    if (manifest.id === constants.PROXY_APP_APPSTORE_ID) return;
-
     const info = await docker.info();
     const [dfError, diskUsage] = await safe(df.file(info.DockerRootDir));
     if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error getting file system info: ${dfError.message}`);
@@ -290,9 +304,6 @@ async function startApp(app) {
 
     if (app.runState === apps.RSTATE_STOPPED) return;
 
-    // skip for relay app
-    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) return;
-
     await docker.startContainer(app.id);
 }
 
@@ -664,10 +675,11 @@ async function start(app, args, progressCallback) {
     await progressCallback({ percent: 10, message: 'Starting app services' });
     await services.startAppServices(app);
 
-    if (app.manifest.id !== constants.PROXY_APP_APPSTORE_ID) {
-        await progressCallback({ percent: 35, message: 'Starting container' });
-        await docker.startContainer(app.id);
-    }
+    await progressCallback({ percent: 35, message: 'Starting container' });
+    await docker.startContainer(app.id);
+
+    await progressCallback({ percent: 60, message: 'Adding collectd profile' });
+    await addCollectdProfile(app);
 
     // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
     await progressCallback({ percent: 80, message: 'Configuring reverse proxy' });
@@ -689,6 +701,9 @@ async function stop(app, args, progressCallback) {
     await progressCallback({ percent: 50, message: 'Stopping app services' });
     await services.stopAppServices(app);
 
+    await progressCallback({ percent: 80, message: 'Removing collectd profile' });
+    await removeCollectdProfile(app);
+
     await progressCallback({ percent: 100, message: 'Done' });
     await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });
 }
@@ -698,17 +713,8 @@ async function restart(app, args, progressCallback) {
     assert.strictEqual(typeof args, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    if (app.manifest.id !== constants.PROXY_APP_APPSTORE_ID) {
-        await progressCallback({ percent: 10, message: 'Starting app services' });
-        await services.startAppServices(app);
-
-        await progressCallback({ percent: 20, message: 'Restarting container' });
-        await docker.restartContainer(app.id);
-    }
-
-    // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
-    await progressCallback({ percent: 80, message: 'Configuring reverse proxy' });
-    await reverseProxy.configureApp(app, AuditSource.APPTASK);
+    await progressCallback({ percent: 20, message: 'Restarting container' });
+    await docker.restartContainer(app.id);
 
     await progressCallback({ percent: 100, message: 'Done' });
     await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null });

src/backupcleaner.js

@@ -15,6 +15,7 @@ const apps = require('./apps.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:backupcleaner'),
     moment = require('moment'),
+    mounts = require('./mounts.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -271,9 +272,12 @@ async function run(progressCallback) {
 
     const backupConfig = await settings.getBackupConfig();
 
-    const status = await storage.api(backupConfig.provider).getBackupProviderStatus(backupConfig);
-    debug(`clean: mount point status is ${JSON.stringify(status)}`);
-    if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
+    if (mounts.isManagedProvider(backupConfig.provider) || backupConfig.provider === 'mountpoint') {
+        const hostPath = mounts.isManagedProvider(backupConfig.provider) ? paths.MANAGED_BACKUP_MOUNT_DIR : backupConfig.mountPoint;
+        const status = await mounts.getStatus(backupConfig.provider, hostPath); // { state, message }
+        debug(`clean: mount point status is ${JSON.stringify(status)}`);
+        if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
+    }
 
     if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
         debug('cleanup: keeping all backups');
@@ -289,7 +293,7 @@ async function run(progressCallback) {
     await progressCallback({ percent: 40, message: 'Cleaning app backups' });
     const removedAppBackupPaths = await cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback);
 
-    await progressCallback({ percent: 70, message: 'Checking storage backend and removing stale entries in database' });
+    await progressCallback({ percent: 70, message: 'Cleaning missing backups' });
     const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback);
 
     await progressCallback({ percent: 90, message: 'Cleaning snapshots' });

src/backupformat/rsync.js

@@ -27,7 +27,7 @@ function getBackupFilePath(backupConfig, remotePath) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof remotePath, 'string');
 
-    const rootPath = storage.api(backupConfig.provider).getBackupRootPath(backupConfig);
+    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
     return path.join(rootPath, remotePath);
 }
 
@@ -46,7 +46,7 @@ function sync(backupConfig, remotePath, dataLayout, progressCallback, callback)
     syncer.sync(dataLayout, function processTask(task, iteratorCallback) {
         debug('sync: processing task: %j', task);
         // the empty task.path is special to signify the directory
-        const destPath = task.path && backupConfig.encryptedFilenames ? hush.encryptFilePath(task.path, backupConfig.encryption) : task.path;
+        const destPath = task.path && backupConfig.encryption ? hush.encryptFilePath(task.path, backupConfig.encryption) : task.path;
         const backupFilePath = path.join(getBackupFilePath(backupConfig, remotePath), destPath);
 
         if (task.operation === 'removedir') {
@@ -164,7 +164,7 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
 
     function downloadFile(entry, done) {
         let relativePath = path.relative(backupFilePath, entry.fullPath);
-        if (backupConfig.encryptedFilenames) {
+        if (backupConfig.encryption) {
             const { error, result } = hush.decryptFilePath(relativePath, backupConfig.encryption);
             if (error) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file'));
             relativePath = result;

src/backupformat/tgz.js

@@ -14,7 +14,7 @@ const assert = require('assert'),
     { DecryptStream, EncryptStream } = require('../hush.js'),
     once = require('../once.js'),
     path = require('path'),
-    ProgressStream = require('../progress-stream.js'),
+    progressStream = require('progress-stream'),
     storage = require('../storage.js'),
     tar = require('tar-fs'),
     zlib = require('zlib');
@@ -23,7 +23,7 @@ function getBackupFilePath(backupConfig, remotePath) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof remotePath, 'string');
 
-    const rootPath = storage.api(backupConfig.provider).getBackupRootPath(backupConfig);
+    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
 
     const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz';
     return path.join(rootPath, remotePath + fileType);
@@ -51,7 +51,7 @@ function tarPack(dataLayout, encryption) {
     });
 
     const gzip = zlib.createGzip({});
-    const ps = new ProgressStream({ interval: 10000 }); // emit 'progress' every 10 seconds
+    const ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds
 
     pack.on('error', function (error) {
         debug('tarPack: tar stream error.', error);
@@ -84,7 +84,7 @@ function tarExtract(inStream, dataLayout, encryption) {
     assert.strictEqual(typeof encryption, 'object');
 
     const gunzip = zlib.createGunzip({});
-    const ps = new ProgressStream({ interval: 10000 }); // display a progress every 10 seconds
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
     const extract = tar.extract('/', {
         map: function (header) {
             header.name = dataLayout.toLocalPath(header.name);
@@ -173,8 +173,6 @@ async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
     assert.strictEqual(typeof dataLayout, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    debug(`upload: Uploading ${dataLayout.toString()} to ${remotePath}`);
-
     return new Promise((resolve, reject) => {
         async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
             retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error

src/backups.js

@@ -18,6 +18,8 @@ exports = module.exports = {
     injectPrivateFields,
     removePrivateFields,
 
+    configureCollectd,
+
     generateEncryptionKeysSync,
 
     getSnapshotInfo,
@@ -42,12 +44,15 @@ exports = module.exports = {
 
 const assert = require('assert'),
     BoxError = require('./boxerror.js'),
+    collectd = require('./collectd.js'),
     constants = require('./constants.js'),
     CronJob = require('cron').CronJob,
     crypto = require('crypto'),
     database = require('./database.js'),
     debug = require('debug')('box:backups'),
+    ejs = require('ejs'),
     eventlog = require('./eventlog.js'),
+    fs = require('fs'),
     hat = require('./hat.js'),
     locker = require('./locker.js'),
     path = require('path'),
@@ -57,6 +62,8 @@ const assert = require('assert'),
     storage = require('./storage.js'),
     tasks = require('./tasks.js');
 
+const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/cloudron-backup.ejs', { encoding: 'utf8' });
+
 const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOnJson', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
 
 function postProcess(result) {
@@ -120,8 +127,7 @@ async function add(data) {
 
     const creationTime = data.creationTime || new Date(); // allow tests to set the time
     const manifestJson = JSON.stringify(data.manifest);
-    const prefixId = data.type === exports.BACKUP_TYPE_APP ? `${data.type}_${data.identifier}` : data.type; // type and identifier are same for other types
-    const id = `${prefixId}_v${data.packageVersion}_${hat(256)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying
+    const id = `${data.type}_${data.identifier}_v${data.packageVersion}_${hat(256)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying
 
     const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, format, preserveSecs) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
         [ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.format, data.preserveSecs ]));
@@ -170,7 +176,7 @@ function validateLabel(label) {
     assert.strictEqual(typeof label, 'string');
 
     if (label.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'label too long');
-    if (/[^a-zA-Z0-9._() -]/.test(label)) return new BoxError(BoxError.BAD_FIELD, 'label can only contain alphanumerals, space, dot, hyphen, brackets or underscore');
+    if (/[^a-zA-Z0-9._()-]/.test(label)) return new BoxError(BoxError.BAD_FIELD, 'label can only contain alphanumerals, dot, hyphen, brackets or underscore');
 
     return null;
 }
@@ -235,7 +241,7 @@ async function startBackupTask(auditSource) {
         const errorMessage = error ? error.message : '';
         const timedOut = error ? error.code === tasks.ETIMEOUT : false;
 
-        const backup = backupId ? await get(backupId) : null;
+        const backup = await get(backupId);
         await safe(eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { taskId, errorMessage, timedOut, backupId, remotePath: backup?.remotePath }), { debug });
     });
 
@@ -313,6 +319,17 @@ async function startCleanupTask(auditSource) {
     return taskId;
 }
 
+async function configureCollectd(backupConfig) {
+    assert.strictEqual(typeof backupConfig, 'object');
+
+    if (backupConfig.provider === 'filesystem') {
+        const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { backupDir: backupConfig.backupFolder });
+        await collectd.addProfile('cloudron-backup', collectdConf);
+    } else {
+        await collectd.removeProfile('cloudron-backup');
+    }
+}
+
 async function testConfig(backupConfig) {
     assert.strictEqual(typeof backupConfig, 'object');
 

src/backuptask.js

@@ -47,41 +47,6 @@ function canBackupApp(app) {
             app.installationState === apps.ISTATE_PENDING_UPDATE; // called from apptask
 }
 
-// binary units (non SI) 1024 based
-function prettyBytes(bytes) {
-    assert.strictEqual(typeof bytes, 'number');
-
-    const i = Math.floor(Math.log(bytes) / Math.log(1024)),
-        sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
-
-    return (bytes / Math.pow(1024, i)).toFixed(2) * 1 + '' + sizes[i];
-}
-
-async function checkPreconditions(backupConfig, dataLayout) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-
-    // check mount status before uploading
-    const status = await storage.api(backupConfig.provider).getBackupProviderStatus(backupConfig);
-    debug(`upload: mount point status is ${JSON.stringify(status)}`);
-    if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not active: ${status.message}`);
-
-    // check availabe size. this requires root for df to work
-    const df = await storage.api(backupConfig.provider).getAvailableSize(backupConfig);
-    let used = 0;
-    for (const localPath of dataLayout.localPaths()) {
-        debug(`checkPreconditions: getting disk usage of ${localPath}`);
-        const result = safe.child_process.execSync(`du -Dsb ${localPath}`, { encoding: 'utf8' });
-        if (!result) throw new BoxError(BoxError.FS_ERROR, `du error: ${safe.error.message}`);
-        used += parseInt(result, 10);
-    }
-
-    debug(`checkPreconditions: total required =${used} available=${df.available}`);
-
-    const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
-    if (df.available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${prettyBytes(needed)} Available: ${prettyBytes(df.available)}`);
-}
-
 // this function is called via backupupload (since it needs root to traverse app's directory)
 async function upload(remotePath, format, dataLayoutString, progressCallback) {
     assert.strictEqual(typeof remotePath, 'string');
@@ -93,8 +58,7 @@ async function upload(remotePath, format, dataLayoutString, progressCallback) {
 
     const dataLayout = DataLayout.fromString(dataLayoutString);
     const backupConfig = await settings.getBackupConfig();
-
-    await checkPreconditions(backupConfig, dataLayout);
+    await safe(storage.api(backupConfig.provider).checkPreconditions(backupConfig, dataLayout));
 
     await backupFormat.api(format).upload(backupConfig, remotePath, dataLayout, progressCallback);
 }

src/blobs.js

@@ -9,8 +9,6 @@ exports = module.exports = {
     setString,
     del,
 
-    listCertIds,
-
     ACME_ACCOUNT_KEY: 'acme_account_key',
     ADDON_TURN_SECRET: 'addon_turn_secret',
     SFTP_PUBLIC_KEY: 'sftp_public_key',
@@ -18,7 +16,6 @@ exports = module.exports = {
     PROXY_AUTH_TOKEN_SECRET: 'proxy_auth_token_secret',
 
     CERT_PREFIX: 'cert',
-    CERT_SUFFIX: 'cert',
 
     _clear: clear
 };
@@ -65,8 +62,3 @@ async function del(id) {
 async function clear() {
     await database.query('DELETE FROM blobs');
 }
-
-async function listCertIds() {
-    const result = await database.query('SELECT id FROM blobs WHERE id LIKE ?', [ `${exports.CERT_PREFIX}-%.${exports.CERT_SUFFIX}` ]);
-    return result.map(r => r.id);
-}

src/cloudron.js

@@ -19,8 +19,6 @@ exports = module.exports = {
     renewCerts,
     syncDnsRecords,
 
-    updateDiskUsage,
-
     runSystemChecks
 };
 
@@ -28,6 +26,7 @@ const apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
     AuditSource = require('./auditsource.js'),
+    backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     branding = require('./branding.js'),
     constants = require('./constants.js'),
@@ -36,9 +35,9 @@ const apps = require('./apps.js'),
     delay = require('./delay.js'),
     dns = require('./dns.js'),
     dockerProxy = require('./dockerproxy.js'),
+    domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
-    LogStream = require('./log-stream.js'),
     mail = require('./mail.js'),
     notifications = require('./notifications.js'),
     path = require('path'),
@@ -50,6 +49,7 @@ const apps = require('./apps.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     spawn = require('child_process').spawn,
+    split = require('split'),
     sysinfo = require('./sysinfo.js'),
     tasks = require('./tasks.js'),
     users = require('./users.js');
@@ -108,11 +108,18 @@ async function runStartupTasks() {
     // stop all the systemd tasks
     tasks.push(platform.stopAllTasks);
 
+    // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
+    tasks.push(async function () {
+        const backupConfig = await settings.getBackupConfig();
+        await backups.configureCollectd(backupConfig);
+    });
+
     // always generate webadmin config since we have no versioning mechanism for the ejs
     tasks.push(async function () {
         if (!settings.dashboardDomain()) return;
 
-        await reverseProxy.writeDashboardConfig(settings.dashboardDomain());
+        const domainObject = await domains.get(settings.dashboardDomain());
+        await reverseProxy.writeDashboardConfig(domainObject);
     });
 
     tasks.push(async function () {
@@ -133,7 +140,7 @@ async function runStartupTasks() {
     // we used to run tasks in parallel but simultaneous nginx reloads was causing issues
     for (let i = 0; i < tasks.length; i++) {
         const [error] = await safe(tasks[i]());
-        if (error) debug(`Startup task at index ${i} failed: ${error.message} ${error.stack}`);
+        if (error) debug(`Startup task at index ${i} failed: ${error.message}`);
     }
 }
 
@@ -228,12 +235,25 @@ async function getLogs(unit, options) {
 
     const cp = spawn('/usr/bin/tail', args);
 
-    const logStream = new LogStream({ format, source: unit });
-    logStream.close = cp.kill.bind(cp, 'SIGKILL'); // hook for caller. closing stream kills the child process
+    const transformStream = split(function mapper(line) {
+        if (format !== 'json') return line + '\n';
 
-    cp.stdout.pipe(logStream);
+        const data = line.split(' '); // logs are <ISOtimestamp> <msg>
+        let timestamp = (new Date(data[0])).getTime();
+        if (isNaN(timestamp)) timestamp = 0;
 
-    return logStream;
+        return JSON.stringify({
+            realtimeTimestamp: timestamp * 1000,
+            message: line.slice(data[0].length+1),
+            source: unit
+        }) + '\n';
+    });
+
+    transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process
+
+    cp.stdout.pipe(transformStream);
+
+    return transformStream;
 }
 
 async function prepareDashboardDomain(domain, auditSource) {
@@ -244,12 +264,15 @@ async function prepareDashboardDomain(domain, auditSource) {
 
     if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
 
-    const fqdn = dns.fqdn(constants.DASHBOARD_SUBDOMAIN, domain);
+    const domainObject = await domains.get(domain);
+    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');
+
+    const fqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
     const result = await apps.list();
     if (result.some(app => app.fqdn === fqdn)) throw new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app');
 
-    const taskId = await tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_SUBDOMAIN, domain, auditSource ]);
+    const taskId = await tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_LOCATION, domain, auditSource ]);
 
     tasks.startTask(taskId, {});
 
@@ -263,8 +286,11 @@ async function setDashboardDomain(domain, auditSource) {
 
     debug(`setDashboardDomain: ${domain}`);
 
-    await reverseProxy.writeDashboardConfig(domain);
-    const fqdn = dns.fqdn(constants.DASHBOARD_SUBDOMAIN, domain);
+    const domainObject = await domains.get(domain);
+    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');
+
+    await reverseProxy.writeDashboardConfig(domainObject);
+    const fqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
     await settings.setDashboardLocation(domain, fqdn);
 
@@ -302,7 +328,8 @@ async function setupDnsAndCert(subdomain, domain, auditSource, progressCallback)
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
 
-    const dashboardFqdn = dns.fqdn(subdomain, domain);
+    const domainObject = await domains.get(domain);
+    const dashboardFqdn = dns.fqdn(subdomain, domainObject);
 
     const ipv4 = await sysinfo.getServerIPv4();
     const ipv6 = await sysinfo.getServerIPv6();
@@ -314,8 +341,7 @@ async function setupDnsAndCert(subdomain, domain, auditSource, progressCallback)
     await dns.waitForDnsRecord(subdomain, domain, 'A', ipv4, { interval: 30000, times: 50000 });
     if (ipv6) await dns.waitForDnsRecord(subdomain, domain, 'AAAA', ipv6, { interval: 30000, times: 50000 });
     progressCallback({ percent: 60, message: `Getting certificate of ${dashboardFqdn}` });
-    const location = { subdomain, domain, fqdn: dashboardFqdn, type: apps.LOCATION_TYPE_DASHBOARD, certificate: null };
-    await reverseProxy.ensureCertificate(location, auditSource);
+    await reverseProxy.ensureCertificate(dns.fqdn(subdomain, domainObject), domain, auditSource);
 }
 
 async function syncDnsRecords(options) {
@@ -325,9 +351,3 @@ async function syncDnsRecords(options) {
     tasks.startTask(taskId, {});
     return taskId;
 }
-
-async function updateDiskUsage() {
-    const taskId = await tasks.add(tasks.TASK_UPDATE_DISK_USAGE, []);
-    tasks.startTask(taskId, {});
-    return taskId;
-}

src/collectd.js

@@ -0,0 +1,43 @@
+'use strict';
+
+exports = module.exports = {
+    addProfile,
+    removeProfile
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    debug = require('debug')('collectd'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js');
+
+const CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh');
+
+async function addProfile(name, profile) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof profile, 'string');
+
+    const configFilePath = path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`);
+
+    // skip restarting collectd if the profile already exists with the same contents
+    const currentProfile = safe.fs.readFileSync(configFilePath, 'utf8') || '';
+    if (currentProfile === profile) return;
+
+    if (!safe.fs.writeFileSync(configFilePath, profile)) throw new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${safe.error.message}`);
+
+    const [error] = await safe(shell.promises.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}));
+    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config');
+}
+
+async function removeProfile(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (!safe.fs.unlinkSync(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`))) {
+        if (safe.error.code !== 'ENOENT') debug('Error removing collectd profile', safe.error);
+    }
+
+    const [error] = await safe(shell.promises.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}));
+    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config');
+}

src/collectd/app_cgroup_v1.ejs

@@ -0,0 +1,42 @@
+LoadPlugin "table"
+<Plugin table>
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.stat">
+    Instance "<%= appId %>-memory"
+    Separator " \\n"
+    <Result>
+      Type gauge
+      InstancesFrom 0
+      ValuesFrom 1
+    </Result>
+  </Table>
+
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.max_usage_in_bytes">
+    Instance "<%= appId %>-memory"
+    Separator "\\n"
+    <Result>
+      Type gauge
+      InstancePrefix "max_usage_in_bytes"
+      ValuesFrom 0
+    </Result>
+  </Table>
+
+  <Table "/sys/fs/cgroup/cpuacct/docker/<%= containerId %>/cpuacct.stat">
+    Instance "<%= appId %>-cpu"
+    Separator " \\n"
+    <Result>
+      Type gauge
+      InstancesFrom 0
+      ValuesFrom 1
+    </Result>
+  </Table>
+</Plugin>
+
+<Plugin python>
+  <Module du>
+    <Path>
+        Instance "<%= appId %>"
+        Dir "<%= appDataDir %>"
+    </Path>
+  </Module>
+</Plugin>
+

src/collectd/app_cgroup_v2.ejs

@@ -0,0 +1,42 @@
+LoadPlugin "table"
+<Plugin table>
+  <Table "/sys/fs/cgroup/docker/<%= containerId %>/memory.stat">
+    Instance "<%= appId %>-memory"
+    Separator " \\n"
+    <Result>
+      Type gauge
+      InstancesFrom 0
+      ValuesFrom 1
+    </Result>
+  </Table>
+
+  <Table "/sys/fs/cgroup/docker/<%= containerId %>/memory.max">
+    Instance "<%= appId %>-memory"
+    Separator "\\n"
+    <Result>
+      Type gauge
+      InstancePrefix "max_usage_in_bytes"
+      ValuesFrom 0
+    </Result>
+  </Table>
+
+  <Table "/sys/fs/cgroup/docker/<%= containerId %>/cpu.stat">
+    Instance "<%= appId %>-cpu"
+    Separator " \\n"
+    <Result>
+      Type gauge
+      InstancesFrom 0
+      ValuesFrom 1
+    </Result>
+  </Table>
+</Plugin>
+
+<Plugin python>
+  <Module du>
+    <Path>
+        Instance "<%= appId %>"
+        Dir "<%= appDataDir %>"
+    </Path>
+  </Module>
+</Plugin>
+

src/constants.js

@@ -7,8 +7,8 @@ const CLOUDRON = process.env.BOX_ENV === 'cloudron',
     TEST = process.env.BOX_ENV === 'test';
 
 exports = module.exports = {
-    SMTP_SUBDOMAIN: 'smtp',
-    IMAP_SUBDOMAIN: 'imap',
+    SMTP_LOCATION: 'smtp',
+    IMAP_LOCATION: 'imap',
 
     // These are combined into one array because users and groups become mailboxes
     RESERVED_NAMES: [
@@ -22,7 +22,7 @@ exports = module.exports = {
         'admins', 'users'         // ldap code uses 'users' pseudo group
     ],
 
-    DASHBOARD_SUBDOMAIN: 'my',
+    DASHBOARD_LOCATION: 'my',
 
     PORT: CLOUDRON ? 3000 : 5454,
     INTERNAL_SMTP_PORT: 2525, // this value comes from the mail container
@@ -49,8 +49,6 @@ exports = module.exports = {
     ],
     DEMO_APP_LIMIT: 20,
 
-    PROXY_APP_APPSTORE_ID: 'io.cloudron.builtin.appproxy',
-
     AUTOUPDATE_PATTERN_NEVER: 'never',
 
     // the db field is a blob so we make this explicit

src/cron.js

@@ -28,13 +28,13 @@ const appHealthMonitor = require('./apphealthmonitor.js'),
     dyndns = require('./dyndns.js'),
     eventlog = require('./eventlog.js'),
     janitor = require('./janitor.js'),
-    paths = require('./paths.js'),
     safe = require('safetydance'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
     system = require('./system.js'),
     updater = require('./updater.js'),
     updateChecker = require('./updatechecker.js'),
+    userdirectory = require('./userdirectory.js'),
     _ = require('underscore');
 
 const gJobs = {
@@ -50,8 +50,7 @@ const gJobs = {
     dockerVolumeCleaner: null,
     dynamicDns: null,
     schedulerSync: null,
-    appHealthMonitor: null,
-    diskUsage: null
+    appHealthMonitor: null
 };
 
 // cron format
@@ -62,46 +61,16 @@ const gJobs = {
 // Months: 0-11
 // Day of Week: 0-6
 
-function getCronSeed() {
-    let hour = null;
-    let minute = null;
-
-    const seedData = safe.fs.readFileSync(paths.CRON_SEED_FILE, 'utf8') || '';
-    const parts = seedData.split(':');
-    if (parts.length === 2) {
-        hour = parseInt(parts[0]) || null;
-        minute = parseInt(parts[1]) || null;
-    }
-
-    if ((hour == null || hour < 0 || hour > 23) || (minute == null || minute < 0 || minute > 60)) {
-        hour = Math.floor(24 * Math.random());
-        minute = Math.floor(60 * Math.random());
-
-        debug(`getCronSeed: writing new cron seed file with ${hour}:${minute} to ${paths.CRON_SEED_FILE}`);
-
-        safe.fs.writeFileSync(paths.CRON_SEED_FILE, `${hour}:${minute}`);
-    }
-
-    return { hour, minute };
-}
-
 async function startJobs() {
-    const { hour, minute } = getCronSeed();
-
-    debug(`startJobs: starting cron jobs with hour ${hour} and minute ${minute}`);
+    debug('startJobs: starting cron jobs');
 
+    const randomTick = Math.floor(60*Math.random());
     gJobs.systemChecks = new CronJob({
-        cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
+        cronTime: `${randomTick} ${randomTick} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
         onTick: async () => await safe(cloudron.runSystemChecks(), { debug }),
         start: true
     });
 
-    gJobs.diskUsage = new CronJob({
-        cronTime: `00 ${minute} 3 * * *`, // once a day
-        onTick: async () => await safe(cloudron.updateDiskUsage(), { debug }),
-        start: true
-    });
-
     gJobs.diskSpaceChecker = new CronJob({
         cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
         onTick: async () => await safe(system.checkDiskSpace(), { debug }),
@@ -110,7 +79,7 @@ async function startJobs() {
 
     // this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
     gJobs.updateCheckerJob = new CronJob({
-        cronTime: `00 ${minute} 1,5,9,13,17,21,23 * * *`,
+        cronTime: `${randomTick} ${randomTick} 1,5,9,13,17,21,23 * * *`,
         onTick: async () => await safe(updateChecker.checkForUpdates({ automatic: true }), { debug }),
         start: true
     });
@@ -129,7 +98,7 @@ async function startJobs() {
 
     gJobs.cleanupEventlog = new CronJob({
         cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 60 * 60 * 24 * 60 * 1000) }), { debug }), // 60 days ago
+        onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 60 * 60 * 24 * 10 * 1000) }), { debug }), // 10 days ago
         start: true
     });
 
@@ -145,9 +114,8 @@ async function startJobs() {
         start: true
     });
 
-    // randomized per Cloudron based on hourlySeed
     gJobs.certificateRenew = new CronJob({
-        cronTime: `00 10 ${hour} * * *`,
+        cronTime: '00 00 */12 * * *', // every 12 hours
         onTick: async () => await safe(cloudron.renewCerts({}, AuditSource.CRON), { debug }),
         start: true
     });
@@ -180,6 +148,10 @@ async function handleSettingsChanged(key, value) {
         await stopJobs();
         await startJobs();
         break;
+    case settings.USER_DIRECTORY_KEY:
+        if (value.enabled) await userdirectory.start();
+        else await userdirectory.stop();
+        break;
     default:
         break;
     }

src/database.js

@@ -61,9 +61,8 @@ async function initialize() {
         // note the pool also has an 'acquire' event but that is called whenever we do a getConnection()
         connection.on('error', (error) => debug(`Connection ${connection.threadId} error: ${error.message} ${error.code}`));
 
-        connection.query(`USE ${gDatabase.name}`);
+        connection.query('USE ' + gDatabase.name);
         connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
-        connection.query('SET SESSION group_concat_max_len = 65536'); // GROUP_CONCAT has only 1024 default
     });
 }
 

src/df.js

@@ -1,46 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    disks,
-    file
-};
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    safe = require('safetydance');
-
-function parseLine(line) {
-    const parts = line.split(/\s+/, 7); // this way the mountpoint can have spaces in it
-
-    return {
-        filesystem: parts[0],
-        type: parts[1],
-        size: Number.parseInt(parts[2], 10),
-        used: Number.parseInt(parts[3], 10),
-        available: Number.parseInt(parts[4], 10),
-        capacity: Number.parseInt(parts[5], 10) / 100, // note: this has a trailing %
-        mountpoint: parts[6]
-    };
-}
-
-async function disks() {
-    const output = safe.child_process.execSync('df -B1 --output=source,fstype,size,used,avail,pcent,target', { encoding: 'utf8' });
-    if (!output) throw new BoxError(BoxError.FS_ERROR, `Error running df: ${safe.error.message}`);
-
-    const lines = output.trim().split('\n').slice(1); // discard header
-    const result = [];
-    for (const line of lines) {
-        result.push(parseLine(line));
-    }
-    return result;
-}
-
-async function file(filename) {
-    assert.strictEqual(typeof filename, 'string');
-
-    const output = safe.child_process.execSync(`df -B1 --output=source,fstype,size,used,avail,pcent,target ${filename}`, { encoding: 'utf8' });
-    if (!output) throw new BoxError(BoxError.FS_ERROR, `Error running df: ${safe.error.message}`);
-
-    const lines = output.trim().split('\n').slice(1); // discard header
-    return parseLine(lines[0]);
-}

src/dns.js

@@ -59,28 +59,28 @@ function api(provider) {
     }
 }
 
-function fqdn(subdomain, domain) {
+function fqdn(subdomain, domainObject) {
     assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof domainObject, 'object');
 
-    return subdomain + (subdomain ? '.' : '') + domain;
+    return subdomain + (subdomain ? '.' : '') + domainObject.domain;
 }
 
 // Hostname validation comes from RFC 1123 (section 2.1)
 // Domain name validation comes from RFC 2181 (Name syntax)
 // https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
 // We are validating the validity of the location-fqdn as host name (and not dns name)
-function validateHostname(subdomain, domain) {
+function validateHostname(subdomain, domainObject) {
     assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof domainObject, 'object');
 
-    const hostname = fqdn(subdomain, domain);
+    const hostname = fqdn(subdomain, domainObject);
 
-    const RESERVED_SUBDOMAINS = [
-        constants.SMTP_SUBDOMAIN,
-        constants.IMAP_SUBDOMAIN
+    const RESERVED_LOCATIONS = [
+        constants.SMTP_LOCATION,
+        constants.IMAP_LOCATION
     ];
-    if (RESERVED_SUBDOMAINS.indexOf(subdomain) !== -1) return new BoxError(BoxError.BAD_FIELD, `subdomain '${subdomain}' is reserved`);
+    if (RESERVED_LOCATIONS.indexOf(subdomain) !== -1) return new BoxError(BoxError.BAD_FIELD, `subdomain '${subdomain}' is reserved`);
 
     if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, `subdomain '${subdomain}' is reserved`);
 
@@ -184,11 +184,11 @@ async function waitForDnsRecord(subdomain, domain, type, value, options) {
     await api(domainObject.provider).wait(domainObject, subdomain, type, value, options);
 }
 
-function makeWildcard(fqdn) {
-    assert.strictEqual(typeof fqdn, 'string');
+function makeWildcard(vhost) {
+    assert.strictEqual(typeof vhost, 'string');
 
-    // if the fqdn is like *.example.com, this function will do nothing
-    const parts = fqdn.split('.');
+    // if the vhost is like *.example.com, this function will do nothing
+    let parts = vhost.split('.');
     parts[0] = '*';
     return parts.join('.');
 }
@@ -295,7 +295,7 @@ async function syncDnsRecords(options, progressCallback) {
         progress += Math.round(100/(1+allDomains.length));
 
         let locations = [];
-        if (domain.domain === settings.dashboardDomain()) locations.push({ subdomain: constants.DASHBOARD_SUBDOMAIN, domain: settings.dashboardDomain() });
+        if (domain.domain === settings.dashboardDomain()) locations.push({ subdomain: constants.DASHBOARD_LOCATION, domain: settings.dashboardDomain() });
         if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.dashboardFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
 
         for (const app of allApps) {

src/dns/cloudflare.js

@@ -105,7 +105,7 @@ async function upsert(domainObject, location, type, values) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject.domain);
+        fqdn = dns.fqdn(location, domainObject);
 
     debug('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
@@ -166,7 +166,7 @@ async function get(domainObject, location, type) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject.domain);
+        fqdn = dns.fqdn(location, domainObject);
 
     const zone = await getZoneByName(domainConfig, zoneName);
     const result = await getDnsRecords(domainConfig, zone.id, fqdn, type);
@@ -182,7 +182,7 @@ async function del(domainObject, location, type, values) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject.domain);
+        fqdn = dns.fqdn(location, domainObject);
 
     const zone = await getZoneByName(domainConfig, zoneName);
 
@@ -212,7 +212,7 @@ async function wait(domainObject, subdomain, type, value, options) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(subdomain, domainObject.domain);
+        fqdn = dns.fqdn(subdomain, domainObject);
 
     debug('wait: %s for zone %s of type %s', fqdn, zoneName, type);
 

src/dns/digitalocean.js

@@ -200,17 +200,11 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
 
-// https://stackoverflow.com/questions/14313183/javascript-regex-how-do-i-check-if-the-string-is-ascii-only
-function isASCII(str) {
-    // eslint-disable-next-line no-control-regex
-    return /^[\x00-\x7F]*$/.test(str);
-}
-
 async function verifyDomainConfig(domainObject) {
     assert.strictEqual(typeof domainObject, 'object');
 
@@ -218,7 +212,6 @@ async function verifyDomainConfig(domainObject) {
         zoneName = domainObject.zoneName;
 
     if (!domainConfig.token || typeof domainConfig.token !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string');
-    if (!isASCII(domainConfig.token)) throw new BoxError(BoxError.BAD_FIELD, 'token contains invalid characters');
 
     const ip = '127.0.0.1';
 

src/dns/gandi.js

@@ -119,7 +119,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/gcdns.js

@@ -76,7 +76,7 @@ async function upsert(domainObject, location, type, values) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject.domain);
+        fqdn = dns.fqdn(location, domainObject);
 
     debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
@@ -105,7 +105,7 @@ async function get(domainObject, location, type) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject.domain);
+        fqdn = dns.fqdn(location, domainObject);
 
     const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
 
@@ -130,7 +130,7 @@ async function del(domainObject, location, type, values) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject.domain);
+        fqdn = dns.fqdn(location, domainObject);
 
     const zone = await getZoneByName(getDnsCredentials(domainConfig), zoneName);
 
@@ -151,7 +151,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/godaddy.js

@@ -151,7 +151,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
@@ -178,7 +178,7 @@ async function verifyDomainConfig(domainObject) {
     if (error && error.code === 'ENOTFOUND') throw new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain');
     if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
-    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1 || n.toLowerCase().indexOf('.secureserver.net') !== -1; })) {
+    if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('.domaincontrol.com') !== -1; })) {
         debug('verifyDomainConfig: %j does not contain GoDaddy NS', nameservers);
         throw new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to GoDaddy');
     }

src/dns/hetzner.js

@@ -13,7 +13,7 @@ exports = module.exports = {
 const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
-    debug = require('debug')('box:dns/hetzner'),
+    debug = require('debug')('box:dns/digitalocean'),
     dig = require('../dig.js'),
     dns = require('../dns.js'),
     safe = require('safetydance'),
@@ -216,7 +216,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/linode.js

@@ -225,7 +225,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/manual.js

@@ -62,7 +62,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/namecheap.js

@@ -237,7 +237,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/namecom.js

@@ -206,7 +206,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/netcup.js

@@ -217,7 +217,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/route53.js

@@ -95,7 +95,7 @@ async function upsert(domainObject, location, type, values) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject.domain);
+        fqdn = dns.fqdn(location, domainObject);
 
     debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
@@ -134,7 +134,7 @@ async function get(domainObject, location, type) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject.domain);
+        fqdn = dns.fqdn(location, domainObject);
 
     const zone = await getZoneByName(domainConfig, zoneName);
 
@@ -165,7 +165,7 @@ async function del(domainObject, location, type, values) {
 
     const domainConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = dns.fqdn(location, domainObject.domain);
+        fqdn = dns.fqdn(location, domainObject);
 
     const zone = await getZoneByName(domainConfig, zoneName);
 
@@ -212,7 +212,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/vultr.js

@@ -195,7 +195,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }

src/dns/wildcard.js

@@ -62,7 +62,7 @@ async function wait(domainObject, subdomain, type, value, options) {
     assert.strictEqual(typeof value, 'string');
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
 
-    const fqdn = dns.fqdn(subdomain, domainObject.domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await waitForDns(fqdn, domainObject.zoneName, type, value, options);
 }
@@ -77,7 +77,7 @@ async function verifyDomainConfig(domainObject) {
     if (error || !nameservers) throw new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers');
 
     const location = 'cloudrontestdns';
-    const fqdn = dns.fqdn(location, domainObject.domain);
+    const fqdn = dns.fqdn(location, domainObject);
 
     const [ipv4Error, ipv4Result] = await safe(dig.resolve(fqdn, 'A', { server: '127.0.0.1', timeout: 5000 }));
     if (ipv4Error && (ipv4Error.code === 'ENOTFOUND' || ipv4Error.code === 'ENODATA')) throw new BoxError(BoxError.BAD_FIELD, `Unable to resolve IPv4 of ${fqdn}. Please check if you have set up *.${domainObject.domain} to point to this server's IP`);

src/docker.js

@@ -8,7 +8,6 @@ exports = module.exports = {
     ping,
 
     info,
-    df,
     downloadImage,
     createContainer,
     startContainer,
@@ -39,7 +38,7 @@ const apps = require('./apps.js'),
     debug = require('debug')('box:docker'),
     delay = require('./delay.js'),
     Docker = require('dockerode'),
-    paths = require('./paths.js'),
+    reverseProxy = require('./reverseproxy.js'),
     services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
@@ -205,11 +204,18 @@ async function getAddonMounts(app) {
             break;
         }
         case 'tls': {
-            const certificateDir = `${paths.PLATFORM_DATA_DIR}/tls/${app.id}`;
+            const bundle = await reverseProxy.getCertificatePath(app.fqdn, app.domain);
 
             mounts.push({
-                Target: '/etc/certs',
-                Source: certificateDir,
+                Target: '/etc/certs/tls_cert.pem',
+                Source: bundle.certFilePath,
+                Type: 'bind',
+                ReadOnly: true
+            });
+
+            mounts.push({
+                Target: '/etc/certs/tls_key.pem',
+                Source: bundle.keyFilePath,
                 Type: 'bind',
                 ReadOnly: true
             });
@@ -308,15 +314,6 @@ async function createSubcontainer(app, name, cmd, options) {
     const mounts = await getMounts(app);
 
     const addonEnv = await services.getEnvironment(app);
-    const runtimeVolumes = {
-        '/tmp': {},
-        '/run': {},
-        '/home/cloudron/.cache': {},
-        '/root/.cache': {}
-    };
-    if (app.manifest.runtimeDirs) {
-        app.manifest.runtimeDirs.forEach(dir => runtimeVolumes[dir] = {});
-    }
 
     let containerOptions = {
         name: name, // for referencing containers
@@ -325,7 +322,10 @@ async function createSubcontainer(app, name, cmd, options) {
         Cmd: (isAppContainer && app.debugMode && app.debugMode.cmd) ? app.debugMode.cmd : cmd,
         Env: stdEnv.concat(addonEnv).concat(portEnv).concat(appEnv).concat(secondaryDomainsEnv),
         ExposedPorts: isAppContainer ? exposedPorts : { },
-        Volumes: runtimeVolumes,
+        Volumes: { // see also ReadonlyRootfs
+            '/tmp': {},
+            '/run': {}
+        },
         Labels: {
             'fqdn': app.fqdn,
             'appId': app.id,
@@ -342,7 +342,7 @@ async function createSubcontainer(app, name, cmd, options) {
                     'syslog-format': 'rfc5424'
                 }
             },
-            Memory: await system.getMemoryAllocation(memoryLimit),
+            Memory: system.getMemoryAllocation(memoryLimit),
             MemorySwap: memoryLimit, // Memory + Swap
             PortBindings: isAppContainer ? dockerPortBindings : { },
             PublishAllPorts: false,
@@ -630,12 +630,6 @@ async function info() {
     return result;
 }
 
-async function df() {
-    const [error, result] = await safe(gConnection.df());
-    if (error) throw new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker');
-    return result;
-}
-
 async function update(name, memory, memorySwap) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof memory, 'number');

src/domains.js

@@ -9,8 +9,6 @@ module.exports = exports = {
     del,
     clear,
 
-    getDomainObjectMap,
-
     removePrivateFields,
     removeRestrictedFields,
 };
@@ -19,7 +17,6 @@ const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     crypto = require('crypto'),
     database = require('./database.js'),
-    debug = require('debug')('box:domains'),
     eventlog = require('./eventlog.js'),
     mail = require('./mail.js'),
     reverseProxy = require('./reverseproxy.js'),
@@ -80,13 +77,13 @@ async function verifyDomainConfig(domainConfig, domain, zoneName, provider) {
     if (!backend) throw new BoxError(BoxError.BAD_FIELD, 'Invalid provider');
 
     const domainObject = { config: domainConfig, domain: domain, zoneName: zoneName };
-    const [error, sanitizedConfig] = await safe(api(provider).verifyDomainConfig(domainObject));
-    if (error && error.reason === BoxError.ACCESS_DENIED) throw new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`);
-    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`);
-    if (error && error.reason === BoxError.EXTERNAL_ERROR) throw new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`);
-    if (error) throw error;
+    const [error, result] = await safe(api(provider).verifyDomainConfig(domainObject));
+    if (error && error.reason === BoxError.ACCESS_DENIED) return { error: new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`) };
+    if (error && error.reason === BoxError.NOT_FOUND) return { error: new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`) };
+    if (error && error.reason === BoxError.EXTERNAL_ERROR) return { error: new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`) };
+    if (error) return { error };
 
-    return sanitizedConfig;
+    return { error: null, sanitizedConfig: result };
 }
 
 function validateTlsConfig(tlsConfig, dnsProvider) {
@@ -137,7 +134,7 @@ async function add(domain, data, auditSource) {
     }
 
     if (fallbackCertificate) {
-        let error = reverseProxy.validateCertificate('test', domain, fallbackCertificate);
+        let error = reverseProxy.validateCertificate('test', { domain, config }, fallbackCertificate);
         if (error) throw error;
     } else {
         fallbackCertificate = await reverseProxy.generateFallbackCertificate(domain);
@@ -154,11 +151,12 @@ async function add(domain, data, auditSource) {
         dkimSelector = `cloudron-${suffix}`;
     }
 
-    const sanitizedConfig = await verifyDomainConfig(config, domain, zoneName, provider);
+    const result = await verifyDomainConfig(config, domain, zoneName, provider);
+    if (result.error) throw result.error;
 
-    const queries = [
+    let queries = [
         { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson, fallbackCertificateJson) VALUES (?, ?, ?, ?, ?, ?)',
-            args: [ domain, zoneName, provider, JSON.stringify(sanitizedConfig), JSON.stringify(tlsConfig), JSON.stringify(fallbackCertificate) ] },
+            args: [ domain, zoneName, provider, JSON.stringify(result.sanitizedConfig), JSON.stringify(tlsConfig), JSON.stringify(fallbackCertificate) ] },
         { query: 'INSERT INTO mail (domain, dkimKeyJson, dkimSelector) VALUES (?, ?, ?)', args: [ domain, JSON.stringify(dkimKey), dkimSelector || 'cloudron' ] },
     ];
 
@@ -170,7 +168,7 @@ async function add(domain, data, auditSource) {
 
     await eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
 
-    safe(mail.onDomainAdded(domain), { debug }); // background
+    safe(mail.onDomainAdded(domain)); // background
 }
 
 async function get(domain) {
@@ -197,6 +195,7 @@ async function setConfig(domain, data, auditSource) {
     assert.strictEqual(typeof auditSource, 'object');
 
     let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
+    let error;
 
     if (settings.isDemo() && (domain === settings.dashboardDomain())) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
 
@@ -208,19 +207,20 @@ async function setConfig(domain, data, auditSource) {
     }
 
     if (fallbackCertificate) {
-        let error = reverseProxy.validateCertificate('test', domain, fallbackCertificate);
+        let error = reverseProxy.validateCertificate('test', domainObject, fallbackCertificate);
         if (error) throw error;
     }
 
-    const tlsConfigError = validateTlsConfig(tlsConfig, provider);
-    if (tlsConfigError) throw tlsConfigError;
+    error = validateTlsConfig(tlsConfig, provider);
+    if (error) throw error;
 
     if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config);
 
-    const sanitizedConfig = await verifyDomainConfig(config, domain, zoneName, provider);
+    const result = await verifyDomainConfig(config, domain, zoneName, provider);
+    if (result.error) throw result.error;
 
     const newData = {
-        config: sanitizedConfig,
+        config: result.sanitizedConfig,
         zoneName,
         provider,
         tlsConfig,
@@ -228,7 +228,7 @@ async function setConfig(domain, data, auditSource) {
 
     if (fallbackCertificate) newData.fallbackCertificate = fallbackCertificate;
 
-    const args = [], fields = [];
+    let args = [ ], fields = [ ];
     for (const k in newData) {
         if (k === 'config' || k === 'tlsConfig' || k === 'fallbackCertificate') { // json fields
             fields.push(`${k}Json = ?`);
@@ -240,11 +240,13 @@ async function setConfig(domain, data, auditSource) {
     }
     args.push(domain);
 
-    const result = await database.query('UPDATE domains SET ' + fields.join(', ') + ' WHERE domain=?', args);
-    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
+    [error] = await safe(database.query('UPDATE domains SET ' + fields.join(', ') + ' WHERE domain=?', args));
+    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 
-    if (fallbackCertificate) await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
-    if (!_.isEqual(domainObject.tlsConfig, tlsConfig.provider)) await reverseProxy.handleCertificateProviderChanged(domain);
+    if (!fallbackCertificate) return;
+
+    await reverseProxy.setFallbackCertificate(domain, fallbackCertificate);
 
     await eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, zoneName, provider });
 }
@@ -254,11 +256,12 @@ async function setWellKnown(domain, wellKnown, auditSource) {
     assert.strictEqual(typeof wellKnown, 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
-    const wellKnownError = validateWellKnown(wellKnown);
-    if (wellKnownError) throw wellKnownError;
+    let error = validateWellKnown(wellKnown);
+    if (error) throw error;
 
-    const result = await database.query('UPDATE domains SET wellKnownJson = ? WHERE domain=?', [ JSON.stringify(wellKnown), domain ]);
-    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
+    [error] = await safe(database.query('UPDATE domains SET wellKnownJson = ? WHERE domain=?', [ JSON.stringify(wellKnown), domain ]));
+    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.NOT_FOUND, 'Domain not found');
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 
     await eventlog.add(eventlog.ACTION_DOMAIN_UPDATE, auditSource, { domain, wellKnown });
 }
@@ -308,10 +311,3 @@ function removeRestrictedFields(domain) {
 
     return result;
 }
-
-async function getDomainObjectMap() {
-    const domainObjects = await list();
-    let domainObjectMap = {};
-    for (let d of domainObjects) { domainObjectMap[d.domain] = d; }
-    return domainObjectMap;
-}

src/dyndns.js

@@ -36,8 +36,8 @@ async function sync(auditSource) {
     }
 
     debug(`refreshDNS: updating IP from ${info.ipv4} to ipv4: ${ipv4} (changed: ${ipv4Changed}) ipv6: ${ipv6} (changed: ${ipv6Changed})`);
-    if (ipv4Changed) await dns.upsertDnsRecords(constants.DASHBOARD_SUBDOMAIN, settings.dashboardDomain(), 'A', [ ipv4 ]);
-    if (ipv6Changed) await dns.upsertDnsRecords(constants.DASHBOARD_SUBDOMAIN, settings.dashboardDomain(), 'AAAA', [ ipv6 ]);
+    if (ipv4Changed) await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'A', [ ipv4 ]);
+    if (ipv6Changed) await dns.upsertDnsRecords(constants.DASHBOARD_LOCATION, settings.dashboardDomain(), 'AAAA', [ ipv6 ]);
 
     const result = await apps.list();
     for (const app of result) {

src/eventlog.js

@@ -35,7 +35,7 @@ exports = module.exports = {
     ACTION_BACKUP_CLEANUP_FINISH: 'backup.cleanup.finish',
 
     ACTION_CERTIFICATE_NEW: 'certificate.new',
-    ACTION_CERTIFICATE_RENEWAL: 'certificate.renew', // obsolete
+    ACTION_CERTIFICATE_RENEWAL: 'certificate.renew',
     ACTION_CERTIFICATE_CLEANUP: 'certificate.cleanup',
 
     ACTION_DASHBOARD_DOMAIN_UPDATE: 'dashboard.domain.update',
@@ -69,7 +69,6 @@ exports = module.exports = {
 
     ACTION_USER_ADD: 'user.add',
     ACTION_USER_LOGIN: 'user.login',
-    ACTION_USER_LOGIN_GHOST: 'user.login.ghost',
     ACTION_USER_LOGOUT: 'user.logout',
     ACTION_USER_REMOVE: 'user.remove',
     ACTION_USER_UPDATE: 'user.update',

src/externalldap.js

@@ -2,7 +2,6 @@
 
 exports = module.exports = {
     verifyPassword,
-    verifyPasswordAndTotpToken,
     maybeCreateUser,
 
     testConfig,
@@ -45,7 +44,6 @@ function translateUser(ldapConfig, ldapUser) {
     return {
         username: ldapUser[ldapConfig.usernameField].toLowerCase(),
         email: ldapUser.mail || ldapUser.mailPrimaryAddress,
-        twoFactorAuthenticationEnabled: !!ldapUser.twoFactorAuthenticationEnabled,
         displayName: ldapUser.displayName || ldapUser.cn // user.giveName + ' ' + user.sn
     };
 }
@@ -256,11 +254,8 @@ async function maybeCreateUser(identifier) {
         throw error;
     }
 
-    // fetch the full record and amend potential twoFA settings
-    const newUser = await users.get(userId);
-    if (user.twoFactorAuthenticationEnabled) newUser.twoFactorAuthenticationEnabled = true;
-
-    return newUser;
+    // fetch the full record
+    return await users.get(userId);
 }
 
 async function verifyPassword(user, password) {
@@ -284,32 +279,6 @@ async function verifyPassword(user, password) {
     return translateUser(externalLdapConfig, ldapUsers[0]);
 }
 
-async function verifyPasswordAndTotpToken(user, password, totpToken) {
-    assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof password, 'string');
-    assert.strictEqual(typeof totpToken, 'string');
-
-    const externalLdapConfig = await settings.getExternalLdapConfig();
-    if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');
-
-    const ldapUsers = await ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${user.username}` });
-    if (ldapUsers.length === 0) throw new BoxError(BoxError.NOT_FOUND);
-    if (ldapUsers.length > 1) throw new BoxError(BoxError.CONFLICT);
-
-    const client = await getClient(externalLdapConfig, { bind: false });
-
-    // inject totptoken into first attribute
-    const rdns = ldapUsers[0].dn.split(',');
-    const totpTokenDn = `${rdns[0]}+totptoken=${totpToken},` + rdns.slice(1).join(',');
-
-    const [error] = await safe(util.promisify(client.bind.bind(client))(totpTokenDn, password));
-    client.unbind();
-    if (error instanceof ldap.InvalidCredentialsError) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, error);
-
-    return translateUser(externalLdapConfig, ldapUsers[0]);
-}
-
 async function startSyncer() {
     const externalLdapConfig = await settings.getExternalLdapConfig();
     if (externalLdapConfig.provider === 'noop') throw new BoxError(BoxError.BAD_STATE, 'not enabled');

src/graphs.js

@@ -1,125 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    getSystem,
-    getContainerStats
-};
-
-const apps = require('./apps.js'),
-    assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    docker = require('./docker.js'),
-    os = require('os'),
-    safe = require('safetydance'),
-    services = require('./services.js'),
-    superagent = require('superagent');
-
-// for testing locally: curl 'http://${graphite-ip}:8000/graphite-web/render?format=json&from=-1min&target=absolute(collectd.localhost.du-docker.capacity-usage)'
-// the datapoint is (value, timestamp) https://graphite.readthedocs.io/en/latest/
-async function getGraphiteUrl() {
-    const [error, result] = await safe(docker.inspect('graphite'));
-    if (error && error.reason === BoxError.NOT_FOUND) return { status: exports.SERVICE_STATUS_STOPPED };
-    if (error) throw error;
-
-    const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
-    if (!ip) throw new BoxError(BoxError.INACTIVE, 'Error getting IP of graphite service');
-
-    return `http://${ip}:8000/graphite-web/render`;
-}
-
-async function getContainerStats(name, fromMinutes, noNullPoints) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof fromMinutes, 'number');
-    assert.strictEqual(typeof noNullPoints, 'boolean');
-
-    const timeBucketSize = fromMinutes > (24 * 60) ? (6*60) : 5;
-    const graphiteUrl = await getGraphiteUrl();
-
-    // https://collectd.org/wiki/index.php/Data_source . the gauge is point in time value. counter is the change of value
-    const targets = [
-        `summarize(collectd.localhost.docker-stats-${name}.gauge-cpu-perc, "${timeBucketSize}min", "avg")`,
-        `summarize(collectd.localhost.docker-stats-${name}.gauge-mem-used, "${timeBucketSize}min", "avg")`,
-        // `summarize(collectd.localhost.docker-stats-${name}.gauge-mem-max, "${timeBucketSize}min", "avg")`,
-        `summarize(collectd.localhost.docker-stats-${name}.counter-blockio-read, "${timeBucketSize}min", "sum")`,
-        `summarize(collectd.localhost.docker-stats-${name}.counter-blockio-write, "${timeBucketSize}min", "sum")`,
-        `summarize(collectd.localhost.docker-stats-${name}.counter-network-read, "${timeBucketSize}min", "sum")`,
-        `summarize(collectd.localhost.docker-stats-${name}.counter-network-write, "${timeBucketSize}min", "sum")`,
-        `summarize(collectd.localhost.docker-stats-${name}.gauge-blockio-read, "${fromMinutes}min", "max")`,
-        `summarize(collectd.localhost.docker-stats-${name}.gauge-blockio-write, "${fromMinutes}min", "max")`,
-        `summarize(collectd.localhost.docker-stats-${name}.gauge-network-read, "${fromMinutes}min", "max")`,
-        `summarize(collectd.localhost.docker-stats-${name}.gauge-network-write, "${fromMinutes}min", "max")`,
-    ];
-
-    const results = [];
-
-    for (const target of targets) {
-        const query = {
-            target: target,
-            format: 'json',
-            from: `-${fromMinutes}min`,
-            until: 'now',
-            noNullPoints: !!noNullPoints
-        };
-
-        const [error, response] = await safe(superagent.get(graphiteUrl).query(query).timeout(30 * 1000).ok(() => true));
-        if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-        if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error with ${target}: ${response.status} ${response.text}`);
-
-        results.push(response.body[0] && response.body[0].datapoints ? response.body[0].datapoints : []);
-    }
-
-    // results are datapoints[[value, ts], [value, ts], ...];
-    return {
-        cpu: results[0],
-        memory: results[1],
-        blockRead: results[2],
-        blockWrite: results[3],
-        networkRead: results[4],
-        networkWrite: results[5],
-        blockReadTotal: results[6][0] && results[6][0][0] ? results[6][0][0] : 0,
-        blockWriteTotal: results[7][0] && results[7][0][0] ? results[7][0][0] : 0,
-        networkReadTotal: results[8][0] && results[8][0][0] ? results[8][0][0] : 0,
-        networkWriteTotal: results[9][0] && results[9][0][0] ? results[9][0][0] : 0,
-        cpuCount: os.cpus().length
-    };
-}
-
-async function getSystem(fromMinutes, noNullPoints) {
-    assert.strictEqual(typeof fromMinutes, 'number');
-    assert.strictEqual(typeof noNullPoints, 'boolean');
-
-    const timeBucketSize = fromMinutes > (24 * 60) ? (6*60) : 5;
-    const graphiteUrl = await getGraphiteUrl();
-
-    const cpuQuery = `summarize(sum(collectd.localhost.aggregation-cpu-sum.cpu-system, collectd.localhost.aggregation-cpu-sum.cpu-user), "${timeBucketSize}min", "avg")`;
-    const memoryQuery = `summarize(collectd.localhost.memory.memory-used, "${timeBucketSize}min", "avg")`;
-
-    const query = {
-        target: [ cpuQuery, memoryQuery ],
-        format: 'json',
-        from: `-${fromMinutes}min`,
-        until: 'now'
-    };
-
-    const [memCpuError, memCpuResponse] = await safe(superagent.get(graphiteUrl).query(query).timeout(30 * 1000).ok(() => true));
-    if (memCpuError) throw new BoxError(BoxError.NETWORK_ERROR, memCpuError.message);
-    if (memCpuResponse.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${memCpuResponse.status} ${memCpuResponse.text}`);
-
-    const appResponses = {};
-    for (const app of await apps.list()) {
-        appResponses[app.id] = await getContainerStats(app.id, fromMinutes, noNullPoints);
-    }
-
-    const serviceResponses = {};
-    for (const serviceId of await services.listServices()) {
-        serviceResponses[serviceId] = await getContainerStats(serviceId, fromMinutes, noNullPoints);
-    }
-
-    return {
-        cpu: memCpuResponse.body[0] && memCpuResponse.body[0].datapoints ? memCpuResponse.body[0].datapoints : [],
-        memory: memCpuResponse.body[1] && memCpuResponse.body[1].datapoints ? memCpuResponse.body[1].datapoints : [],
-        apps: appResponses,
-        services: serviceResponses,
-        cpuCount: os.cpus().length
-    };
-}

src/hush.js

@@ -5,7 +5,7 @@ const assert = require('assert'),
     crypto = require('crypto'),
     debug = require('debug')('box:hush'),
     fs = require('fs'),
-    ProgressStream = require('./progress-stream.js'),
+    progressStream = require('progress-stream'),
     TransformStream = require('stream').Transform;
 
 class EncryptStream extends TransformStream {
@@ -157,7 +157,7 @@ function createReadStream(sourceFile, encryption) {
     assert.strictEqual(typeof encryption, 'object');
 
     const stream = fs.createReadStream(sourceFile);
-    const ps = new ProgressStream({ interval: 10000 }); // display a progress every 10 seconds
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
 
     stream.on('error', function (error) {
         debug(`createReadStream: read stream error at ${sourceFile}`, error);
@@ -185,7 +185,7 @@ function createWriteStream(destFile, encryption) {
     assert.strictEqual(typeof encryption, 'object');
 
     const stream = fs.createWriteStream(destFile);
-    const ps = new ProgressStream({ interval: 10000 }); // display a progress every 10 seconds
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
 
     stream.on('error', function (error) {
         debug(`createWriteStream: write stream error ${destFile}`, error);

src/infra_version.js

@@ -6,7 +6,7 @@
 
 exports = module.exports = {
     // a version change recreates all containers with latest docker config
-    'version': '49.4.0',
+    'version': '49.0.0',
 
     'baseImages': [
         { repo: 'cloudron/base', tag: 'cloudron/base:3.2.0@sha256:ba1d566164a67c266782545ea9809dc611c4152e27686fd14060332dd88263ea' }
@@ -16,12 +16,12 @@ exports = module.exports = {
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
         'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.4.0@sha256:45817f1631992391d585f171498d257487d872480fd5646723a2b956cc4ef15d' },
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.2.2@sha256:8648ca5a16fcdec72799b919c5f62419fd19e922e3d98d02896b921ae6127ef4' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.3.5@sha256:bc8cb91cbd48ee9a2f5a609b6131cd21a0210c15aaf127ee77963d90a125530a' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.2@sha256:df928d7dce1ac6454fc584787fa863f6d5e7ee0abb775dde5916a555fc94c3c7' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.3.1@sha256:383e11a5c7a54d17eb6bbceb0ffa92f486167be6ea9978ec745c8c8e9b7dfb19' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.7.4@sha256:8ddbf13ee3fd479e18923c7bf1370d9d8aa5f12a94cbbda5afac8b5a4af72a28' },
-        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.2.0@sha256:182e5cae69fbddc703cb9f91be909452065c7ae159e9836cc88317c7a00f0e62' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.2.1@sha256:75cef64ba4917ba9ec68bc0c9d9ba3a9eeae00a70173cd6d81cc6118038737d9' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.3.1@sha256:b0c564d097b765d4a639330843e2e813d2c87fc8ed34b7df7550bf2c6df0012c' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.2.1@sha256:f7f689beea07b1c6a9503a48f6fb38ef66e5b22f59fc585a92842a6578b33d46' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.3.0@sha256:89c4e8083631b6d16b5d630d9b27f8ecf301c62f81219d77bd5948a1f4a4375c' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.6.1@sha256:b8b93f007105080d4812a05648e6bc5e15c95c63f511c829cbc14a163d9ea029' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.1.0@sha256:30ec3a01964a1e01396acf265183997c3e17fb07eac1a82b979292cc7719ff4b' },
         'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.6.1@sha256:ba4b9a1fe274c0ef0a900e5d0deeb8f3da08e118798d1d90fbf995cc0cf6e3a3' }
     }
 };

src/janitor.js

@@ -28,7 +28,7 @@ async function cleanupTmpVolume(containerInfo) {
 
     const cmd = 'find /tmp -type f -mtime +10 -exec rm -rf {} +'.split(' '); // 10 day old files
 
-    debug(`cleanupTmpVolume ${JSON.stringify(containerInfo.Names)}`);
+    debug('cleanupTmpVolume %j', containerInfo.Names);
 
     const [error, execContainer] = await safe(gConnection.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }));
     if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Failed to exec container: ${error.message}`);
@@ -53,6 +53,4 @@ async function cleanupDockerVolumes() {
     for (const container of containers) {
         await safe(cleanupTmpVolume(container), { debug }); // intentionally ignore error
     }
-
-    debug('Cleaned up docker volumes');
 }

src/ldap.js

@@ -25,6 +25,9 @@ let gServer = null;
 
 const NOOP = function () {};
 
+const GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
+const GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
+
 // Will attach req.app if successful
 async function authenticateApp(req, res, next) {
     const sourceIp = req.connection.ldap.id.split(':')[0];
@@ -147,9 +150,6 @@ async function userSearch(req, res, next) {
     const [error, result] = await safe(getUsersWithAccessToApp(req));
     if (error) return next(new ldap.OperationsError(error.toString()));
 
-    const [groupsError, allGroups] = await safe(groups.listWithMembers());
-    if (groupsError) return next(new ldap.OperationsError(error.toString()));
-
     let results = [];
 
     // send user objects
@@ -159,6 +159,9 @@ async function userSearch(req, res, next) {
 
         const dn = ldap.parseDN('cn=' + user.id + ',ou=users,dc=cloudron');
 
+        const memberof = [ GROUP_USERS_DN ];
+        if (users.compareRoles(user.role, users.ROLE_ADMIN) >= 0) memberof.push(GROUP_ADMINS_DN);
+
         const displayName = user.displayName || user.username || ''; // displayName can be empty and username can be null
         const nameParts = displayName.split(' ');
         const firstName = nameParts[0];
@@ -178,7 +181,7 @@ async function userSearch(req, res, next) {
                 givenName: firstName,
                 username: user.username,
                 samaccountname: user.username,      // to support ActiveDirectory clients
-                memberof: allGroups.filter(function (g) { return g.userIds.indexOf(user.id) !== -1; }).map(function (g) { return `cn=${g.name},ou=groups,dc=cloudron`; })
+                memberof: memberof
             }
         };
 
@@ -201,6 +204,9 @@ async function userSearch(req, res, next) {
 async function groupSearch(req, res, next) {
     debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
+    const [error, usersWithAccess] = await safe(getUsersWithAccessToApp(req));
+    if (error) return next(new ldap.OperationsError(error.toString()));
+
     const results = [];
 
     let [errorGroups, resultGroups] = await safe(groups.listWithMembers());
@@ -211,15 +217,15 @@ async function groupSearch(req, res, next) {
     }
 
     resultGroups.forEach(function (group) {
-        const dn = ldap.parseDN(`cn=${group.name},ou=groups,dc=cloudron`);
+        const dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+        const members = group.userIds.filter(function (uid) { return usersWithAccess.map(function (u) { return u.id; }).indexOf(uid) !== -1; });
 
         const obj = {
             dn: dn.toString(),
             attributes: {
                 objectclass: ['group'],
                 cn: group.name,
-                gidnumber: group.id,
-                memberuid: group.userIds
+                memberuid: members
             }
         };
 
@@ -268,35 +274,25 @@ async function groupAdminsCompare(req, res, next) {
 async function mailboxSearch(req, res, next) {
     debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
-    // if cn is set OR filter is mail= we only search for one mailbox specifically
-    let email, dn;
+    // if cn is set we only search for one mailbox specifically
     if (req.dn.rdns[0].attrs.cn) {
-        email = req.dn.rdns[0].attrs.cn.value.toLowerCase();
-        dn = req.dn.toString();
-    } else if (req.filter instanceof ldap.EqualityFilter && req.filter.attribute === 'mail') {
-        email = req.filter.value.toLowerCase();
-        dn = `cn=${email},${req.dn.toString()}`;
-    }
-
-    if (email) {
+        const email = req.dn.rdns[0].attrs.cn.value.toLowerCase();
         const parts = email.split('@');
-        if (parts.length !== 2) return next(new ldap.NoSuchObjectError(dn.toString()));
+        if (parts.length !== 2) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
         const [error, mailbox] = await safe(mail.getMailbox(parts[0], parts[1]));
         if (error) return next(new ldap.OperationsError(error.toString()));
-        if (!mailbox) return next(new ldap.NoSuchObjectError(dn.toString()));
-        if (!mailbox.active) return next(new ldap.NoSuchObjectError(dn.toString()));
+        if (!mailbox) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (!mailbox.active) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
         const obj = {
-            dn: dn.toString(),
+            dn: req.dn.toString(),
             attributes: {
                 objectclass: ['mailbox'],
                 objectcategory: 'mailbox',
                 cn: `${mailbox.name}@${mailbox.domain}`,
                 uid: `${mailbox.name}@${mailbox.domain}`,
-                mail: `${mailbox.name}@${mailbox.domain}`,
-                storagequota: mailbox.storageQuota,
-                messagesquota: mailbox.messagesQuota,
+                mail: `${mailbox.name}@${mailbox.domain}`
             }
         };
 
@@ -309,7 +305,7 @@ async function mailboxSearch(req, res, next) {
         } else {
             res.end();
         }
-    } else { // new sogo and dovecot listing (doveadm -A)
+    } else { // new sogo
         // TODO figure out how proper pagination here could work
         let [error, mailboxes] = await safe(mail.listAllMailboxes(1, 100000));
         if (error) return next(new ldap.OperationsError(error.toString()));
@@ -333,9 +329,7 @@ async function mailboxSearch(req, res, next) {
                     displayname: mailbox.ownerType === mail.OWNERTYPE_USER ? ownerObject.displayName : ownerObject.name,
                     cn: `${mailbox.name}@${mailbox.domain}`,
                     uid: `${mailbox.name}@${mailbox.domain}`,
-                    mail: `${mailbox.name}@${mailbox.domain}`,
-                    storagequota: mailbox.storageQuota,
-                    messagesquota: mailbox.messagesQuota,
+                    mail: `${mailbox.name}@${mailbox.domain}`
                 }
             };
 
@@ -365,7 +359,7 @@ async function mailAliasSearch(req, res, next) {
     const parts = email.split('@');
     if (parts.length !== 2) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-    const [error, alias] = await safe(mail.searchAlias(parts[0], parts[1]));
+    const [error, alias] = await safe(mail.getAlias(parts[0], parts[1]));
     if (error) return next(new ldap.OperationsError(error.toString()));
     if (!alias) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
@@ -379,7 +373,7 @@ async function mailAliasSearch(req, res, next) {
         attributes: {
             objectclass: ['nisMailAlias'],
             objectcategory: 'nisMailAlias',
-            cn: `${parts[0]}@${alias.domain}`, // alias.name can contain wildcard character
+            cn: `${alias.name}@${alias.domain}`,
             rfc822MailMember: `${alias.aliasName}@${alias.aliasDomain}`
         }
     };
@@ -455,7 +449,7 @@ async function authorizeUserForApp(req, res, next) {
     // we return no such object, to avoid leakage of a users existence
     if (!canAccess) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-    await eventlog.upsertLoginEvent(req.user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: req.app.id }, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+    await eventlog.upsertLoginEvent(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: req.app.id }, { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
     res.end();
 }
@@ -600,7 +594,7 @@ async function authenticateService(serviceId, dn, req, res, next) {
     if (verifyError && verifyError.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(dn.toString()));
     if (verifyError) return next(new ldap.OperationsError(verifyError.message));
 
-    eventlog.upsertLoginEvent(result.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
+    eventlog.upsertLoginEvent(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
 
     res.end();
 }

src/log-stream.js

@@ -1,52 +0,0 @@
-'use strict';
-
-const stream = require('stream'),
-    { StringDecoder } = require('string_decoder'),
-    TransformStream = stream.Transform;
-
-class LogStream extends TransformStream {
-    constructor(options) {
-        super();
-        this._options = Object.assign({ source: 'unknown', format: 'json' }, options);
-        this._decoder = new StringDecoder();
-        this._soFar = '';
-    }
-
-    _format(line) {
-        if (this._options.format !== 'json') return line + '\n';
-
-        const data = line.split(' '); // logs are <ISOtimestamp> <msg>
-        let timestamp = (new Date(data[0])).getTime();
-        if (isNaN(timestamp)) timestamp = 0;
-        const message = line.slice(data[0].length+1);
-
-        // ignore faulty empty logs
-        if (!timestamp && !message) return;
-
-        return JSON.stringify({
-            realtimeTimestamp: timestamp * 1000,
-            message: message,
-            source: this._options.source
-        }) + '\n';
-    }
-
-    _transform(chunk, encoding, callback) {
-        const data = this._soFar + this._decoder.write(chunk);
-        let start = this._soFar.length, end = -1;
-        while ((end = data.indexOf('\n', start)) !== -1) {
-            const line = data.slice(start, end); // does not include end
-            this.push(this._format(line));
-            start = end + 1;
-        }
-        this._soFar = data.slice(start);
-        callback(null);
-    }
-
-    _flush(callback) {
-        const line = this._soFar + this._decoder.end();
-        this.push(this._format(line));
-        callback(null);
-    }
-}
-
-exports = module.exports = LogStream;

src/logrotate.ejs

@@ -1,12 +1,11 @@
 # Generated by apptask
 
-# keep upto 5 rotated logs. rotation triggered weekly or ahead of time if size is > 10M
+# keep upto 7 rotated logs. rotation triggered daily or ahead of time if size is > 1M
 <%= volumePath %>/*.log <%= volumePath %>/*/*.log <%= volumePath %>/*/*/*.log {
-    rotate 5
-    weekly
-    maxage 14
+    rotate 7
+    daily
     compress
-    maxsize 10M
+    maxsize 1M
     missingok
     delaycompress
     # this truncates the original log file and not the rotated one
@@ -16,9 +15,7 @@
 /home/yellowtent/platformdata/logs/<%= appId %>/*.log {
     # only keep one rotated file, we currently do not send that over the api
     rotate 1
-    weekly
-    maxage 14
-    maxsize 10M
+    size 10M
     missingok
     # we never compress so we can simply tail the files
     nocompress

src/mail.js

@@ -32,7 +32,7 @@ exports = module.exports = {
 
     startMail,
     restartMail,
-    checkCertificate,
+    handleCertChanged,
     getMailAuth,
 
     sendTestMail,
@@ -48,7 +48,6 @@ exports = module.exports = {
     getAlias,
     getAliases,
     setAliases,
-    searchAlias,
 
     getLists,
     getList,
@@ -97,11 +96,13 @@ const assert = require('assert'),
     services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
+    smtpTransport = require('nodemailer-smtp-transport'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     system = require('./system.js'),
     tasks = require('./tasks.js'),
     users = require('./users.js'),
+    util = require('util'),
     validator = require('validator'),
     _ = require('underscore');
 
@@ -111,7 +112,7 @@ const REMOVE_MAILBOX_CMD = path.join(__dirname, 'scripts/rmmailbox.sh');
 const OWNERTYPES = [ exports.OWNERTYPE_USER, exports.OWNERTYPE_GROUP, exports.OWNERTYPE_APP ];
 
 // if you add a field here, listMailboxes has to be updated
-const MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'ownerType', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain', 'active', 'enablePop3', 'storageQuota', 'messagesQuota' ].join(',');
+const MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'ownerType', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain', 'active', 'enablePop3' ].join(',');
 const MAILDB_FIELDS = [ 'domain', 'enabled', 'mailFromValidation', 'catchAllJson', 'relayJson', 'dkimKeyJson', 'dkimSelector', 'bannerJson' ].join(',');
 
 function postProcessMailbox(data) {
@@ -168,26 +169,12 @@ function validateName(name) {
     return null;
 }
 
-function validateAlias(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox name must be atleast 1 char');
-    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'mailbox name too long');
-
-    // also need to consider valid LDAP characters here (e.g '+' is reserved). keep hyphen at the end so it doesn't become a range.
-    if (/[^a-zA-Z0-9._*-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox name can only contain alphanumerals, dot, hyphen, asterisk or underscore');
-
-    return null;
-}
-
 function validateDisplayName(name) {
     assert.strictEqual(typeof name, 'string');
 
     if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name must be atleast 1 char');
     if (name.length >= 100) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name too long');
-    // technically only ":" is disallowed it seems (https://www.rfc-editor.org/rfc/rfc5322#section-2.2)
-    // in https://www.rfc-editor.org/rfc/rfc2822.html, display-name is a "phrase"
-    if (/["<>)(,;\\@:]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name is not valid');
+    if (/["<>@]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name is not valid');
 
     return null;
 }
@@ -237,8 +224,7 @@ async function checkSmtpRelay(relay) {
         connectionTimeout: 5000,
         greetingTimeout: 5000,
         host: relay.host,
-        port: relay.port,
-        secure: false // haraka relay only supports STARTTLS
+        port: relay.port
     };
 
     // only set auth if either username or password is provided, some relays auth based on IP (range)
@@ -251,9 +237,9 @@ async function checkSmtpRelay(relay) {
 
     if (relay.acceptSelfSignedCerts) options.tls = { rejectUnauthorized: false };
 
-    const transporter = nodemailer.createTransport(options);
+    const transporter = nodemailer.createTransport(smtpTransport(options));
 
-    const [error] = await safe(transporter.verify());
+    const [error] = await safe(util.promisify(transporter.verify)());
     result.status = !error;
     if (error) {
         result.value = result.errorMessage = error.message;
@@ -518,11 +504,7 @@ const RBL_LIST = [
 
 // this function currently only looks for black lists based on IP. TODO: also look up by domain
 async function checkRblStatus(domain) {
-    const [error, ip] = await safe(sysinfo.getServerIPv4());
-    if (error) {
-        debug(`checkRblStatus: unable to determine server IPv4: ${error.message}`);
-        return { status: false, ip: null, servers: [] };
-    }
+    const ip = await sysinfo.getServerIPv4();
 
     const flippedIp = ip.split('.').reverse().join('.');
 
@@ -660,7 +642,7 @@ async function createMailConfig(mailFqdn) {
 
     // create sections for per-domain configuration
     for (const domain of mailDomains) {
-        const catchAll = domain.catchAll.join(',');
+        const catchAll = domain.catchAll.map(function (c) { return `${c}@${domain.domain}`; }).join(',');
         const mailFromValidation = domain.mailFromValidation;
 
         if (!safe.fs.appendFileSync(`${paths.MAIL_CONFIG_DIR}/mail.ini`,
@@ -684,8 +666,7 @@ async function createMailConfig(mailFqdn) {
         const enableRelay = relay.provider !== 'cloudron-smtp' && relay.provider !== 'noop',
             host = relay.host || '',
             port = relay.port || 25,
-            // office365 removed plain auth (https://support.microsoft.com/en-us/office/outlook-com-no-longer-supports-auth-plain-authentication-07f7d5e9-1697-465f-84d2-4513d4ff0145)
-            authType = relay.username ? (relay.provider === 'office365-legacy-smtp' ? 'login' : 'plain') : '',
+            authType = relay.username ? 'plain' : '',
             username = relay.username || '',
             password = relay.password || '',
             forceFromAddress = relay.forceFromAddress ? 'true' : 'false';
@@ -714,18 +695,18 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) {
 
     const tag = infra.images.mail.tag;
     const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT;
-    const memory = await system.getMemoryAllocation(memoryLimit);
+    const memory = system.getMemoryAllocation(memoryLimit);
     const cloudronToken = hat(8 * 128), relayToken = hat(8 * 128);
 
-    const certificate = await reverseProxy.getMailCertificate();
+    const bundle = await reverseProxy.getCertificatePath(mailFqdn, mailDomain);
 
     const dhparamsFilePath = `${paths.MAIL_CONFIG_DIR}/dhparams.pem`;
     const mailCertFilePath = `${paths.MAIL_CONFIG_DIR}/tls_cert.pem`;
     const mailKeyFilePath =  `${paths.MAIL_CONFIG_DIR}/tls_key.pem`;
 
-    if (!safe.child_process.execSync(`cp ${paths.DHPARAMS_FILE} ${dhparamsFilePath}`)) throw new BoxError(BoxError.FS_ERROR, `Could not copy dhparams: ${safe.error.message}`);
-    if (!safe.fs.writeFileSync(mailCertFilePath, certificate.cert)) throw new BoxError(BoxError.FS_ERROR, `Could not create cert file: ${safe.error.message}`);
-    if (!safe.fs.writeFileSync(mailKeyFilePath, certificate.key)) throw new BoxError(BoxError.FS_ERROR, `Could not create key file: ${safe.error.message}`);
+    if (!safe.child_process.execSync(`cp ${paths.DHPARAMS_FILE} ${dhparamsFilePath}`)) throw new BoxError(BoxError.FS_ERROR, 'Could not copy dhparams:' + safe.error.message);
+    if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) throw new BoxError(BoxError.FS_ERROR, 'Could not create cert file:' + safe.error.message);
+    if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) throw new BoxError(BoxError.FS_ERROR, 'Could not create key file:' + safe.error.message);
 
     // if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code
     if (!safe.fs.chmodSync(mailKeyFilePath, 0o644)) throw new BoxError(BoxError.FS_ERROR, `Could not chmod key file: ${safe.error.message}`);
@@ -796,7 +777,6 @@ async function restartMail() {
 async function startMail(existingInfra) {
     assert.strictEqual(typeof existingInfra, 'object');
 
-    debug('startMail: starting');
     await restartMail();
 }
 
@@ -808,19 +788,11 @@ async function restartMailIfActivated() {
         return; // not provisioned yet, do not restart container after dns setup
     }
 
-    debug('restartMailIfActivated: restarting on activated');
     await restartMail();
 }
 
-async function checkCertificate() {
-    const certificate = await reverseProxy.getMailCertificate();
-    const cert = safe.fs.readFileSync(`${paths.MAIL_CONFIG_DIR}/tls_cert.pem`, { encoding: 'utf8' });
-    if (cert === certificate.cert) {
-        debug('checkCertificate: certificate has not changed');
-        return;
-    }
-    debug('checkCertificate: certificate has changed');
-
+async function handleCertChanged() {
+    debug('handleCertChanged: will restart if activated');
     await restartMailIfActivated();
 }
 
@@ -994,7 +966,8 @@ async function setLocation(subdomain, domain, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
 
-    const fqdn = dns.fqdn(subdomain, domain);
+    const domainObject = await domains.get(domain);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     await settings.setMailLocation(domain, fqdn);
 
@@ -1010,7 +983,6 @@ async function onDomainAdded(domain) {
 
     if (!settings.mailFqdn()) return; // mail domain is not set yet (when provisioning)
 
-    debug(`onDomainAdded: configuring mail for added domain ${domain}`);
     await upsertDnsRecords(domain, settings.mailFqdn());
     await restartMailIfActivated();
 }
@@ -1018,7 +990,6 @@ async function onDomainAdded(domain) {
 async function onDomainRemoved(domain) {
     assert.strictEqual(typeof domain, 'string');
 
-    debug(`onDomainRemoved: configuring mail for removed domain ${domain}`);
     await restartMail();
 }
 
@@ -1058,10 +1029,6 @@ async function setCatchAllAddress(domain, addresses) {
     assert.strictEqual(typeof domain, 'string');
     assert(Array.isArray(addresses));
 
-    for (const address of addresses) {
-        if (!validator.isEmail(address)) throw new BoxError(BoxError.BAD_FIELD, `Invalid catch all address: ${address}`);
-    }
-
     await updateDomain(domain, { catchAll: addresses });
 
     safe(restartMail(), { debug }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
@@ -1122,7 +1089,7 @@ async function listMailboxes(domain, search, page, perPage) {
     const escapedSearch = mysql.escape('%' + search + '%'); // this also quotes the string
     const searchQuery = search ? ` HAVING (name LIKE ${escapedSearch} OR aliasNames LIKE ${escapedSearch} OR aliasDomains LIKE ${escapedSearch})` : ''; // having instead of where because of aggregated columns use
 
-    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains, m1.enablePop3 AS enablePop3, m1.storageQuota AS storageQuota, m1.messagesQuota AS messagesQuota '
+    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains, m1.enablePop3 AS enablePop3 '
         + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
         + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
         + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
@@ -1143,7 +1110,7 @@ async function listAllMailboxes(page, perPage) {
     assert.strictEqual(typeof page, 'number');
     assert.strictEqual(typeof perPage, 'number');
 
-    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains, m1.enablePop3 AS enablePop3, m1.storageQuota AS storageQuota, m1.messagesQuota AS messagesQuota '
+    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains, m1.enablePop3 AS enablePop3 '
         + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
         + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
         + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
@@ -1197,12 +1164,10 @@ async function addMailbox(name, domain, data, auditSource) {
     assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
-    const { ownerId, ownerType, active, storageQuota, messagesQuota } = data;
+    const { ownerId, ownerType, active } = data;
     assert.strictEqual(typeof ownerId, 'string');
     assert.strictEqual(typeof ownerType, 'string');
     assert.strictEqual(typeof active, 'boolean');
-    assert(Number.isInteger(storageQuota) && storageQuota >= 0);
-    assert(Number.isInteger(messagesQuota) && messagesQuota >= 0);
 
     name = name.toLowerCase();
 
@@ -1211,13 +1176,12 @@ async function addMailbox(name, domain, data, auditSource) {
 
     if (!OWNERTYPES.includes(ownerType)) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');
 
-    [error] = await safe(database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, active, storageQuota, messagesQuota) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
-        [ name, exports.TYPE_MAILBOX, domain, ownerId, ownerType, active, storageQuota, messagesQuota ]));
+    [error] = await safe(database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, active) VALUES (?, ?, ?, ?, ?, ?)', [ name, exports.TYPE_MAILBOX, domain, ownerId, ownerType, active ]));
     if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
     if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.includes('mailboxes_domain_constraint')) throw new BoxError(BoxError.NOT_FOUND, `no such domain '${domain}'`);
     if (error) throw error;
 
-    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active, storageQuota, messageQuota: messagesQuota });
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active });
 }
 
 async function updateMailbox(name, domain, data, auditSource) {
@@ -1226,30 +1190,23 @@ async function updateMailbox(name, domain, data, auditSource) {
     assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
 
-    const args = [];
-    const fields = [];
-    for (const k in data) {
-        if (k === 'enablePop3' || k === 'active') {
-            fields.push(k + ' = ?');
-            args.push(data[k] ? 1 : 0);
-            continue;
-        }
+    const { ownerId, ownerType, active, enablePop3 } = data;
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
+    assert.strictEqual(typeof active, 'boolean');
+    assert.strictEqual(typeof enablePop3, 'boolean');
 
-        if (k === 'ownerType' && !OWNERTYPES.includes(data[k])) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');
+    name = name.toLowerCase();
 
-        fields.push(k + ' = ?');
-        args.push(data[k]);
-    }
-    args.push(name.toLowerCase());
-    args.push(domain);
+    if (!OWNERTYPES.includes(ownerType)) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');
 
     const mailbox = await getMailbox(name, domain);
     if (!mailbox) throw new BoxError(BoxError.NOT_FOUND, 'No such mailbox');
 
-    const result = await safe(database.query('UPDATE mailboxes SET ' + fields.join(', ') + ' WHERE name = ? AND domain = ?', args));
+    const result = await database.query('UPDATE mailboxes SET ownerId = ?, ownerType = ?, active = ?, enablePop3 = ? WHERE name = ? AND domain = ?', [ ownerId, ownerType, active, enablePop3, name, domain ]);
     if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');
 
-    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, Object.assign(data, { name, domain, oldUserId: mailbox.userId }) );
+    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: mailbox.userId, ownerId, ownerType, active });
 }
 
 async function removeSolrIndex(mailbox) {
@@ -1301,18 +1258,6 @@ async function getAlias(name, domain) {
     return results[0];
 }
 
-async function searchAlias(name, domain) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof domain, 'string');
-
-    const results = await database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE ? LIKE REPLACE(REPLACE(name, '*', '%'), '_', '\\_') AND type = ? AND domain = ?`, [ name, exports.TYPE_ALIAS, domain ]);
-    if (results.length === 0) return null;
-
-    results.forEach(function (result) { postProcessMailbox(result); });
-
-    return results[0];
-}
-
 async function getAliases(name, domain) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
@@ -1332,7 +1277,7 @@ async function setAliases(name, domain, aliases, auditSource) {
         const name = aliases[i].name.toLowerCase();
         const domain = aliases[i].domain.toLowerCase();
 
-        const error = validateAlias(name);
+        const error = validateName(name);
         if (error) throw error;
 
         const mailDomain = await getDomain(domain);

src/mail_templates/password_reset-html.ejs

@@ -6,11 +6,10 @@
 
 <p>{{ passwordResetEmail.description }}</p>
 
-<br/>
+<p>
+  <a href="<%= resetLink %>">{{ passwordResetEmail.resetAction }}</a>
+</p>
 
-<a style="border-radius: 2px; background-color: #2196f3; color: white; padding: 6px 12px; text-decoration: none;" href="<%= resetLink %>">{{ passwordResetEmail.resetAction }}</a>
-
-<br/>
 <br/>
 
 {{ passwordResetEmail.expireNote }}

src/mail_templates/welcome_user-html.ejs

@@ -5,9 +5,9 @@
 <h3>{{ welcomeEmail.salutation }}</h3>
 <h2>{{ welcomeEmail.welcomeTo }}</h2>
 
-<br/>
-
-<a style="border-radius: 2px; background-color: #2196f3; color: white; padding: 6px 12px; text-decoration: none;" href="<%= inviteLink %>">{{ welcomeEmail.inviteLinkAction }}</a>
+<p>
+  <a href="<%= inviteLink %>">{{ welcomeEmail.inviteLinkAction }}</a>
+</p>
 
 <br/>
 <br/>

src/mailer.js

@@ -25,6 +25,7 @@ const assert = require('assert'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     translation = require('./translation.js'),
+    smtpTransport = require('nodemailer-smtp-transport'),
     util = require('util');
 
 const MAIL_TEMPLATES_DIR = path.join(__dirname, 'mail_templates');
@@ -51,14 +52,14 @@ async function sendMail(mailOptions) {
 
     const data = await mail.getMailAuth();
 
-    const transport = nodemailer.createTransport({
+    const transport = nodemailer.createTransport(smtpTransport({
         host: data.ip,
         port: data.port,
         auth: {
             user: mailOptions.authUser || `no-reply@${settings.dashboardDomain()}`,
             pass: data.relayToken
         }
-    });
+    }));
 
     const transportSendMail = util.promisify(transport.sendMail.bind(transport));
     const [error] = await safe(transportSendMail(mailOptions));

src/mounts.js

@@ -184,7 +184,7 @@ async function getStatus(mountType, hostPath) {
 
             if (end !== -1) message = lines.slice(start, end+1).map(line => line['MESSAGE']).join('\n');
         }
-        if (!message) message = `Could not determine mount failure reason. ${safe.error ? safe.error.message : ''}`;
+        if (!message) message = `Could not determine failure reason. ${safe.error ? safe.error.message : ''}`;
     } else {
         message = 'Mounted';
     }
@@ -196,7 +196,7 @@ async function tryAddMount(mount, options) {
     assert.strictEqual(typeof mount, 'object'); // { name, hostPath, mountType, mountOptions }
     assert.strictEqual(typeof options, 'object'); // { timeout, skipCleanup }
 
-    if (mount.mountType === 'mountpoint' || mount.mountType === 'filesystem') return;
+    if (mount.mountType === 'mountpoint') return;
 
     if (constants.TEST) return;
 
@@ -215,7 +215,7 @@ async function tryAddMount(mount, options) {
 async function remount(mount) {
     assert.strictEqual(typeof mount, 'object'); // { name, hostPath, mountType, mountOptions }
 
-    if (mount.mountType === 'mountpoint' || mount.mountType === 'filesystem') return;
+    if (mount.mountType === 'mountpoint') return;
 
     if (constants.TEST) return;
 

src/nginxconfig.ejs

@@ -45,7 +45,7 @@ server {
     location / {
 <% if ( endpoint === 'dashboard' || endpoint === 'setup' ) { %>
         return 301 https://$host$request_uri;
-<% } else if ( endpoint === 'app' || endpoint === 'external' ) { %>
+<% } else if ( endpoint === 'app' ) { %>
         return 301 https://$host$request_uri;
 <% } else if ( endpoint === 'redirect' ) { %>
         return 301 https://<%= redirectTo %>$request_uri;
@@ -147,9 +147,7 @@ server {
     proxy_read_timeout       3500;
     proxy_connect_timeout    3250;
 
-<% if ( endpoint !== 'external' ) { %>
     proxy_set_header   Host               $host;
-<% } %>
     proxy_set_header   X-Forwarded-For    $remote_addr;
     proxy_set_header   X-Forwarded-Host   $host;
     proxy_set_header   X-Forwarded-Port   $server_port;
@@ -177,12 +175,6 @@ server {
         proxy_pass http://127.0.0.1:3000;
 <% } else if ( endpoint === 'app' ) { %>
         proxy_pass http://<%= ip %>:<%= port %>;
-<% } else if ( endpoint === 'external' ) { %>
-        # without a variable, nginx will not start if upstream is down or
-        resolver 127.0.0.1 valid=30s;
-        set $upstream <%= upstreamUri %>;
-        proxy_ssl_verify off;
-        proxy_pass $upstream;
 <% } else if ( endpoint === 'redirect' ) { %>
         return 302 https://<%= redirectTo %>$request_uri;
 <% } %>
@@ -249,10 +241,10 @@ server {
         client_max_body_size 0;
     }
 
-    # graphite paths (uncomment block below and visit /graphite-web/)
+    # graphite paths (uncomment block below and visit /graphite-web/dashboard)
     # remember to comment out the CSP policy as well to access the graphite dashboard
     # location ~ ^/graphite-web/ {
-    #     proxy_pass http://172.18.0.6:8000;
+    #     proxy_pass   http://127.0.0.1:8417;
     #     client_max_body_size 1m;
     # }
 
@@ -334,14 +326,6 @@ server {
         # to clear a permanent redirect on the browser
         return 302 https://<%= redirectTo %>$request_uri;
     }
-<% } else if ( endpoint === 'external' ) { %>
-    location / {
-        # without a variable, nginx will not start if upstream is down or unavailable
-        resolver 127.0.0.1 valid=30s;
-        set $upstream <%= upstreamUri %>;
-        proxy_ssl_verify off;
-        proxy_pass $upstream;
-    }
 <% } else if ( endpoint === 'ip' ) { %>
     location /notfound.html {
         root <%= sourceDir %>/dashboard/dist;

src/notifications.js

@@ -14,7 +14,6 @@ exports = module.exports = {
     ALERT_REBOOT: 'reboot',
     ALERT_BOX_UPDATE: 'boxUpdate',
     ALERT_UPDATE_UBUNTU: 'ubuntuUpdate',
-    ALERT_MANUAL_APP_UPDATE: 'manualAppUpdate',
 
     alert,
 
@@ -190,16 +189,16 @@ async function boxUpdateError(eventId, errorMessage) {
     await add(eventId, 'Cloudron update failed', `Failed to update Cloudron: ${errorMessage}.`);
 }
 
-async function certificateRenewalError(eventId, fqdn, errorMessage) {
+async function certificateRenewalError(eventId, vhost, errorMessage) {
     assert.strictEqual(typeof eventId, 'string');
-    assert.strictEqual(typeof fqdn, 'string');
+    assert.strictEqual(typeof vhost, 'string');
     assert.strictEqual(typeof errorMessage, 'string');
 
-    await add(eventId, `Certificate renewal of ${fqdn} failed`, `Failed to renew certs of ${fqdn}: ${errorMessage}. Renewal will be retried in 12 hours.`);
+    await add(eventId, `Certificate renewal of ${vhost} failed`, `Failed to renew certs of ${vhost}: ${errorMessage}. Renewal will be retried in 12 hours.`);
 
     const admins = await users.getAdmins();
     for (const admin of admins) {
-        await mailer.certificateRenewalError(admin.email, fqdn, errorMessage);
+        await mailer.certificateRenewalError(admin.email, vhost, errorMessage);
     }
 }
 

src/once.js

@@ -2,15 +2,10 @@
 
 exports = module.exports = once;
 
-const debug = require('debug')('box:once');
-
 // https://github.com/isaacs/once/blob/main/LICENSE (ISC)
 function once (fn) {
     const f = function () {
-        if (f.called) {
-            debug(`${f.name} was already called, returning previous return value`);
-            return f.value;
-        }
+        if (f.called) return f.value;
         f.called = true;
         return f.value = fn.apply(this, arguments);
     };

src/paths.js

@@ -16,7 +16,6 @@ exports = module.exports = {
 
     CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
     INFRA_VERSION_FILE: path.join(baseDir(), 'platformdata/INFRA_VERSION'),
-    CRON_SEED_FILE: path.join(baseDir(), 'platformdata/CRON_SEED'),
     DASHBOARD_DIR: constants.TEST ? path.join(__dirname, '../../dashboard/src') : path.join(baseDir(), 'box/dashboard/dist'),
 
     PROVIDER_FILE: '/etc/cloudron/PROVIDER',
@@ -31,6 +30,7 @@ exports = module.exports = {
     ACME_CHALLENGES_DIR: path.join(baseDir(), 'platformdata/acme'),
     ADDON_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons'),
     MAIL_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons/mail'),
+    COLLECTD_APPCONFIG_DIR: path.join(baseDir(), 'platformdata/collectd/collectd.conf.d'),
     LOGROTATE_CONFIG_DIR: path.join(baseDir(), 'platformdata/logrotate.d'),
     NGINX_CONFIG_DIR: path.join(baseDir(), 'platformdata/nginx'),
     NGINX_APPCONFIG_DIR: path.join(baseDir(), 'platformdata/nginx/applications'),
@@ -38,7 +38,6 @@ exports = module.exports = {
     BACKUP_INFO_DIR: path.join(baseDir(), 'platformdata/backup'),
     UPDATE_DIR: path.join(baseDir(), 'platformdata/update'),
     UPDATE_CHECKER_FILE: path.join(baseDir(), 'platformdata/update/updatechecker.json'),
-    DISK_USAGE_FILE: path.join(baseDir(), 'platformdata/diskusage.json'),
     SNAPSHOT_INFO_FILE: path.join(baseDir(), 'platformdata/backup/snapshot-info.json'),
     DYNDNS_INFO_FILE: path.join(baseDir(), 'platformdata/dyndns-info.json'),
     DHPARAMS_FILE: path.join(baseDir(), 'platformdata/dhparams.pem'),
@@ -51,7 +50,6 @@ exports = module.exports = {
     SFTP_PRIVATE_KEY_FILE: path.join(baseDir(), 'platformdata/sftp/ssh/ssh_host_rsa_key'),
     FIREWALL_BLOCKLIST_FILE: path.join(baseDir(), 'platformdata/firewall/blocklist.txt'),
     LDAP_ALLOWLIST_FILE: path.join(baseDir(), 'platformdata/firewall/ldap_allowlist.txt'),
-    REVERSE_PROXY_REBUILD_FILE: path.join(baseDir(), 'platformdata/nginx/rebuild-needed'),
 
     BOX_DATA_DIR: path.join(baseDir(), 'boxdata/box'),
     MAIL_DATA_DIR: path.join(baseDir(), 'boxdata/mail'),

src/platform.js

@@ -90,7 +90,28 @@ async function onPlatformReady(infraChanged) {
 
 async function pruneInfraImages() {
     debug('pruneInfraImages: checking existing images');
-    await shell.promises.exec('pruneInfraImages', 'docker image prune -a --force');
+
+    // cannot blindly remove all unused images since redis image may not be used
+    const images = infra.baseImages.concat(Object.keys(infra.images).map(function (addon) { return infra.images[addon]; }));
+
+    for (const image of images) {
+        let output = safe.child_process.execSync(`docker images --digests ${image.repo} --format "{{.ID}} {{.Repository}}:{{.Tag}}@{{.Digest}}"`, { encoding: 'utf8' });
+        if (output === null) {
+            debug(`Failed to list images of ${image}`, safe.error);
+            throw safe.error;
+        }
+
+        let lines = output.trim().split('\n');
+        for (let line of lines) {
+            if (!line) continue;
+            let parts = line.split(' '); // [ ID, Repo:Tag@Digest ]
+            if (image.tag === parts[1]) continue; // keep
+            debug(`pruneInfraImages: removing unused image of ${image.repo}: tag: ${parts[1]} id: ${parts[0]}`);
+
+            let result = safe.child_process.execSync(`docker rmi ${parts[0]}`, { encoding: 'utf8' });
+            if (result === null) debug(`Error removing image ${parts[0]}: ${safe.error.mesage}`);
+        }
+    }
 }
 
 async function createDockerNetwork() {

src/progress-stream.js

@@ -1,44 +0,0 @@
-'use strict';
-
-const stream = require('stream'),
-    TransformStream = stream.Transform;
-
-class ProgressStream extends TransformStream {
-    constructor(options) {
-        super();
-        this._options = Object.assign({ interval: 10 * 1000 }, options);
-        this._transferred = 0;
-        this._delta = 0;
-        this._started = false;
-        this._startTime = null;
-        this._interval = null;
-    }
-
-    _start() {
-        this._startTime = Date.now();
-        this._started = true;
-        this._interval = setInterval(() => {
-            const speed = this._delta * 1000 / this._options.interval;
-            this._delta = 0;
-            this.emit('progress', { speed, transferred: this._transferred });
-        }, this._options.interval);
-    }
-
-    _stop() {
-        clearInterval(this._interval);
-    }
-
-    _transform(chunk, encoding, callback) {
-        if (!this._started) this._start();
-        this._transferred += chunk.length;
-        this._delta += chunk.length;
-        callback(null, chunk);
-    }
-
-    _flush(callback) {
-        this._stop();
-        callback(null);
-    }
-}
-
-exports = module.exports = ProgressStream;

src/provision.js

@@ -70,7 +70,7 @@ async function setupTask(domain, auditSource) {
     assert.strictEqual(typeof auditSource, 'object');
 
     try {
-        await cloudron.setupDnsAndCert(constants.DASHBOARD_SUBDOMAIN, domain, auditSource, (progress) => setProgress('setup', progress.message));
+        await cloudron.setupDnsAndCert(constants.DASHBOARD_LOCATION, domain, auditSource, (progress) => setProgress('setup', progress.message));
         await ensureDhparams();
         await cloudron.setDashboardDomain(domain, auditSource);
         setProgress('setup', 'Done'),
@@ -111,7 +111,7 @@ async function setup(domainConfig, sysinfoConfig, auditSource) {
             dkimSelector: 'cloudron'
         };
 
-        await settings.setMailLocation(domain, `${constants.DASHBOARD_SUBDOMAIN}.${domain}`); // default mail location. do this before we add the domain for upserting mail DNS
+        await settings.setMailLocation(domain, `${constants.DASHBOARD_LOCATION}.${domain}`); // default mail location. do this before we add the domain for upserting mail DNS
         await domains.add(domain, data, auditSource);
         await settings.setSysinfoConfig(sysinfoConfig);
 
@@ -174,7 +174,7 @@ async function restoreTask(backupConfig, remotePath, sysinfoConfig, options, aud
         await reverseProxy.restoreFallbackCertificates();
 
         const dashboardDomain = settings.dashboardDomain(); // load this fresh from after the backup.restore
-        if (!options.skipDnsSetup) await cloudron.setupDnsAndCert(constants.DASHBOARD_SUBDOMAIN, dashboardDomain, auditSource, (progress) => setProgress('restore', progress.message));
+        if (!options.skipDnsSetup) await cloudron.setupDnsAndCert(constants.DASHBOARD_LOCATION, dashboardDomain, auditSource, (progress) => setProgress('restore', progress.message));
         await cloudron.setDashboardDomain(dashboardDomain, auditSource);
         await settings.setBackupCredentials(backupConfig); // update just the credentials and not the policy and flags
         await eventlog.add(eventlog.ACTION_RESTORE, auditSource, { remotePath });

src/proxyauth.js

@@ -50,23 +50,15 @@ function jwtVerify(req, res, next) {
     });
 }
 
-async function authorizationHeader(req, res, next) {
+async function basicAuthVerify(req, res, next) {
     const appId = req.headers['x-app-id'] || '';
-    if (!appId) return next();
-
-    if (!req.headers.authorization) return next();
+    const credentials = basicAuth(req);
+    if (!appId || !credentials) return next();
 
     const [error, app] = await safe(apps.get(appId));
     if (error) return next(new HttpError(503, error.message));
-    if (!app) return next(new HttpError(503, 'Error getting app'));
 
-    // only if the app supports bearer auth, pass it through to the app. without this flag, anyone can access the app with Bearer auth!
-    if (req.headers.authorization.startsWith('Bearer ') && app.manifest.addons.proxyAuth.supportsBearerAuth) return next(new HttpSuccess(200, {}));
-
-    const credentials = basicAuth(req);
-    if (!credentials) return next();
-
-    if (!app.manifest.addons.proxyAuth.basicAuth) return next(); // this is a flag because this allows auth to bypass 2FA
+    if (!app.manifest.addons.proxyAuth.basicAuth) return next();
 
     const verifyFunc = credentials.name.indexOf('@') !== -1 ? users.verifyWithEmail : users.verifyWithUsername;
     const [verifyError, user] = await safe(verifyFunc(credentials.name, credentials.pass, appId));
@@ -147,7 +139,7 @@ function auth(req, res, next) {
     res.set('x-remote-email', req.user.email);
     res.set('x-remote-name', req.user.displayName);
 
-    next(new HttpSuccess(200, {}));
+    return next(new HttpSuccess(200, {}));
 }
 
 // endpoint called by login page, username and password posted as JSON body
@@ -251,7 +243,7 @@ function initializeAuthwallExpressSync() {
         .use(middleware.lastMile());
 
     router.get ('/login',     loginPage);
-    router.get ('/auth',      jwtVerify, authorizationHeader, auth); // called by nginx before accessing protected page
+    router.get ('/auth',      jwtVerify, basicAuthVerify, auth); // called by nginx before accessing protected page
     router.post('/login',     json, passwordAuth, authorize);
     router.get ('/logout',    logoutPage);
     router.post('/logout',    json, logoutPage);

src/routes/accesscontrol.js

@@ -8,8 +8,8 @@ exports = module.exports = {
     authorizeOperator,
 };
 
-const apps = require('../apps.js'),
-    tokens = require('../tokens.js'),
+const accesscontrol = require('../accesscontrol.js'),
+    apps = require('../apps.js'),
     assert = require('assert'),
     BoxError = require('../boxerror.js'),
     externalLdap = require('../externalldap.js'),
@@ -43,13 +43,8 @@ async function passwordAuth(req, res, next) {
     if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
         if (!totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
 
-        if (user.source === 'ldap') {
-            const [error] = await safe(externalLdap.verifyPasswordAndTotpToken(user, password, totpToken));
-            if (error) return next(new HttpError(401, 'Invalid totpToken'));
-        } else {
-            const verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: totpToken, window: 2 });
-            if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
-        }
+        const verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: totpToken, window: 2 });
+        if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
     }
 
     req.user = user;
@@ -58,32 +53,27 @@ async function passwordAuth(req, res, next) {
 }
 
 async function tokenAuth(req, res, next) {
-    let accessToken;
+    let token;
 
     // this determines the priority
-    if (req.body && req.body.access_token) accessToken = req.body.access_token;
-    if (req.query && req.query.access_token) accessToken = req.query.access_token;
+    if (req.body && req.body.access_token) token = req.body.access_token;
+    if (req.query && req.query.access_token) token = req.query.access_token;
     if (req.headers && req.headers.authorization) {
         const parts = req.headers.authorization.split(' ');
         if (parts.length == 2) {
             const [scheme, credentials] = parts;
 
-            if (/^Bearer$/i.test(scheme)) accessToken = credentials;
+            if (/^Bearer$/i.test(scheme)) token = credentials;
         }
     }
 
-    if (!accessToken) return next(new HttpError(401, 'Token required'));
+    if (!token) return next(new HttpError(401, 'Token required'));
 
-    const token = await tokens.getByAccessToken(accessToken);
-    if (!token) return next(new HttpError(401, 'No such token'));
+    const [error, user] = await safe(accesscontrol.verifyToken(token));
+    if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, error.message));
+    if (error) return next(new HttpError(500, error.message));
 
-    const user = await users.get(token.identifier);
-    if (!user) return next(new HttpError(401,'User not found'));
-    if (!user.active) return next(new HttpError(401,'User not active'));
-
-    await safe(tokens.update(token.id, { lastUsedTime: new Date() })); // ignore any error
-
-    req.token = token;
+    req.access_token = token; // used in logout route
     req.user = user;
 
     next();
@@ -94,10 +84,8 @@ function authorize(requiredRole) {
 
     return function (req, res, next) {
         assert.strictEqual(typeof req.user, 'object');
-        assert.strictEqual(typeof req.token, 'object');
 
         if (users.compareRoles(req.user.role, requiredRole) < 0) return next(new HttpError(403, `role '${requiredRole}' is required but user has only '${req.user.role}'`));
-        if (!tokens.hasScope(req.token, req.method, req.path)) return next(new HttpError(403, 'access token does not have this scope'));
 
         next();
     };
@@ -107,9 +95,7 @@ async function authorizeOperator(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
     assert.strictEqual(typeof req.user, 'object');
     assert.strictEqual(typeof req.app, 'object');
-    assert.strictEqual(typeof req.token, 'object');
 
-    if (!tokens.hasScope(req.token, req.method, req.path)) return next(new HttpError(403, 'access token does not have this scope'));
     if (apps.isOperator(req.app, req.user)) return next();
 
     return next(new HttpError(403, 'user is not an operator'));

src/routes/applinks.js

@@ -1,91 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    listByUser,
-    add,
-    get,
-    update,
-    remove,
-    getIcon
-};
-
-const assert = require('assert'),
-    applinks = require('../applinks.js'),
-    BoxError = require('../boxerror.js'),
-    safe = require('safetydance'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess;
-
-async function listByUser(req, res, next) {
-    assert.strictEqual(typeof req.user, 'object');
-
-    const [error, result] = await safe(applinks.listByUser(req.user));
-    if (error) return next(BoxError.toHttpError(error));
-
-    // we have a separate route for this
-    result.forEach(function (a) { delete a.icon; });
-
-    next(new HttpSuccess(200, { applinks: result }));
-}
-
-async function add(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (!req.body.upstreamUri || typeof req.body.upstreamUri !== 'string') return next(new HttpError(400, 'upstreamUri must be a non-empty string'));
-    if ('label' in req.body && typeof req.body.label !== 'string') return next(new HttpError(400, 'label must be a string'));
-    if ('tags' in req.body && !Array.isArray(req.body.tags)) return next(new HttpError(400, 'tags must be an array with strings'));
-    if ('accessRestriction' in req.body && typeof req.body.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction must be an object'));
-
-    const [error] = await safe(applinks.add(req.body));
-    if (error) return next(BoxError.toHttpError(error));
-
-    next(new HttpSuccess(201, {}));
-}
-
-async function get(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    const [error, result] = await safe(applinks.get(req.params.id));
-    if (error) return next(BoxError.toHttpError(error));
-    if (!result) return next(new HttpError(404, 'Applink not found'));
-
-    // we have a separate route for this
-    delete result.icon;
-
-    next(new HttpSuccess(200, result));
-}
-
-async function update(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (!req.body.upstreamUri || typeof req.body.upstreamUri !== 'string') return next(new HttpError(400, 'upstreamUri must be a non-empty string'));
-    if ('label' in req.body && typeof req.body.label !== 'string') return next(new HttpError(400, 'label must be a string'));
-    if ('tags' in req.body && !Array.isArray(req.body.tags)) return next(new HttpError(400, 'tags must be an array with strings'));
-    if ('accessRestriction' in req.body && typeof req.body.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction must be an object'));
-    if ('icon' in req.body && typeof req.body.icon !== 'string') return next(new HttpError(400, 'icon must be a string'));
-
-    const [error] = await safe(applinks.update(req.params.id, req.body));
-    if (error) return next(BoxError.toHttpError(error));
-
-    next(new HttpSuccess(202, {}));
-}
-
-async function remove(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    const [error] = await safe(applinks.remove(req.params.id));
-    if (error) return next(BoxError.toHttpError(error));
-
-    next(new HttpSuccess(204));
-}
-
-async function getIcon(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    const [error, icon] = await safe(applinks.getIcon(req.params.id, { original: req.query.original }));
-    if (error) return next(BoxError.toHttpError(error));
-    if (!icon) return next(new HttpError(404, 'no such icon'));
-
-    res.send(icon);
-}

src/routes/apps.js

@@ -37,7 +37,6 @@ exports = module.exports = {
     setLocation,
     setStorage,
     setMounts,
-    setUpstreamUri,
 
     stop,
     start,
@@ -56,10 +55,8 @@ exports = module.exports = {
     downloadFile,
 
     updateBackup,
-    downloadBackup,
 
     getLimits,
-    getGraphs,
 
     load
 };
@@ -68,9 +65,7 @@ const apps = require('../apps.js'),
     assert = require('assert'),
     AuditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
     debug = require('debug')('box:routes/apps'),
-    graphs = require('../graphs.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     safe = require('safetydance'),
@@ -179,8 +174,6 @@ async function install(req, res, next) {
     let [error, result] = await safe(apps.downloadManifest(data.appStoreId, data.manifest));
     if (error) return next(BoxError.toHttpError(error));
 
-    if (result.appStoreId === constants.PROXY_APP_APPSTORE_ID && typeof data.upstreamUri !== 'string') return next(new HttpError(400, 'upstreamUri must be a non empty string'));
-
     if (safe.query(result.manifest, 'addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is required to install app with docker addon'));
 
     data.appStoreId = result.appStoreId;
@@ -410,6 +403,7 @@ async function setLocation(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
 
     if (typeof req.body.subdomain !== 'string') return next(new HttpError(400, 'subdomain must be string')); // subdomain may be an empty string
+    if (!req.body.domain) return next(new HttpError(400, 'domain is required'));
     if (typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be string'));
 
     if ('portBindings' in req.body && typeof req.body.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
@@ -498,17 +492,16 @@ async function importApp(req, res, next) {
     const data = req.body;
 
     if ('remotePath' in data) { // if not provided, we import in-place
-        if (typeof data.remotePath !== 'string' || !data.remotePath) return next(new HttpError(400, 'remotePath must be non-empty string'));
+        if (typeof data.remotePath !== 'string') return next(new HttpError(400, 'remotePath must be string'));
         if (typeof data.backupFormat !== 'string') return next(new HttpError(400, 'backupFormat must be string'));
 
         if ('backupConfig' in data && typeof data.backupConfig !== 'object') return next(new HttpError(400, 'backupConfig must be an object'));
 
         const backupConfig = req.body.backupConfig;
 
-        if (backupConfig) {
+        if (req.body.backupConfig) {
             if (typeof backupConfig.provider !== 'string') return next(new HttpError(400, 'provider is required'));
             if ('password' in backupConfig && typeof backupConfig.password !== 'string') return next(new HttpError(400, 'password must be a string'));
-            if ('encryptedFilenames' in backupConfig && typeof backupConfig.encryptedFilenames !== 'boolean') return next(new HttpError(400, 'encryptedFilenames must be a boolean'));
             if ('acceptSelfSignedCerts' in backupConfig && typeof backupConfig.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
 
             // testing backup config can take sometime
@@ -726,11 +719,9 @@ async function createExec(req, res, next) {
     if ('tty' in req.body && typeof req.body.tty !== 'boolean') return next(new HttpError(400, 'tty must be boolean'));
     const tty = !!req.body.tty;
 
-    if ('lang' in req.body && typeof req.body.lang !== 'string') return next(new HttpError(400, 'lang must be a string'));
-
     if (safe.query(req.app, 'manifest.addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is requied to exec app with docker addon'));
 
-    const [error, id] = await safe(apps.createExec(req.app, { cmd, tty, lang: req.body.lang }));
+    const [error, id] = await safe(apps.createExec(req.app, { cmd, tty }));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(200, { id }));
@@ -853,17 +844,6 @@ async function updateBackup(req, res, next) {
     next(new HttpSuccess(200, {}));
 }
 
-async function downloadBackup(req, res, next) {
-    assert.strictEqual(typeof req.app, 'object');
-    assert.strictEqual(typeof req.params.backupId, 'string');
-
-    const [error, result] = await safe(apps.getBackupDownloadStream(req.app, req.params.backupId));
-    if (error) return next(BoxError.toHttpError(error));
-
-    res.attachment(`${req.params.backupId}.tgz`);
-    result.pipe(res);
-}
-
 async function uploadFile(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
 
@@ -918,19 +898,6 @@ async function setMounts(req, res, next) {
     next(new HttpSuccess(202, { taskId: result.taskId }));
 }
 
-async function setUpstreamUri(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.app, 'object');
-
-    if (req.app.appStoreId !== constants.PROXY_APP_APPSTORE_ID) return next(new HttpError(400, 'upstreamUri can only be set for proxy app'));
-    if (typeof req.body.upstreamUri !== 'string') return next(new HttpError(400, 'upstreamUri must be a string'));
-
-    const [error] = await safe(apps.setUpstreamUri(req.app, req.body.upstreamUri, AuditSource.fromRequest(req)));
-    if (error) return next(BoxError.toHttpError(error));
-
-    next(new HttpSuccess(200, {}));
-}
-
 async function listEventlog(req, res, next) {
     assert.strictEqual(typeof req.app, 'object');
 
@@ -974,16 +941,3 @@ async function getLimits(req, res, next) {
 
     next(new HttpSuccess(200, { limits }));
 }
-
-async function getGraphs(req, res, next) {
-    assert.strictEqual(typeof req.app, 'object');
-
-    if (!req.query.fromMinutes || !parseInt(req.query.fromMinutes)) return next(new HttpError(400, 'fromMinutes must be a number'));
-
-    const fromMinutes = parseInt(req.query.fromMinutes);
-    const noNullPoints = !!req.query.noNullPoints;
-    const [error, result] = await safe(graphs.getContainerStats(req.app.id, fromMinutes, noNullPoints));
-    if (error) return next(new HttpError(500, error));
-
-    next(new HttpSuccess(200, result));
-}

src/routes/appstore.js

@@ -20,9 +20,7 @@ const appstore = require('../appstore.js'),
     _ = require('underscore');
 
 async function getApps(req, res, next) {
-    const repository = req.query.repository || 'core';
-
-    const [error, apps] = await safe(appstore.getApps(repository));
+    const [error, apps] = await safe(appstore.getApps());
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(200, { apps }));

src/routes/cloudron.js

@@ -10,8 +10,6 @@ exports = module.exports = {
     isRebootRequired,
     getConfig,
     getDisks,
-    getDiskUsage,
-    updateDiskUsage,
     getMemory,
     getUpdateInfo,
     update,
@@ -25,8 +23,7 @@ exports = module.exports = {
     getServerIpv6,
     getLanguages,
     syncExternalLdap,
-    syncDnsRecords,
-    getSystemGraphs
+    syncDnsRecords
 };
 
 const assert = require('assert'),
@@ -37,7 +34,6 @@ const assert = require('assert'),
     debug = require('debug')('box:routes/cloudron'),
     eventlog = require('../eventlog.js'),
     externalLdap = require('../externalldap.js'),
-    graphs = require('../graphs.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     safe = require('safetydance'),
@@ -67,7 +63,7 @@ async function login(req, res, next) {
     [error, token] = await safe(tokens.add({ clientId: type, identifier: req.user.id, expires: Date.now() + constants.DEFAULT_TOKEN_EXPIRATION_MSECS }));
     if (error) return next(new HttpError(500, error));
 
-    await eventlog.add(req.user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+    await eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
     if (!req.user.ghost) safe(users.notifyLoginLocation(req.user, ip, userAgent, auditSource), { debug });
 
@@ -75,11 +71,11 @@ async function login(req, res, next) {
 }
 
 async function logout(req, res) {
-    assert.strictEqual(typeof req.token, 'object');
+    assert.strictEqual(typeof req.access_token, 'string');
 
     await eventlog.add(eventlog.ACTION_USER_LOGOUT, AuditSource.fromRequest(req), { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
-    await safe(tokens.delByAccessToken(req.token.accessToken));
+    await safe(tokens.delByAccessToken(req.access_token));
     res.redirect('/login.html');
 }
 
@@ -87,7 +83,7 @@ async function passwordResetRequest(req, res, next) {
     if (!req.body.identifier || typeof req.body.identifier !== 'string') return next(new HttpError(401, 'A identifier must be non-empty string'));
 
     const [error] = await safe(users.sendPasswordResetByIdentifier(req.body.identifier, AuditSource.fromRequest(req)));
-    if (error && !(error.reason === BoxError.NOT_FOUND || error.reason === BoxError.CONFLICT)) return next(BoxError.toHttpError(error));
+    if (error && error.reason !== BoxError.NOT_FOUND) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(202, {}));
 }
@@ -167,27 +163,10 @@ async function getConfig(req, res, next) {
 }
 
 async function getDisks(req, res, next) {
-    const [getDisksError, disks] = await safe(system.getDisks());
-    if (getDisksError) return next(BoxError.toHttpError(getDisksError));
-
-    let [getSwapsError, swaps] = await safe(system.getSwaps());
-    if (getSwapsError) return next(BoxError.toHttpError(getSwapsError));
-
-    next(new HttpSuccess(200, { disks, swaps }));
-}
-
-async function getDiskUsage(req, res, next) {
-    const [error, result] = await safe(system.getDiskUsage());
+    const [error, result] = await safe(system.getDisks());
     if (error) return next(BoxError.toHttpError(error));
 
-    next(new HttpSuccess(200, { usage: result }));
-}
-
-async function updateDiskUsage(req, res, next) {
-    const [error, taskId] = await safe(cloudron.updateDiskUsage());
-    if (error) return next(BoxError.toHttpError(error));
-
-    next(new HttpSuccess(201, { taskId }));
+    next(new HttpSuccess(200, result));
 }
 
 async function getMemory(req, res, next) {
@@ -275,7 +254,7 @@ async function getLogStream(req, res, next) {
     res.on('close', logStream.close);
     logStream.on('data', function (data) {
         const obj = JSON.parse(data);
-        res.write(sse(obj.realtimeTimestamp, JSON.stringify(obj))); // send timestamp as id
+        res.write(sse(obj.monotonicTimestamp, JSON.stringify(obj))); // send timestamp as id
     });
     logStream.on('end', res.end.bind(res));
     logStream.on('error', res.end.bind(res, null));
@@ -300,9 +279,9 @@ async function prepareDashboardDomain(req, res, next) {
 }
 
 async function renewCerts(req, res, next) {
-    if ('rebuild' in req.body && typeof req.body.rebuild !== 'boolean') return next(new HttpError(400, 'rebuild must be a boolean'));
+    if ('domain' in req.body && typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
 
-    const [error, taskId] = await safe(cloudron.renewCerts(req.body, AuditSource.fromRequest(req)));
+    const [error, taskId] = await safe(cloudron.renewCerts({ domain: req.body.domain || null }, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
     next(new HttpSuccess(202, { taskId }));
@@ -348,14 +327,3 @@ async function syncDnsRecords(req, res, next) {
 
     next(new HttpSuccess(201, { taskId }));
 }
-
-async function getSystemGraphs(req, res, next) {
-    if (!req.query.fromMinutes || !parseInt(req.query.fromMinutes)) return next(new HttpError(400, 'fromMinutes must be a number'));
-
-    const fromMinutes = parseInt(req.query.fromMinutes);
-    const noNullPoints = !!req.query.noNullPoints;
-    const [error, result] = await safe(graphs.getSystem(fromMinutes, noNullPoints));
-    if (error) return next(new HttpError(500, error));
-
-    next(new HttpSuccess(200, result));
-}

src/routes/graphs.js

@@ -0,0 +1,38 @@
+'use strict';
+
+exports = module.exports = {
+    getGraphs
+};
+
+const middleware = require('../middleware/index.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    url = require('url');
+
+// for testing locally: curl 'http://127.0.0.1:8417/graphite-web/render?format=json&from=-1min&target=absolute(collectd.localhost.du-docker.capacity-usage)'
+// the datapoint is (value, timestamp) https://buildmedia.readthedocs.org/media/pdf/graphite/0.9.16/graphite.pdf
+const graphiteProxy = middleware.proxy(url.parse('http://127.0.0.1:8417'));
+
+function getGraphs(req, res, next) {
+    const parsedUrl = url.parse(req.url, true /* parseQueryString */);
+    delete parsedUrl.query['access_token'];
+    delete req.headers['authorization'];
+    delete req.headers['cookies'];
+
+    // 'graphite-web' is the URL_PREFIX in docker-graphite
+    req.url = url.format({ pathname: 'graphite-web/render', query: parsedUrl.query });
+
+    // graphs may take very long to respond so we run into headers already sent issues quite often
+    // nginx still has a request timeout which can deal with this then.
+    req.clearTimeout();
+
+    graphiteProxy(req, res, function (error) {
+        if (!error) return next();
+
+        if (error.code === 'ECONNREFUSED') return next(new HttpError(424, 'Unable to connect to graphite'));
+        // ECONNRESET here is most likely because of a bug in the query or the uwsgi buffer size is too small
+        if (error.code === 'ECONNRESET') return next(new HttpError(424, 'Unable to query graphite'));
+
+        next(new HttpError(500, error));
+    });
+}
+

src/routes/groups.js

@@ -6,7 +6,7 @@ exports = module.exports = {
     add,
     update,
     remove,
-    setMembers
+    updateMembers
 };
 
 const assert = require('assert'),
@@ -49,7 +49,7 @@ async function update(req, res, next) {
     next(new HttpSuccess(200, { }));
 }
 
-async function setMembers(req, res, next) {
+async function updateMembers(req, res, next) {
     assert.strictEqual(typeof req.params.groupId, 'string');
 
     if (!req.body.userIds) return next(new HttpError(404, 'missing or invalid userIds fields'));

src/routes/index.js

@@ -4,7 +4,6 @@ exports = module.exports = {
     accesscontrol: require('./accesscontrol.js'),
     appPasswords: require('./apppasswords.js'),
     apps: require('./apps.js'),
-    applinks: require('./applinks.js'),
     appstore: require('./appstore.js'),
     backups: require('./backups.js'),
     branding: require('./branding.js'),
@@ -12,6 +11,7 @@ exports = module.exports = {
     domains: require('./domains.js'),
     eventlog: require('./eventlog.js'),
     filemanager: require('./filemanager.js'),
+    graphs: require('./graphs.js'),
     groups: require('./groups.js'),
     mail: require('./mail.js'),
     mailserver: require('./mailserver.js'),

src/routes/mail.js

@@ -177,11 +177,6 @@ async function addMailbox(req, res, next) {
     if (typeof req.body.ownerType !== 'string') return next(new HttpError(400, 'ownerType must be a string'));
     if (typeof req.body.active !== 'boolean') return next(new HttpError(400, 'active must be a boolean'));
 
-    if (!Number.isInteger(req.body.storageQuota)) return next(new HttpError(400, 'storageQuota must be an integer'));
-    if (req.body.storageQuota < 0) return next(new HttpError(400, 'storageQuota must be a postive integer or zero'));
-    if (!Number.isInteger(req.body.messagesQuota)) return next(new HttpError(400, 'messagesQuota must be an integer'));
-    if (req.body.messagesQuota < 0) return next(new HttpError(400, 'messagesQuota must be a positive integer or zero'));
-
     const [error] = await safe(mail.addMailbox(req.body.name, req.params.domain, req.body, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));
 
@@ -197,11 +192,6 @@ async function updateMailbox(req, res, next) {
     if (typeof req.body.active !== 'boolean') return next(new HttpError(400, 'active must be a boolean'));
     if (typeof req.body.enablePop3 !== 'boolean') return next(new HttpError(400, 'enablePop3 must be a boolean'));
 
-    if (!Number.isInteger(req.body.storageQuota)) return next(new HttpError(400, 'storageQuota must be an integer'));
-    if (req.body.storageQuota < 0) return next(new HttpError(400, 'storageQuota must be a postive integer or zero'));
-    if (!Number.isInteger(req.body.messagesQuota)) return next(new HttpError(400, 'messagesQuota must be an integer'));
-    if (req.body.messagesQuota < 0) return next(new HttpError(400, 'messagesQuota must be a positive integer or zero'));
-
     const [error] = await safe(mail.updateMailbox(req.params.name, req.params.domain, req.body, AuditSource.fromRequest(req)));
     if (error) return next(BoxError.toHttpError(error));