total stats are reported directly as single value

Clean stats api response to have specific response types
Also collect total I/O stats for the containers
2022-10-14 12:00:24 +02:00 · 2022-10-14 11:25:43 +02:00 · 2022-10-14 11:15:52 +02:00 · 2022-10-14 10:43:30 +02:00 · 2022-10-14 08:22:04 +02:00 · 2022-10-13 23:35:01 +02:00
294 changed files with 37250 additions and 29779 deletions
@@ -5,7 +5,7 @@
    },
    "extends": "eslint:recommended",
    "parserOptions": {
-        "ecmaVersion": 8
+        "ecmaVersion": 2020
    },
    "rules": {
        "indent": [
@@ -1,5 +1,6 @@
 node_modules/
 coverage/
+.nyc_output/
 webadmin/dist/
 installer/src/certs/server.key

@@ -1,5 +1,5 @@
 {
    "node": true,
    "unused": true,
-    "esversion": 8
+    "esversion": 11
 }
@@ -2286,3 +2286,266 @@
 [6.3.1]
 * Fix cert migration issues

+[6.3.2]
+* Avatar was migrated as base64 instead of binary
+* Fix issue where filemanager came up empty for CIFS mounts
+
+[6.3.3]
+* volumes: add filesystem volume type for shared folders
+* mail: enable sieve extension editheader
+* mail: update solr to 8.9.0
+
+[6.3.4]
+* Fix issue where old nginx configs where not removed before upgrade
+
+[6.3.5]
+* Fix permission issues with sshfs
+* filemanager: reset selection if directory has changed
+* branding: fix error highlight with empty cloudron name
+* better text instead of "Cloudron in the wild"
+* Make sso login hint translatable
+* Give unread notifications a small left border
+* Fix issue where clicking update indicator opened app in new tab
+* Ensure notifications are only fetched and shown for at least admins
+* setupaccount: Show input field errors below input field
+* Set focus automatically for new alias or redirect
+* eventlog: fix issue where old events are not periodically removed
+* ssfs: fix chown
+
+[6.3.6]
+* Fix broken reboot button
+* app updated notification shown despite failure
+* Update translation for sso login information
+* Hide groups/tags/state filter in app listing for normal users
+* filemanager: Ensure breadcrumbs and hash are correctly updated on folder navigation
+* cloudron-setup: check if nginx/docker is already installed
+* Use the addresses of all available interfaces for port 53 binding
+* refresh config on appstore login
+* password reset: check 2fa when enabled
+
+[7.0.0]
+* Ubuntu 16 is not supported anymore
+* Do not use Gravatar as the default but only an option
+* redis: suppress password warning
+* setup UI: fix dark mode
+* wellknown: response to .wellknown/matrix/client
+* purpose field is not required anymore during appstore signup
+* sftp: fix symlink deletion
+* Show correct/new app version info in updated finished notification
+* Make new login email translatable
+* Hide ticket form if cloudron.io mail is not verified
+* Refactor code to use async/await
+* postgresql: bump shm size and disable parallel queries
+* update nodejs to 14.17.6
+* external ldap: If we detect a local user with the same username as found on LDAP/AD we map it
+* add basic eventlog for apps in app view
+* Enable sshfs/cifs/nfs in app import UI
+* Require password for fallback email change
+* Make password reset logic translatable
+* support: only verified email address can open support tickets
+* Logout users without 2FA when mandatory 2fa is enabled
+* notifications: better oom message for redis
+* Add way to impersonate users for presetup
+* mail: open up port 465 for mail submission (TLS)
+* Implement operator role for apps
+* sftp: normal users do not have SFTP access anymore. Use operator role instead
+* eventlog: add service rebuild/restart/configure events
+* upcloud: add object storage integration
+* Each app can now have a custom crontab
+* services: add recovery mode
+* postgresql: fix restore issue with long table names
+* recvmail: make the addon work again
+* mail: update solr to 8.10.0
+* mail: POP3 support
+* update docker to 20.10.7
+* volumes: add remount button
+* mail: add spam eventlog filter type
+* mail: configure dnsbl
+* mail: add duplication detection for lists
+* mail: add SRS for Sieve Forwarding
+
+[7.0.1]
+* Fix matrix wellKnown client migration
+
+[7.0.2]
+* mail: POP3 flag was not returned correctly
+* external ldap: fix crash preventing users from logging in
+* volumes: ensure we don't crash if mount status is unexpected
+* backups: set default backup memory limit to 800
+* users: allow admins to specify password recovery email
+* retry startup tasks on database error
+
+[7.0.3]
+* support: fix remoe support not working for 'root' user
+* Fix cog icon on app grid item hover for darkmode
+* Disable password reset and impersonate button for self user instead of hiding them
+* pop3: fix crash with auth of non-existent mailbox
+* mail: fix direction field in eventlog of deferred mails
+* mail: fix eventlog search
+* mail: save message-id in eventlog
+* backups: fix issue which resulted in incomplete backups when an app has backups disabled
+* restore: do not redirect until mail data has been restored
+* proxyauth: set viewport meta tag in login view
+
+[7.0.4]
+* Add password reveal button to login pages
+* appstore: fix crash if account already registered
+* Do not nuke all the logrotate configs on update
+* Remove unused httpPaths from manifest
+* cloudron-support: add option to reset cloudron.io account
+* Fix flicker in login page
+* Fix LE account key re-use issue in DO 1-click image
+* mail: add non-tls ports for recvmail addon
+* backups: fix issue where mail backups where not cleaned up
+* notifications: fix automatic app update notifications
+
+[7.1.0]
+* Add mail manager role
+* mailbox: app can be set as owner when recvmail addon enabled
+* domains: add well known config UI (for jitsi configuration)
+* Prefix email addon variables with CLOUDRON_EMAIL instead of CLOUDRON_MAIL
+* remove support for manifest version 1
+* Add option to enable/disable mailbox sharing
+* base image 3.2.0
+* Update node to 16.13.1
+* mongodb: update to 4.4
+* Add `upstreamVersion` to manifest
+* Add `logPaths` to manifest
+* Add cifs seal support for backup and volume mounts
+* add a way for admins to set username when profiles are locked
+* Add support for secondary domains
+* postgresql: enable postgis
+* remove nginx config of stopped apps
+* mail: use port25check.cloudron.io to check outbound port 25 connectivity
+* Add import/export of mailboxes and users
+* LDAP server can now be exposed
+* Update monaco-editor to 0.32.1
+* Update xterm.js to 4.17.0
+* Update docker to 20.10.12
+* IPv6 support
+
+[7.1.1]
+* Fix issue where dkimKey of a mail domain is sometimes null
+* firewall: add retry for xtables lock
+* redis: fix issue where protected mode was enabled with no password
+
+[7.1.2]
+* Fix crash in cloudron-firewall when ports are whitelisted
+* eventlog: add event for certificate cleanup
+* eventlog: log event for mailbox alias update
+* backups: fix incorrect mountpoint check with managed mounts
+
+[7.1.3]
+* Fix security issue where an admin can impersonate an owner
+* block list: can upload up to 2MB
+* dns: fix issue where link local address was picked up for ipv6
+* setup: ufw may not be installed
+* mysql: fix default collation of databases
+
+[7.1.4]
+* wildcard dns: fix handling of ENODATA
+* cloudflare: fix error handling
+* openvpn: ipv6 support
+* dyndns: fix issue where eventlog was getting filled with empty entries
+* mandatory 2fa: Fix typo in 2FA check
+
+[7.2.0]
+* mail: hide log button for non-superadmins
+* firewall: do not add duplicate ldap redirect rules
+* ldap: respond to RootDSE
+* Check if CNAME record exists and remove it if overwrite is set
+* cifs: use credentials file for better password support
+* installer: rework script to fix DNS resolution issues
+* backup cleaner: do not clean if not mounted
+* restore: fix sftp private key perms
+* support: add a separate system user named cloudron-support
+* sshfs: fix bug where sshfs mounts were generated without unbound dependancy
+* cloudron-setup: add --setup-token
+* notifications: add installation event
+* backups: set label of backup and control it's retention
+* wasabi: add new regions (London, Frankfurt, Paris, Toronto)
+* docker: update to 20.10.14
+* Ensure LDAP usernames are always treated lowercase
+* Add a way to make LDAP users local
+* proxyAuth: set X-Remote-User (rfc3875)
+* GoDaddy: there is now a delete API
+* nginx: use ubuntu packages for ubuntu 20.04 and 22.04
+* Ubuntu 22.04 LTS support
+* Add Hetzner DNS
+* cron: add support for extensions (@reboot, @weekly etc)
+* Add profile backgroundImage api
+* exec: rework API to get exit code
+* Add update available filter
+
+[7.2.1]
+* Refactor backup code to use async/await
+* mongodb: fix bug where a small timeout prevented import of large backups
+* Add update available filter
+* exec: rework API to get exit code
+* Add profile backgroundImage api
+* cron: add support for extensions (@reboot, @weekly etc)
+
+[7.2.2]
+* Update cloudron-manifestformat for new scheduler patterns
+* collectd: FQDNLookup causes collectd install to fail
+
+[7.2.3]
+* appstore: allow re-registration on server side delete
+* transfer ownership route is not used anymore
+* graphite: fix issue where disk names with '.' do not render
+* dark mode fixes
+* sendmail: mail from display name
+* Use volumes for app data instead of raw path
+* initial xfs support
+
+[7.2.4]
+* volumes: Ensure long volume names do not overflow the table
+* Move all appstore filter to the left
+* app data: allow sameness of old and new dir
+
+[7.2.5]
+* Fix storage volume migration
+* Fix issue where only 25 group members were returned
+* Fix eventlog display
+
+[7.3.0]
+* Proxied apps
+* Applinks - app bookmarks in dashboard
+* backups: optional encryption of backup file names
+* eventlog: add event for impersonated user login
+* ldap & user directory: Remove virtual user and admin groups
+* Randomize certificate generation cronjob to lighten load on Let's Encrypt servers
+* mail: catch all address can be any domain
+* mail: accept only STARTTLS servers for relay
+* graphs: cgroup v2 support
+* mail: fix issue where signature was appended to text attachments
+* redis: restart button will now rebuild if the container is missing
+* backups: allow space in label name
+* mail: fix crash when solr is enabled on Ubuntu 22 (cgroup v2 detection fix)
+* mail: fix issue where certificate renewal did not restart the mail container properly
+* notification: Fix crash when backupId is null
+* IPv6: initial support for ipv6 only server
+* User directory: Cloudron connector uses 2FA auth
+* port bindings: add read only flag
+* mail: add storage quota support
+* mail: allow aliases to have wildcard
+* proxyAuth: add supportsBearerAuth flag
+* backups: Fix precondition check which was not erroring if mount is missing
+* mail: add queue management API and UI
+* graphs: show app disk usage graphs
+* UI: fix issue where mailbox display name was not init correctly
+* wasabi: add singapore and sydney regions
+* filemanager: add split view
+* nginx: fix zero length certs when out of disk space
+* read only API tokens
+
+[7.3.1]
+* Add cloudlare R2
+* app proxy: fixes to https proxying
+* app links: fix icons
+
+[7.3.2]
+* support: require owner permissions
+* postgresql: fix issue when restoring large dumps
+* graphs: add cpu/disk/network usage
+* graphs: new disk usage UI
@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2020 Cloudron UG
+Copyright (c) 2022 Cloudron UG

 With regard to the Cloudron Software:

@@ -1,178 +0,0 @@
-#!/bin/bash
-
-set -euv -o pipefail
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-readonly arg_infraversionpath="${SOURCE_DIR}/../src"
-
-function die {
-    echo $1
-    exit 1
-}
-
-export DEBIAN_FRONTEND=noninteractive
-
-readonly ubuntu_codename=$(lsb_release -cs)
-readonly ubuntu_version=$(lsb_release -rs)
-
-# hold grub since updating it breaks on some VPS providers. also, dist-upgrade will trigger it
-apt-mark hold grub* >/dev/null
-apt-get -o Dpkg::Options::="--force-confdef" update -y
-apt-get -o Dpkg::Options::="--force-confdef" upgrade -y
-apt-mark unhold grub* >/dev/null
-
-echo "==> Installing required packages"
-
-debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
-debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
-
-# this enables automatic security upgrades (https://help.ubuntu.com/community/AutomaticSecurityUpdates)
-# resolvconf is needed for unbound to work property after disabling systemd-resolved in 18.04
-
-gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
-mysql_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "mysql-server-8.0" || echo "mysql-server-5.7")
-apt-get -y install --no-install-recommends \
-    acl \
-    apparmor \
-    build-essential \
-    cifs-utils \
-    cron \
-    curl \
-    debconf-utils \
-    dmsetup \
-    $gpg_package \
-    ipset \
-    iptables \
-    libpython2.7 \
-    linux-generic \
-    logrotate \
-    $mysql_package \
-    nfs-common \
-    openssh-server \
-    pwgen \
-    resolvconf \
-    sshfs \
-    swaks \
-    tzdata \
-    unattended-upgrades \
-    unbound \
-    unzip \
-    xfsprogs
-
-echo "==> installing nginx for xenial for TLSv3 support"
-curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
-# apt install with install deps (as opposed to dpkg -i)
-apt install -y /tmp/nginx.deb
-rm /tmp/nginx.deb
-
-# on some providers like scaleway the sudo file is changed and we want to keep the old one
-apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends sudo
-
-# this ensures that unattended upgades are enabled, if it was disabled during ubuntu install time (see #346)
-# debconf-set-selection of unattended-upgrades/enable_auto_updates + dpkg-reconfigure does not work
-cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
-
-echo "==> Installing node.js"
-readonly node_version=14.15.4
-mkdir -p /usr/local/node-${node_version}
-curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-${node_version}
-ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
-ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
-apt-get install -y --no-install-recommends python   # Install python which is required for npm rebuild
-[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
-
-# https://docs.docker.com/engine/installation/linux/ubuntulinux/
-echo "==> Installing Docker"
-
-# create systemd drop-in file. if you channge options here, be sure to fixup installer.sh as well
-mkdir -p /etc/systemd/system/docker.service.d
-echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
-
-# there are 3 packages for docker - containerd, CLI and the daemon
-readonly docker_version=20.10.3
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
-# apt install with install deps (as opposed to dpkg -i)
-apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
-rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
-
-storage_driver=$(docker info | grep "Storage Driver" | sed 's/.*: //')
-if [[ "${storage_driver}" != "overlay2" ]]; then
-    echo "Docker is using "${storage_driver}" instead of overlay2"
-    exit 1
-fi
-
-# do not upgrade grub because it might prompt user and break this script
-echo "==> Enable memory accounting"
-apt-get -y --no-upgrade --no-install-recommends install grub2-common
-sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
-update-grub
-
-echo "==> Downloading docker images"
-if [ ! -f "${arg_infraversionpath}/infra_version.js" ]; then
-    echo "No infra_versions.js found"
-    exit 1
-fi
-
-images=$(node -e "var i = require('${arg_infraversionpath}/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
-
-echo -e "\tPulling docker images: ${images}"
-for image in ${images}; do
-    docker pull "${image}"
-    docker pull "${image%@sha256:*}" # this will tag the image for readability
-done
-
-echo "==> Install collectd"
-# without this, libnotify4 will install gnome-shell
-apt-get install -y libnotify4 --no-install-recommends
-if ! apt-get install -y --no-install-recommends libcurl3-gnutls collectd collectd-utils; then
-    # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
-    echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
-    sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
-fi
-# https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
-[[ "${ubuntu_version}" == "20.04" ]] && echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
-
-# some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
-echo "==> Configuring host"
-sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
-if systemctl is-active ntp; then
-    systemctl stop ntp
-    apt purge -y ntp
-fi
-timedatectl set-ntp 1
-# mysql follows the system timezone
-timedatectl set-timezone UTC
-
-echo "==> Adding sshd configuration warning"
-sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://docs.cloudron.io/security/#securing-ssh-access' -i /etc/ssh/sshd_config
-
-# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
-echo "==> Disabling motd news"
-if [ -f "/etc/default/motd-news" ]; then
-    sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
-fi
-
-# Disable bind for good measure (on online.net, kimsufi servers these are pre-installed)
-systemctl stop bind9 || true
-systemctl disable bind9 || true
-
-# on ovh images dnsmasq seems to run by default
-systemctl stop dnsmasq || true
-systemctl disable dnsmasq || true
-
-# on ssdnodes postfix seems to run by default
-systemctl stop postfix || true
-systemctl disable postfix || true
-
-# on ubuntu 18.04 and 20.04, this is the default. this requires resolvconf for DNS to work further after the disable
-systemctl stop systemd-resolved || true
-systemctl disable systemd-resolved || true
-
-# ubuntu's default config for unbound does not work if ipv6 is disabled. this config is overwritten in start.sh
-# we need unbound to work as this is required for installer.sh to do any DNS requests
-ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
-echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: ${ip6}" > /etc/unbound/unbound.conf.d/cloudron-network.conf
-systemctl restart unbound
@@ -2,65 +2,76 @@

 'use strict';

-let async = require('async'),
-    dockerProxy = require('./src/dockerproxy.js'),
-    fs = require('fs'),
+const fs = require('fs'),
    ldap = require('./src/ldap.js'),
    paths = require('./src/paths.js'),
    proxyAuth = require('./src/proxyauth.js'),
-    server = require('./src/server.js');
+    safe = require('safetydance'),
+    server = require('./src/server.js'),
+    settings = require('./src/settings.js'),
+    directoryServer = require('./src/directoryserver.js');

-const NOOP_CALLBACK = function () { };
+let logFd;

-function setupLogging(callback) {
-    if (process.env.BOX_ENV === 'test') return callback();
+async function setupLogging() {
+    if (process.env.BOX_ENV === 'test') return;

-    const logfileStream = fs.createWriteStream(paths.BOX_LOG_FILE, { flags:'a' });
-    process.stdout.write = process.stderr.write = logfileStream.write.bind(logfileStream);
-
-    callback();
+    logFd = fs.openSync(paths.BOX_LOG_FILE, 'a');
+    // we used to write using a stream before but it caches internally and there is no way to flush it when things crash
+    process.stdout.write = process.stderr.write = function (...args) {
+        const callback = typeof args[args.length-1] === 'function' ? args.pop() : function () {}; // callback is required for fs.write
+        fs.write.apply(fs, [logFd, ...args, callback]);
+    };
 }

-async.series([
-    setupLogging,
-    server.start, // do this first since it also inits the database
-    proxyAuth.start,
-    ldap.start,
-    dockerProxy.start
-], function (error) {
-    if (error) {
-        console.log('Error starting server', error);
-        process.exit(1);
-    }
+// this is also used as the 'uncaughtException' handler which can only have synchronous functions
+function exitSync(status) {
+    if (status.error) fs.write(logFd, status.error.stack + '\n', function () {});
+    fs.fsyncSync(logFd);
+    fs.closeSync(logFd);
+    process.exit(status.code);
+}

-    // require those here so that logging handler is already setup
-    require('supererror');
+async function startServers() {
+    await setupLogging();
+    await server.start(); // do this first since it also inits the database
+    await proxyAuth.start();
+    await ldap.start();
+
+    const conf = await settings.getDirectoryServerConfig();
+    if (conf.enabled) await directoryServer.start();
+}
+
+async function main() {
+    const [error] = await safe(startServers());
+    if (error) return exitSync({ error: new Error(`Error starting server: ${JSON.stringify(error)}`), code: 1 });
+
+    // require this here so that logging handler is already setup
    const debug = require('debug')('box:box');

-    process.on('SIGINT', function () {
+    process.on('SIGINT', async function () {
        debug('Received SIGINT. Shutting down.');

-        proxyAuth.stop(NOOP_CALLBACK);
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
+        await proxyAuth.stop();
+        await server.stop();
+        await directoryServer.stop();
+        await ldap.stop();
        setTimeout(process.exit.bind(process), 3000);
    });

-    process.on('SIGTERM', function () {
+    process.on('SIGTERM', async function () {
        debug('Received SIGTERM. Shutting down.');

-        proxyAuth.stop(NOOP_CALLBACK);
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
+        await proxyAuth.stop();
+        await server.stop();
+        await directoryServer.stop();
+        await ldap.stop();
        setTimeout(process.exit.bind(process), 3000);
    });

-    process.on('uncaughtException', function (error) {
-        console.error((error && error.stack) ? error.stack : error);
-        setTimeout(process.exit.bind(process, 1), 3000);
-    });
+    process.on('uncaughtException', (error) => exitSync({ error, code: 1 }));

    console.log(`Cloudron is up and running. Logs are at ${paths.BOX_LOG_FILE}`); // this goes to journalctl
-});
+}
+
+main();
@@ -2,27 +2,21 @@

 'use strict';

-var database = require('./src/database.js');
+const database = require('./src/database.js');

-var crashNotifier = require('./src/crashnotifier.js');
+const crashNotifier = require('./src/crashnotifier.js');

 // This is triggered by systemd with the crashed unit name as argument
-function main() {
+async function main() {
    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <unitName>');

-    var unitName = process.argv[2];
+    const unitName = process.argv[2];
    console.log('Started crash notifier for', unitName);

    // eventlog api needs the db
-    database.initialize(function (error) {
-        if (error) return console.error('Cannot connect to database. Unable to send crash log.', error);
+    await database.initialize();

-        crashNotifier.sendFailureLogs(unitName, function (error) {
-            if (error) console.error(error);
-
-            process.exit();
-        });
-    });
+    await crashNotifier.sendFailureLogs(unitName);
 }

 main();
@@ -6,7 +6,7 @@ exports.up = function(db, callback) {
    db.runSql('ALTER TABLE settings ADD COLUMN valueBlob MEDIUMBLOB', function (error) {
        if (error) return callback(error);

-        fs.readFile('/home/yellowtent/boxdata/avatar.png', { encoding: 'base64' }, function (error, avatar) {
+        fs.readFile('/home/yellowtent/boxdata/avatar.png', function (error, avatar) {
            if (error && error.code === 'ENOENT') return callback();
            if (error) return callback(error);

@@ -0,0 +1,26 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM volumes', function (error, volumes) {
+        if (error || volumes.length === 0) return callback(error);
+
+        async.eachSeries(volumes, function (volume, iteratorDone) {
+            if (volume.mountType !== 'noop') return iteratorDone();
+
+            let mountType;
+            if (safe.child_process.execSync(`mountpoint -q -- ${volume.hostPath}`)) {
+                mountType = 'mountpoint';
+            } else {
+                mountType = 'filesystem';
+            }
+            db.runSql('UPDATE volumes SET mountType=? WHERE id=?', [ mountType, volume.id ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,13 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE users SET avatar="gravatar" WHERE avatar IS NULL', function (error) {
+        if (error) return callback(error);
+
+        db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB NOT NULL', callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB', callback);
+};
@@ -0,0 +1,30 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * from domains', [], function (error, results) {
+        if (error) return callback(error);
+
+        async.eachSeries(results, function (r, iteratorDone) {
+            if (!r.wellKnownJson) return iteratorDone();
+
+            const wellKnown = safe.JSON.parse(r.wellKnownJson);
+            if (!wellKnown || !wellKnown['matrix/server']) return iteratorDone();
+            const matrixHostname = JSON.parse(wellKnown['matrix/server'])['m.server'];
+
+            wellKnown['matrix/client'] = JSON.stringify({
+                'm.homeserver': {
+                    'base_url': 'https://' + matrixHostname
+                }
+            });
+
+            db.runSql('UPDATE domains SET wellKnownJson=? WHERE domain=?', [ JSON.stringify(wellKnown), r.domain ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appAddonConfigs MODIFY value TEXT NOT NULL', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appAddonConfigs MODIFY value VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY loginLocationsJson MEDIUMTEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY loginLocationsJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN operatorsJson TEXT', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN operatorsJson', callback);
+};
@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN crontab TEXT', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN crontab', callback);
+};
@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN inviteToken VARCHAR(128) DEFAULT ""', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN inviteToken', callback);
+};
@@ -0,0 +1,19 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN enableInbox BOOLEAN DEFAULT 0'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxName VARCHAR(128)'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxDomain VARCHAR(128)'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN enableInbox'),
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxName'),
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxDomain'),
+    ], callback);
+};
@@ -0,0 +1,35 @@
+'use strict';
+
+const async = require('async'),
+    reverseProxy = require('../src/reverseproxy.js'),
+    safe = require('safetydance');
+
+const NGINX_CERT_DIR = '/home/yellowtent/platformdata/nginx/cert';
+
+// ensure fallbackCertificate of domains are present in database and the cert dir. it seems a bad migration lost them.
+// https://forum.cloudron.io/topic/5683/data-argument-must-be-of-type-received-null-error-during-restore-process
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM domains', [ ], function (error, domains) {
+        if (error) return callback(error);
+
+        // this code is br0ken since async 3.x since async functions won't get iteratorDone anymore
+        // no point fixing this migration though since it won't run again in old cloudrons. and in new cloudron domains will be empty
+        async.eachSeries(domains, async function (domain, iteratorDone) {
+            let fallbackCertificate = safe.JSON.parse(domain.fallbackCertificateJson);
+            if (!fallbackCertificate || !fallbackCertificate.cert || !fallbackCertificate.key) {
+                let error;
+                [error, fallbackCertificate] = await safe(reverseProxy.generateFallbackCertificate(domain.domain));
+                if (error) return iteratorDone(error);
+            }
+
+            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.cert`, fallbackCertificate.cert, 'utf8')) return iteratorDone(safe.error);
+            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.key`, fallbackCertificate.key, 'utf8')) return iteratorDone(safe.error);
+
+            db.runSql('UPDATE domains SET fallbackCertificateJson=? WHERE domain=?', [ JSON.stringify(fallbackCertificate), domain.domain ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes ADD COLUMN enablePop3 BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN enablePop3', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
@@ -0,0 +1,44 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    path = require('path'),
+    safe = require('safetydance');
+
+const MAIL_DATA_DIR = '/home/yellowtent/boxdata/mail';
+const DKIM_DIR = `${MAIL_DATA_DIR}/dkim`;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mail ADD COLUMN dkimKeyJson MEDIUMTEXT', function (error) {
+        if (error) return callback(error);
+
+        fs.readdir(DKIM_DIR, function (error, filenames) {
+            if (error && error.code === 'ENOENT') return callback();
+            if (error) return callback(error);
+
+            async.eachSeries(filenames, function (filename, iteratorCallback) {
+                const domain = filename;
+                const publicKey = safe.fs.readFileSync(path.join(DKIM_DIR, domain, 'public'), 'utf8');
+                const privateKey = safe.fs.readFileSync(path.join(DKIM_DIR, domain, 'private'), 'utf8');
+                if (!publicKey || !privateKey) return iteratorCallback();
+
+                const dkimKey = {
+                    publicKey,
+                    privateKey
+                };
+
+                db.runSql('UPDATE mail SET dkimKeyJson=? WHERE domain=?', [ JSON.stringify(dkimKey), domain ], iteratorCallback);
+            }, function (error) {
+                if (error) return callback(error);
+
+                fs.rmdir(DKIM_DIR, { recursive: true }, callback);
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.run(db, 'ALTER TABLE mail DROP COLUMN dkimKeyJson')
+    ], callback);
+};
@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM blobs WHERE id=?', [ 'dhparams' ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,17 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE source sourceJson TEXT', []),
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE data dataJson TEXT', []),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE sourceJson source TEXT', []),
+        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE dataJson data TEXT', []),
+    ], callback);
+};
@@ -0,0 +1,17 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('SELECT value FROM settings WHERE name=?', [ 'sysinfo_config' ], function (error, result) {
+        if (error || result.length === 0) return callback(error);
+        const sysinfoConfig = JSON.parse(result[0].value);
+        if (sysinfoConfig.provider !== 'fixed' || !sysinfoConfig.ip) return callback();
+        sysinfoConfig.ipv4 = sysinfoConfig.ip;
+        delete sysinfoConfig.ip;
+
+        db.runSql('REPLACE INTO settings (name, value) VALUES(?, ?)', [ 'sysinfo_config', JSON.stringify(sysinfoConfig) ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'directory_config', 'profile_config' ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'profile_config', 'directory_config' ], callback);
+};
@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains ADD COLUMN environmentVariable VARCHAR(128)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains DROP COLUMN environmentVariable', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,19 @@
+'use strict';
+
+const safe = require('safetydance');
+
+const PROXY_AUTH_TOKEN_SECRET_FILE = '/home/yellowtent/platformdata/proxy-auth-token-secret';
+
+exports.up = function (db, callback) {
+    const token = safe.fs.readFileSync(PROXY_AUTH_TOKEN_SECRET_FILE);
+    if (!token) return callback();
+    db.runSql('INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'proxy_auth_token_secret', token ], function (error) {
+        if (error) return callback(error);
+        safe.fs.unlinkSync(PROXY_AUTH_TOKEN_SECRET_FILE);
+        callback();
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,12 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('RENAME TABLE subdomains TO locations', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,27 @@
+'use strict';
+
+const async = require('async'),
+    mail = require('../src/mail.js'),
+    safe = require('safetydance'),
+    util = require('util');
+
+// it seems some mail domains do not have dkimKey in the database for some reason because of some previous bad migration
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM mail', [ ], function (error, mailDomains) {
+        if (error) return callback(error);
+
+        async.eachSeries(mailDomains, function (mailDomain, iteratorDone) {
+            let dkimKey = safe.JSON.parse(mailDomain.dkimKeyJson);
+            if (dkimKey && dkimKey.publicKey && dkimKey.privateKey) return iteratorDone();
+            console.log(`${mailDomain.domain} has no dkim key in the database. generating a new one`);
+            util.callbackify(mail.generateDkimKey)(function (error, dkimKey) {
+                if (error) return iteratorDone(error);
+                db.runSql('UPDATE mail SET dkimKeyJson=? WHERE domain=?', [ JSON.stringify(dkimKey), mailDomain.domain ], iteratorDone);
+            });
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM settings WHERE name=?', [ 'license_key' ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'appstore_api_token', 'cloudron_token' ], callback);
+};
+
+exports.down = function(db, callback) {
+  callback();
+};
@@ -0,0 +1,37 @@
+'use strict';
+
+const superagent = require('superagent');
+
+exports.up = function(db, callback) {
+    db.all('SELECT value FROM settings WHERE name="api_server_origin"', function (error, results) {
+        if (error || results.length === 0) return callback(error);
+        const apiServerOrigin = results[0].value;
+
+        db.all('SELECT value FROM settings WHERE name="appstore_api_token"', function (error, results) {
+            if (error || results.length === 0) return callback(error);
+            const apiToken = results[0].value;
+
+            console.log(`Getting appstore web token from ${apiServerOrigin}`);
+
+            superagent.post(`${apiServerOrigin}/api/v1/user_token`)
+                .send({})
+                .query({ accessToken: apiToken })
+                .timeout(30 * 1000).end(function (error, response) {
+                    if (error && !error.response) {
+                        console.log('Network error getting web token', error);
+                        return callback();
+                    }
+                    if (response.statusCode !== 201 || !response.body.accessToken) {
+                        console.log(`Bad status getting web token: ${response.status} ${response.text}`);
+                        return callback();
+                    }
+
+                    db.runSql('INSERT settings (name, value) VALUES(?, ?)', [ 'appstore_web_token', response.body.accessToken ], callback);
+                });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN label VARCHAR(128) DEFAULT ""', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN label', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
@@ -0,0 +1,51 @@
+'use strict';
+
+const async = require('async'),
+    hat = require('../src/hat.js');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * from backups', function (error, allBackups) {
+        if (error) return callback(error);
+
+        console.log(`Fixing up ${allBackups.length} backup entries`);
+        const idMap = {};
+        allBackups.forEach(b => {
+            b.remotePath = b.id;
+            b.id = `${b.type}_${b.identifier}_v${b.packageVersion}_${hat(256)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying
+            idMap[b.remotePath] = b.id;
+        });
+
+        db.runSql('ALTER TABLE backups ADD COLUMN remotePath VARCHAR(256)', function (error) {
+            if (error) return callback(error);
+
+            db.runSql('ALTER TABLE backups CHANGE COLUMN dependsOn dependsOnJson TEXT', function (error) {
+                if (error) return callback(error);
+
+                async.eachSeries(allBackups, function (backup, iteratorDone) {
+                    const dependsOnPaths = backup.dependsOn ? backup.dependsOn.split(',') : []; // previously, it was paths
+                    let dependsOnIds = [];
+                    dependsOnPaths.forEach(p => { if (idMap[p]) dependsOnIds.push(idMap[p]); });
+
+                    db.runSql('UPDATE backups SET id = ?, remotePath = ?, dependsOnJson = ? WHERE id = ?', [ backup.id, backup.remotePath, JSON.stringify(dependsOnIds), backup.remotePath ], iteratorDone);
+                }, function (error) {
+                    if (error) return callback(error);
+
+                    db.runSql('ALTER TABLE backups MODIFY COLUMN remotePath VARCHAR(256) NOT NULL UNIQUE', callback);
+                });
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN remotePath', function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE backups RENAME COLUMN dependsOnJson to dependsOn', function (error) {
+            if (error) return callback(error);
+
+            callback(error);
+        });
+    });
+};
+
@@ -0,0 +1,22 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM apps', function (error, apps) {
+        if (error) return callback(error);
+
+        async.eachSeries(apps, function (app, iteratorDone) {
+            const manifest = JSON.parse(app.manifestJson);
+            const hasSso = !!manifest.addons['proxyAuth'] || !!manifest.addons['ldap'];
+            if (hasSso || !app.sso) return iteratorDone();
+
+            console.log(`Unsetting sso flag of ${app.id}`);
+            db.runSql('UPDATE apps SET sso=? WHERE id=?', [ 0, app.id ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,20 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM settings WHERE name = ?', [ 'api_server_origin' ], function (error, result) {
+        if (error || result.length === 0) return callback(error);
+
+        let consoleOrigin;
+        switch (result[0].value) {
+        case 'https://api.dev.cloudron.io': consoleOrigin = 'https://console.dev.cloudron.io'; break;
+        case 'https://api.staging.cloudron.io': consoleOrigin = 'https://console.staging.cloudron.io'; break;
+        default: consoleOrigin = 'https://console.cloudron.io'; break;
+        }
+
+        db.runSql('REPLACE INTO settings (name, value) VALUES (?, ?)', [ 'console_server_origin', consoleOrigin ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN backgroundImage MEDIUMBLOB', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN backgroundImage', callback);
+};
@@ -0,0 +1,12 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN mailboxDisplayName VARCHAR(128) DEFAULT "" NOT NULL', [], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN mailboxDisplayName', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,53 @@
+'use strict';
+
+const path = require('path'),
+    safe = require('safetydance'),
+    uuid = require('uuid');
+
+function getMountPoint(dataDir) {
+    const output = safe.child_process.execSync(`df --output=target "${dataDir}" | tail -1`, { encoding: 'utf8' });
+    if (!output) return dataDir;
+    const mountPoint = output.trim();
+    if (mountPoint === '/') return dataDir;
+    return mountPoint;
+}
+
+exports.up = async function(db) {
+    // use safe() here because this migration failed midway in 7.2.4
+    await safe(db.runSql('ALTER TABLE apps ADD storageVolumeId VARCHAR(128), ADD FOREIGN KEY(storageVolumeId) REFERENCES volumes(id)'));
+    await safe(db.runSql('ALTER TABLE apps ADD storageVolumePrefix VARCHAR(128)'));
+    await safe(db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_storageVolume UNIQUE (storageVolumeId, storageVolumePrefix)'));
+
+    const apps = await db.runSql('SELECT * FROM apps WHERE dataDir IS NOT NULL');
+
+    for (const app of apps) {
+        const allVolumes = await db.runSql('SELECT * FROM volumes');
+
+        console.log(`data-dir (${app.id}): migrating data dir ${app.dataDir}`);
+
+        const mountPoint = getMountPoint(app.dataDir);
+        const prefix = path.relative(mountPoint, app.dataDir);
+
+        console.log(`data-dir (${app.id}): migrating to mountpoint ${mountPoint} and prefix ${prefix}`);
+
+        const volume = allVolumes.find(v => v.hostPath === mountPoint);
+        if (volume) {
+            console.log(`data-dir (${app.id}): using existing volume ${volume.id}`);
+            await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ volume.id, prefix, app.id ]);
+            continue;
+        }
+
+        const id = uuid.v4().replace(/-/g, ''); // to make systemd mount file names more readable
+        const name = `appdata-${id}`;
+        const type = app.dataDir === mountPoint ? 'filesystem' : 'mountpoint';
+
+        console.log(`data-dir (${app.id}): creating new volume ${id}`);
+        await db.runSql('INSERT INTO volumes (id, name, hostPath, mountType, mountOptionsJson) VALUES (?, ?, ?, ?, ?)', [ id, name, mountPoint, type, JSON.stringify({}) ]);
+        await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ id, prefix, app.id ]);
+    }
+
+    await db.runSql('ALTER TABLE apps DROP COLUMN dataDir');
+};
+
+exports.down = async function(/*db*/) {
+};
@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = async function (db) {
+    await db.runSql('ALTER TABLE apps ADD COLUMN upstreamUri VARCHAR(256) DEFAULT ""');
+};
+
+exports.down = async function (db) {
+    await db.runSql('ALTER TABLE apps DROP COLUMN upstreamUri');
+};
@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = async function(db) {
+    const result = await db.runSql('SELECT * FROM settings WHERE name=?', [ 'backup_config' ]);
+    if (!result.length) return;
+
+    const backupConfig = JSON.parse(result[0].value);
+
+    if (backupConfig.encryption && backupConfig.format === 'rsync') backupConfig.encryptedFilenames = true;
+
+    await db.runSql('UPDATE settings SET value=? WHERE name=?', [ JSON.stringify(backupConfig), 'backup_config',  ]);
+};
+
+exports.down = async function(/* db */) {
+};
@@ -0,0 +1,22 @@
+'use strict';
+
+exports.up = async function (db) {
+    var cmd = 'CREATE TABLE applinks(' +
+                'id VARCHAR(128) NOT NULL UNIQUE,' +
+                'accessRestrictionJson TEXT,' +
+                'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
+                'updateTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
+                'ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,' +
+                'label VARCHAR(128),' +
+                'tagsJson VARCHAR(2048),' +
+                'icon MEDIUMBLOB,' +
+                'upstreamUri VARCHAR(256) DEFAULT "",' +
+                'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin';
+
+    await db.runSql(cmd);
+};
+
+exports.down = async function (db) {
+    await db.runSql('DROP TABLE applinks');
+};
+
@@ -0,0 +1,17 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN storageQuota BIGINT DEFAULT 0'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN messagesQuota BIGINT DEFAULT 0'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE mailboxes DROP COLUMN storageQuota'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes DROP COLUMN messagesQuota')
+    ], callback);
+};
@@ -0,0 +1,18 @@
+'use strict';
+
+const safe = require('safetydance');
+
+exports.up = async function (db) {
+    const mailDomains = await db.runSql('SELECT * FROM mail', []);
+
+    for (const mailDomain of mailDomains) {
+        let catchAll = safe.JSON.parse(mailDomain.catchAllJson) || [];
+        if (catchAll.length === 0) continue;
+
+        catchAll = catchAll.map(a => `${a}@${mailDomain.domain}`);
+        await db.runSql('UPDATE mail SET catchAllJson = ? WHERE domain = ?', [ JSON.stringify(catchAll), mailDomain.domain ]);
+    }
+};
+
+exports.down = async function( /* db */) {
+};
@@ -0,0 +1,13 @@
+'use strict';
+
+exports.up = async function (db) {
+    await db.runSql('ALTER TABLE tokens DROP COLUMN scope');
+    await db.runSql('ALTER TABLE tokens ADD COLUMN scopeJson TEXT');
+
+    await db.runSql('UPDATE tokens SET scopeJson = ?', [ JSON.stringify({'*':'rw'})]);
+};
+
+exports.down = async function (db) {
+    await db.runSql('ALTER TABLE tokens ADD COLUMN scope VARCHAR(512) NOT NULL DEFAULT ""');
+    await db.runSql('ALTER TABLE tokens DROP COLUMN scopeJson');
+};
@@ -28,11 +28,13 @@ CREATE TABLE IF NOT EXISTS users(
    twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
    source VARCHAR(128) DEFAULT "",
    role VARCHAR(32),
+    inviteToken VARCHAR(128) DEFAULT "",
    resetToken VARCHAR(128) DEFAULT "",
    resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    active BOOLEAN DEFAULT 1,
-    avatar MEDIUMBLOB,
-    locationJson TEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
+    avatar MEDIUMBLOB NOT NULL,
+    backgroundImage MEDIUMBLOB,
+    loginLocationsJson MEDIUMTEXT, // { locations: [{ ip, userAgent, city, country, ts }] }

    INDEX creationTime_index (creationTime),
    PRIMARY KEY(id));
@@ -56,7 +58,7 @@ CREATE TABLE IF NOT EXISTS tokens(
    accessToken VARCHAR(128) NOT NULL UNIQUE,
    identifier VARCHAR(128) NOT NULL, // resourceId: app id or user id
    clientId VARCHAR(128),
-    scope VARCHAR(512) NOT NULL,
+    scopeJson TEXT,
    expires BIGINT NOT NULL, // FIXME: make this a timestamp
    lastUsedTime TIMESTAMP NULL,
    PRIMARY KEY(accessToken));
@@ -84,19 +86,28 @@ CREATE TABLE IF NOT EXISTS apps(
    enableAutomaticUpdate BOOLEAN DEFAULT 1,
    enableMailbox BOOLEAN DEFAULT 1, // whether sendmail addon is enabled
    mailboxName VARCHAR(128), // mailbox of this app
-    mailboxDomain VARCHAR(128), // mailbox domain of this apps
+    mailboxDomain VARCHAR(128), // mailbox domain of this app
+    mailboxDisplayName VARCHAR(128), // mailbox display name
+    enableInbox BOOLEAN DEFAULT 0, // whether recvmail addon is enabled
+    inboxName VARCHAR(128), // mailbox of this app
+    inboxDomain VARCHAR(128), // mailbox domain of this app
    label VARCHAR(128),     // display name
    tagsJson VARCHAR(2048), // array of tags
-    dataDir VARCHAR(256) UNIQUE,
+    storageVolumeId VARCHAR(128),
+    storageVolumePrefix VARCHAR(128),
    taskId INTEGER, // current task
    errorJson TEXT,
    servicesConfigJson TEXT, // app services configuration
    containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair'
    appStoreIcon MEDIUMBLOB,
    icon MEDIUMBLOB,
+    crontab TEXT,
+    upstreamUri VARCHAR(256) DEFAULT "",

    FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
    FOREIGN KEY(taskId) REFERENCES tasks(id),
+    FOREIGN KEY(storageVolumeId) REFERENCES volumes(id),
+    UNIQUE (storageVolumeId, storageVolumePrefix),
    PRIMARY KEY(id));

 CREATE TABLE IF NOT EXISTS appPortBindings(
@@ -117,7 +128,7 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
    appId VARCHAR(128) NOT NULL,
    addonId VARCHAR(32) NOT NULL,
    name VARCHAR(128) NOT NULL,
-    value VARCHAR(512) NOT NULL,
+    value TEXT NOT NULL,
    FOREIGN KEY(appId) REFERENCES apps(id));

 CREATE TABLE IF NOT EXISTS appEnvVars(
@@ -128,12 +139,14 @@ CREATE TABLE IF NOT EXISTS appEnvVars(

 CREATE TABLE IF NOT EXISTS backups(
    id VARCHAR(128) NOT NULL,
+    remotePath VARCHAR(256) NOT NULL UNIQUE,
+    label VARCHAR(128) DEFAULT "",
    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    packageVersion VARCHAR(128) NOT NULL, /* app version or box version */
    encryptionVersion INTEGER, /* when null, unencrypted backup */
    type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
    identifier VARCHAR(128) NOT NULL, /* 'box' or the app id */
-    dependsOn TEXT, /* comma separate list of objects this backup depends on */
+    dependsOnJson TEXT, /* comma separate list of objects this backup depends on */
    state VARCHAR(16) NOT NULL,
    manifestJson TEXT, /* to validate if the app can be installed in this version of box */
    format VARCHAR(16) DEFAULT "tgz",
@@ -145,8 +158,8 @@ CREATE TABLE IF NOT EXISTS backups(
 CREATE TABLE IF NOT EXISTS eventlog(
    id VARCHAR(128) NOT NULL,
    action VARCHAR(128) NOT NULL,
-    source TEXT, /* { userId, username, ip }. userId can be null for cron,sysadmin */
-    data TEXT, /* free flowing json based on action */
+    sourceJson TEXT, /* { userId, username, ip }. userId can be null for cron,sysadmin */
+    dataJson TEXT, /* free flowing json based on action */
    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,

    INDEX creationTime_index (creationTime),
@@ -176,6 +189,7 @@ CREATE TABLE IF NOT EXISTS mail(
    relayJson TEXT,
    bannerJson TEXT,

+    dkimKeyJson MEDIUMTEXT,
    dkimSelector VARCHAR(128) NOT NULL DEFAULT "cloudron",

    FOREIGN KEY(domain) REFERENCES domains(domain),
@@ -202,16 +216,20 @@ CREATE TABLE IF NOT EXISTS mailboxes(
    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    domain VARCHAR(128),
    active BOOLEAN DEFAULT 1,
+    enablePop3 BOOLEAN DEFAULT 0,
+    storageQuota BIGINT DEFAULT 0,
+    messagesQuota BIGINT DEFAULT 0,

    FOREIGN KEY(domain) REFERENCES mail(domain),
    FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
    UNIQUE (name, domain));

-CREATE TABLE IF NOT EXISTS subdomains(
+CREATE TABLE IF NOT EXISTS locations(
    appId VARCHAR(128) NOT NULL,
    domain VARCHAR(128) NOT NULL,
    subdomain VARCHAR(128) NOT NULL,
-    type VARCHAR(128) NOT NULL, /* primary or redirect */
+    type VARCHAR(128) NOT NULL, /* primary, secondary, redirect, alias */
+    environmentVariable VARCHAR(128), /* only set for secondary */

    certificateJson MEDIUMTEXT,

@@ -222,6 +240,7 @@ CREATE TABLE IF NOT EXISTS subdomains(
 CREATE TABLE IF NOT EXISTS tasks(
    id int NOT NULL AUTO_INCREMENT,
    type VARCHAR(32) NOT NULL,
+    argsJson TEXT,
    percent INTEGER DEFAULT 0,
    message TEXT,
    errorJson TEXT,
@@ -278,7 +297,20 @@ CREATE TABLE IF NOT EXISTS appMounts(

 CREATE TABLE IF NOT EXISTS blobs(
    id VARCHAR(128) NOT NULL UNIQUE,
-    value TEXT,
+    value MEDIUMBLOB,
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS appLinks(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
+    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, // when the app was installed
+    updateTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,   // when the last app update was done
+    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, // when this db record was updated (useful for UI caching)
+    label VARCHAR(128),     // display name
+    tagsJson VARCHAR(2048), // array of tags
+    icon MEDIUMBLOB,
+    upstreamUri VARCHAR(256) DEFAULT "",
+
    PRIMARY KEY(id));

 CHARACTER SET utf8 COLLATE utf8_bin;
@@ -11,82 +11,70 @@
    "url": "https://git.cloudron.io/cloudron/box.git"
  },
  "dependencies": {
-    "@google-cloud/dns": "^2.1.0",
-    "@google-cloud/storage": "^5.8.5",
+    "@google-cloud/dns": "^2.2.4",
+    "@google-cloud/storage": "^5.19.2",
    "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^3.2.0",
-    "aws-sdk": "^2.906.0",
+    "async": "^3.2.3",
+    "aws-sdk": "^2.1115.0",
    "basic-auth": "^2.0.1",
-    "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.10.2",
+    "body-parser": "^1.20.0",
+    "cloudron-manifestformat": "^5.18.0",
    "connect": "^3.7.0",
-    "connect-lastmile": "^2.1.0",
+    "connect-lastmile": "^2.1.1",
    "connect-timeout": "^1.9.0",
-    "cookie-parser": "^1.4.5",
-    "cookie-session": "^1.4.0",
+    "cookie-parser": "^1.4.6",
+    "cookie-session": "^2.0.0",
    "cron": "^1.8.2",
-    "db-migrate": "^0.11.12",
-    "db-migrate-mysql": "^2.1.2",
-    "debug": "^4.3.1",
-    "delay": "^5.0.0",
-    "dockerode": "^3.3.0",
-    "delay": "^5.0.0",
+    "db-migrate": "^0.11.13",
+    "db-migrate-mysql": "^2.2.0",
+    "debug": "^4.3.4",
+    "dockerode": "^3.3.1",
    "ejs": "^3.1.6",
-    "ejs-cli": "^2.2.1",
-    "express": "^4.17.1",
-    "ipaddr.js": "^2.0.0",
+    "ejs-cli": "^2.2.3",
+    "express": "^4.17.3",
+    "ipaddr.js": "^2.0.1",
    "js-yaml": "^4.1.0",
+    "jsdom": "^20.0.0",
    "json": "^11.0.0",
    "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.2.4",
+    "ldapjs": "^2.3.2",
    "lodash": "^4.17.21",
-    "lodash.chunk": "^4.2.0",
-    "mime": "^2.5.2",
-    "moment": "^2.29.1",
-    "moment-timezone": "^0.5.33",
+    "moment": "^2.29.2",
+    "moment-timezone": "^0.5.34",
    "morgan": "^1.10.0",
-    "multiparty": "^4.2.2",
-    "mustache-express": "^1.3.0",
+    "multiparty": "^4.2.3",
    "mysql": "^2.18.1",
-    "nodemailer": "^6.6.0",
-    "nodemailer-smtp-transport": "^2.7.4",
-    "once": "^1.4.0",
-    "pretty-bytes": "^5.6.0",
+    "nodemailer": "^6.7.3",
    "progress-stream": "^2.0.0",
-    "proxy-middleware": "^0.15.0",
-    "qrcode": "^1.4.4",
+    "qrcode": "^1.5.0",
    "readdirp": "^3.6.0",
-    "request": "^2.88.2",
-    "rimraf": "^3.0.2",
-    "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^2.0.1",
-    "semver": "^7.3.5",
+    "safetydance": "^2.2.0",
+    "semver": "^7.3.7",
    "speakeasy": "^2.0.0",
    "split": "^1.0.1",
-    "superagent": "^6.1.0",
-    "supererror": "^0.7.2",
+    "superagent": "^7.1.1",
    "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
    "tar-stream": "^2.2.0",
    "tldjs": "^2.3.1",
-    "ua-parser-js": "^0.7.28",
-    "underscore": "^1.13.1",
+    "ua-parser-js": "^1.0.2",
+    "underscore": "^1.13.2",
    "uuid": "^8.3.2",
-    "validator": "^13.6.0",
-    "ws": "^7.4.5",
+    "validator": "^13.7.0",
+    "ws": "^8.5.0",
    "xml2js": "^0.4.23"
  },
  "devDependencies": {
    "expect.js": "*",
    "hock": "^1.4.1",
-    "js2xmlparser": "^4.0.1",
-    "mocha": "^8.4.0",
+    "js2xmlparser": "^4.0.2",
+    "mocha": "^9.2.2",
    "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^13.0.11",
-    "node-sass": "^6.0.0",
-    "recursive-readdir": "^2.2.2"
+    "nock": "^13.2.4",
+    "node-sass": "^7.0.1",
+    "nyc": "^15.1.0"
  },
  "scripts": {
-    "test": "./runTests",
+    "test": "./run-tests",
    "postmerge": "/bin/true",
    "precommit": "/bin/true",
    "prepush": "npm test",
@@ -6,7 +6,7 @@ readonly source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly DATA_DIR="${HOME}/.cloudron_test"
 readonly DEFAULT_TESTS="./src/test/*-test.js ./src/routes/test/*-test.js"

-! "${source_dir}/src/test/checkInstall" && exit 1
+! "${source_dir}/src/test/check-install" && exit 1

 # cleanup old data dirs some of those docker container data requires sudo to be removed
 echo "=> Provide root password to purge any leftover data in ${DATA_DIR} and load apparmor profile:"
@@ -22,7 +22,7 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p boxdata/box boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
 mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update
 sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test

@@ -37,14 +37,15 @@ openssl req -x509 -newkey rsa:2048 -keyout platformdata/nginx/cert/host.key -out
 # clear out any containers if FAST is unset
 if [[ -z ${FAST+x} ]]; then
    echo "=> Delete all docker containers first"
-    docker ps -qa | xargs --no-run-if-empty docker rm -f
-    echo "==> To skip this run with: FAST=1 ./runTests"
+    docker ps -qa --filter "label=isCloudronManaged" | xargs --no-run-if-empty docker rm -f
+    docker rm -f mysql-server
+    echo "==> To skip this run with: FAST=1 ./run-tests"
 else
    echo "==> WARNING!! Skipping docker container cleanup, the database might not be pristine!"
 fi

 # create docker network (while the infra code does this, most tests skip infra setup)
-docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
+docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 --gateway 172.18.0.1 cloudron --ipv6 --subnet=fd00:c107:d509::/64 || true

 # create the same mysql server version to test with
 OUT=`docker inspect mysql-server` || true
@@ -62,6 +63,9 @@ while ! mysqladmin ping -h"${MYSQL_IP}" --silent; do
    sleep 1
 done

+echo "=> Ensure local base image"
+docker pull cloudron/base:3.0.0@sha256:455c70428723e3a823198c57472785437eb6eab082e79b3ff04ea584faf46e92
+
 echo "=> Create iptables blocklist"
 sudo ipset create cloudron_blocklist hash:net || true

@@ -75,10 +79,15 @@ echo "=> Run database migrations"
 cd "${source_dir}"
 BOX_ENV=test DATABASE_URL=mysql://root:password@${MYSQL_IP}/box node_modules/.bin/db-migrate up

-echo "=> Run tests with mocha"
 TESTS=${DEFAULT_TESTS}
 if [[ $# -gt 0 ]]; then
    TESTS="$*"
 fi

-BOX_ENV=test ./node_modules/mocha/bin/_mocha --bail --no-timeouts --exit -R spec ${TESTS}
+if [[ -z ${COVERAGE+x} ]]; then
+    echo "=> Run tests with mocha"
+    BOX_ENV=test ./node_modules/.bin/mocha --bail --no-timeouts --exit -R spec ${TESTS}
+else
+    echo "=> Run tests with mocha and coverage"
+    BOX_ENV=test ./node_modules/.bin/nyc --reporter=html  ./node_modules/.bin/mocha --no-timeouts --exit -R spec ${TESTS}
+fi
@@ -11,7 +11,7 @@ trap exitHandler EXIT
 # change this to a hash when we make a upgrade release
 readonly LOG_FILE="/var/log/cloudron-setup.log"
 readonly MINIMUM_DISK_SIZE_GB="18" # this is the size of "/" and required to fit in docker images 18 is a safe bet for different reporting on 20GB min
-readonly MINIMUM_MEMORY="974"      # this is mostly reported for 1GB main memory (DO 992, EC2 990, Linode 989, Serverdiscounter.com 974)
+readonly MINIMUM_MEMORY="960"      # this is mostly reported for 1GB main memory (DO 992, EC2 967, Linode 989, Serverdiscounter.com 974)

 readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"

@@ -26,8 +26,8 @@ readonly GREEN='\033[32m'
 readonly DONE='\033[m'

 # verify the system has minimum requirements met
-if [[ "${rootfs_type}" != "ext4" ]]; then
-    echo "Error: Cloudron requires '/' to be ext4" # see #364
+if [[ "${rootfs_type}" != "ext4" && "${rootfs_type}" != "xfs" ]]; then
+    echo "Error: Cloudron requires '/' to be ext4 or xfs" # see #364
    exit 1
 fi

@@ -41,23 +41,35 @@ if [[ "${disk_size_gb}" -lt "${MINIMUM_DISK_SIZE_GB}" ]]; then
    exit 1
 fi

+if [[ "$(uname -m)" != "x86_64" ]]; then
+    echo "Error: Cloudron only supports amd64/x86_64"
+    exit 1
+fi
+
+if cvirt=$(systemd-detect-virt --container); then
+    echo "Error: Cloudron does not support ${cvirt}, only runs on bare metal or with full hardware virtualization"
+    exit 1
+fi
+
 # do not use is-active in case box service is down and user attempts to re-install
 if systemctl cat box.service >/dev/null 2>&1; then
    echo "Error: Cloudron is already installed. To reinstall, start afresh"
    exit 1
 fi

-initBaseImage="true"
 provider="generic"
 requestedVersion=""
 installServerOrigin="https://api.cloudron.io"
 apiServerOrigin="https://api.cloudron.io"
 webServerOrigin="https://cloudron.io"
+consoleServerOrigin="https://console.cloudron.io"
 sourceTarballUrl=""
 rebootServer="true"
-setupToken=""
+setupToken="" # this is a OTP for securing an installation (https://forum.cloudron.io/topic/6389/add-password-for-initial-configuration)
+appstoreSetupToken=""
+redo="false"

-args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot,generate-setup-token" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,provider:,version:,env:,skip-reboot,generate-setup-token,setup-token:,redo" -n "$0" -- "$@")
 eval set -- "${args}"

 while true; do
@@ -69,17 +81,20 @@ while true; do
        if [[ "$2" == "dev" ]]; then
            apiServerOrigin="https://api.dev.cloudron.io"
            webServerOrigin="https://dev.cloudron.io"
+            consoleServerOrigin="https://console.dev.cloudron.io"
            installServerOrigin="https://api.dev.cloudron.io"
        elif [[ "$2" == "staging" ]]; then
            apiServerOrigin="https://api.staging.cloudron.io"
            webServerOrigin="https://staging.cloudron.io"
+            consoleServerOrigin="https://console.staging.cloudron.io"
            installServerOrigin="https://api.staging.cloudron.io"
        elif [[ "$2" == "unstable" ]]; then
            installServerOrigin="https://api.dev.cloudron.io"
        fi
        shift 2;;
-    --skip-baseimage-init) initBaseImage="false"; shift;;
    --skip-reboot) rebootServer="false"; shift;;
+    --redo) redo="true"; shift;;
+    --setup-token) appstoreSetupToken="$2"; shift 2;;
    --generate-setup-token) setupToken="$(openssl rand -hex 10)"; shift;;
    --) break;;
    *) echo "Unknown option $1"; exit 1;;
@@ -94,11 +109,18 @@ fi

 # Only --help works with mismatched ubuntu
 ubuntu_version=$(lsb_release -rs)
-if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubuntu_version}" != "20.04" ]]; then
-    echo "Cloudron requires Ubuntu 16.04, 18.04 or 20.04" > /dev/stderr
+if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubuntu_version}" != "20.04" && "${ubuntu_version}" != "22.04" ]]; then
+    echo "Cloudron requires Ubuntu 18.04, 20.04, 22.04" > /dev/stderr
    exit 1
 fi

+if which nginx >/dev/null || which docker >/dev/null || which node > /dev/null; then
+    if [[ "${redo}" == "false" ]]; then
+        echo "Error: Some packages like nginx/docker/nodejs are already installed. Cloudron requires specific versions of these packages and will install them as part of it's installation. Please start with a fresh Ubuntu install and run this script again." > /dev/stderr
+        exit 1
+    fi
+fi
+
 # Install MOTD file for stack script style installations. this is removed by the trap exit handler. Heredoc quotes prevents parameter expansion
 cat > /etc/update-motd.d/91-cloudron-install-in-progress <<'EOF'
 #!/bin/bash
@@ -133,17 +155,15 @@ echo ""
 echo " Join us at https://forum.cloudron.io for any questions."
 echo ""

-if [[ "${initBaseImage}" == "true" ]]; then
-    echo "=> Updating apt and installing script dependencies"
-    if ! apt-get update &>> "${LOG_FILE}"; then
-        echo "Could not update package repositories. See ${LOG_FILE}"
-        exit 1
-    fi
+echo "=> Updating apt and installing script dependencies"
+if ! apt-get update &>> "${LOG_FILE}"; then
+    echo "Could not update package repositories. See ${LOG_FILE}"
+    exit 1
+fi

-    if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install --no-install-recommends curl python3 ubuntu-standard software-properties-common -y &>> "${LOG_FILE}"; then
-        echo "Could not install setup dependencies (curl). See ${LOG_FILE}"
-        exit 1
-    fi
+if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install --no-install-recommends curl python3 ubuntu-standard software-properties-common -y &>> "${LOG_FILE}"; then
+    echo "Could not install setup dependencies (curl). See ${LOG_FILE}"
+    exit 1
 fi

 echo "=> Checking version"
@@ -163,7 +183,7 @@ if ! sourceTarballUrl=$(echo "${releaseJson}" | python3 -c 'import json,sys;obj=
    exit 1
 fi

-echo "=> Downloading version ${version} ..."
+echo "=> Downloading Cloudron version ${version} ..."
 box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)

 if ! $curl -sL "${sourceTarballUrl}" | tar -zxf - -C "${box_src_tmp_dir}"; then
@@ -171,18 +191,16 @@ if ! $curl -sL "${sourceTarballUrl}" | tar -zxf - -C "${box_src_tmp_dir}"; then
    exit 1
 fi

-if [[ "${initBaseImage}" == "true" ]]; then
-    echo -n "=> Installing base dependencies and downloading docker images (this takes some time) ..."
-    # initializeBaseUbuntuImage.sh args (provider, infraversion path) are only to support installation of pre 5.3 Cloudrons
-    if ! /bin/bash "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "generic" "../src" &>> "${LOG_FILE}"; then
-        echo "Init script failed. See ${LOG_FILE} for details"
-        exit 1
-    fi
-    echo ""
+echo -n "=> Installing base dependencies and downloading docker images (this takes some time) ..."
+init_ubuntu_script=$(test -f "${box_src_tmp_dir}/scripts/init-ubuntu.sh" && echo "${box_src_tmp_dir}/scripts/init-ubuntu.sh" || echo "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh")
+if ! /bin/bash "${init_ubuntu_script}" &>> "${LOG_FILE}"; then
+    echo "Init script failed. See ${LOG_FILE} for details"
+    exit 1
 fi
+echo ""

 # The provider flag is still used for marketplace images
-echo "=> Installing version ${version} (this takes some time) ..."
+echo "=> Installing Cloudron version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
 echo "${provider}" > /etc/cloudron/PROVIDER
 [[ ! -z "${setupToken}" ]] && echo "${setupToken}" > /etc/cloudron/SETUP_TOKEN
@@ -194,6 +212,20 @@ fi

 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('api_server_origin', '${apiServerOrigin}');" 2>/dev/null
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('web_server_origin', '${webServerOrigin}');" 2>/dev/null
+mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('console_server_origin', '${consoleServerOrigin}');" 2>/dev/null
+
+if [[ -n "${appstoreSetupToken}" ]]; then
+    if ! setupResponse=$(curl -sX POST -H "Content-type: application/json" --data "{\"setupToken\": \"${appstoreSetupToken}\"}" "${apiServerOrigin}/api/v1/cloudron_setup_done"); then
+        echo "Could not complete setup. See ${LOG_FILE} for details"
+        exit 1
+    fi
+
+    cloudronId=$(echo "${setupResponse}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["cloudronId"])')
+    mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('cloudron_id', '${cloudronId}');" 2>/dev/null
+
+    appstoreApiToken=$(echo "${setupResponse}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["cloudronToken"])')
+    mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('appstore_api_token', '${appstoreApiToken}');" 2>/dev/null
+fi

 echo -n "=> Waiting for cloudron to be ready (this takes some time) ..."
 while true; do
@@ -204,20 +236,31 @@ while true; do
    sleep 10
 done

-if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
-    ip='<IP>'
-fi
+ip4=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
+ip6=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv6.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
+
+url4=""
+url6=""
+fallbackUrl=""
 if [[ -z "${setupToken}" ]]; then
-    url="https://${ip}"
+    [[ -n "${ip4}" ]] && url4="https://${ip4}"
+    [[ -n "${ip6}" ]] && url6="https://[${ip6}]"
+    [[ -z "${ip4}" && -z "${ip6}" ]] && fallbackUrl="https://<IP>"
 else
-    url="https://${ip}/?setupToken=${setupToken}"
+    [[ -n "${ip4}" ]] && url4="https://${ip4}/?setupToken=${setupToken}"
+    [[ -n "${ip6}" ]] && url6="https://[${ip6}]/?setupToken=${setupToken}"
+    [[ -z "${ip4}" && -z "${ip6}" ]] && fallbackUrl="https://<IP>?setupToken=${setupToken}"
 fi
-echo -e "\n\n${GREEN}After reboot, visit ${url} and accept the self-signed certificate to finish setup.${DONE}\n"
+echo -e "\n\n${GREEN}After reboot, visit one of the following URLs and accept the self-signed certificate to finish setup.${DONE}\n"
+[[ -n "${url4}" ]] && echo -e "  * ${GREEN}${url4}${DONE}"
+[[ -n "${url6}" ]] && echo -e "  * ${GREEN}${url6}${DONE}"
+[[ -n "${fallbackUrl}" ]] && echo -e "  * ${GREEN}${fallbackUrl}${DONE}"

 if [[ "${rebootServer}" == "true" ]]; then
    systemctl stop box mysql # sometimes mysql ends up having corrupt privilege tables

-    read -p "The server has to be rebooted to apply all the settings. Reboot now ? [Y/n] " yn
+    # https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#ANSI_002dC-Quoting
+    read -p $'\n'"The server has to be rebooted to apply all the settings. Reboot now ? [Y/n] " yn
    yn=${yn:-y}
    case $yn in
        [Yy]* ) exitHandler; systemctl reboot;;
@@ -8,14 +8,15 @@ set -eu -o pipefail
 PASTEBIN="https://paste.cloudron.io"
 OUT="/tmp/cloudron-support.log"
 LINE="\n========================================================\n"
-CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQVilclYAIu+ioDp/sgzzFz6YU0hPcRYY7ze/LiF/lC7uQqK062O54BFXTvQ3ehtFZCx3bNckjlT2e6gB8Qq07OM66De4/S/g+HJW4TReY2ppSPMVNag0TNGxDzVH8pPHOysAm33LqT2b6L/wEXwC6zWFXhOhHjcMqXvi8Ejaj20H1HVVcf/j8qs5Thkp9nAaFTgQTPu8pgwD8wDeYX1hc9d0PYGesTADvo6HF4hLEoEnefLw7PaStEbzk2fD3j7/g5r5HcgQQXBe74xYZ/1gWOX2pFNuRYOBSEIrNfJEjFJsqk3NR1+ZoMGK7j+AZBR4k0xbrmncQLcQzl6MMDzkp support@cloudron.io"
+CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGWS+930b8QdzbchGljt3KSljH9wRhYvht8srrtQHdzg support@cloudron.io"
 HELP_MESSAGE="
-This script collects diagnostic information to help debug server related issues
+This script collects diagnostic information to help debug server related issues.

 Options:
-   --owner-login   Login as owner
-   --enable-ssh    Enable SSH access for the Cloudron support team
-   --help          Show this message
+   --owner-login             Login as owner
+   --enable-ssh              Enable SSH access for the Cloudron support team
+   --reset-appstore-account  Reset associated cloudron.io account
+   --help                    Show this message
 "

 # We require root
@@ -26,7 +27,7 @@ fi

 enableSSH="false"

-args=$(getopt -o "" -l "help,enable-ssh,admin-login,owner-login" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,enable-ssh,admin-login,owner-login,reset-appstore-account" -n "$0" -- "$@")
 eval set -- "${args}"

 while true; do
@@ -37,13 +38,20 @@ while true; do
        # fall through
        ;&
    --owner-login)
-        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY createdAt LIMIT 1" 2>/dev/null)
+        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY creationTime LIMIT 1" 2>/dev/null)
        admin_password=$(pwgen -1s 12)
-        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/nul)
-        ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
-        printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > "${ghost_file}"
-        chown yellowtent:yellowtent "${ghost_file}" && chmod o-r,g-r "${ghost_file}"
-        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
+        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/null)
+        mysql -NB -uroot -ppassword -e "INSERT INTO box.settings (name, value) VALUES ('ghosts_config', '{\"${admin_username}\":\"${admin_password}\"}') ON DUPLICATE KEY UPDATE name='ghosts_config', value='{\"${admin_username}\":\"${admin_password}\"}'" 2>/dev/null
+        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once."
+        exit 0
+        ;;
+    --reset-appstore-account)
+        echo -e "This will reset the Cloudron.io account associated with this Cloudron. Once reset, you can re-login with a different account in the Cloudron Dashboard. See https://docs.cloudron.io/appstore/#change-account for more information.\n"
+        read -e -p "Reset the Cloudron.io account? [y/N] " choice
+        [[ "$choice" != [Yy]* ]] && exit 1
+        mysql -uroot -ppassword -e "DELETE FROM box.settings WHERE name='cloudron_token';" 2>/dev/null
+        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/null)
+        echo "Account reset. Please re-login at https://${dashboard_domain}/#/appstore"
        exit 0
        ;;
    --) break;;
@@ -69,6 +77,31 @@ if [[ "`df --output="avail" /tmp | sed -n 2p`" -lt "5120" ]]; then
    exit 1
 fi

+if [[ "${enableSSH}" == "true" ]]; then
+    ssh_port=$(cat /etc/ssh/sshd_config | grep "Port " | sed -e "s/.*Port //")
+
+    ssh_user="cloudron-support"
+    keys_file="/home/cloudron-support/.ssh/authorized_keys"
+
+    echo -e $LINE"SSH"$LINE >> $OUT
+    echo "Username: ${ssh_user}" >> $OUT
+    echo "Port:     ${ssh_port}" >> $OUT
+    echo "Key file: ${keys_file}" >> $OUT
+
+    echo -n "Enabling ssh access for the Cloudron support team..."
+    mkdir -p $(dirname "${keys_file}")       # .ssh does not exist sometimes
+    touch "${keys_file}"                # required for concat to work
+    if ! grep -q "${CLOUDRON_SUPPORT_PUBLIC_KEY}" "${keys_file}"; then
+        echo -e "\n${CLOUDRON_SUPPORT_PUBLIC_KEY}" >> "${keys_file}"
+        chmod 600 "${keys_file}"
+        chown "${ssh_user}" "${keys_file}"
+    fi
+
+    echo "Done"
+
+    exit 0
+fi
+
 echo -n "Generating Cloudron Support stats..."

 # clear file
@@ -111,40 +144,8 @@ iptables -L &>> $OUT

 echo "Done"

-if [[ "${enableSSH}" == "true" ]]; then
-    ssh_port=$(cat /etc/ssh/sshd_config | grep "Port " | sed -e "s/.*Port //")
-    permit_root_login=$(grep -q ^PermitRootLogin.*yes /etc/ssh/sshd_config && echo "yes" || echo "no")
-
-    # support.js uses similar logic
-    if [[ -d /home/ubuntu ]]; then
-        ssh_user="ubuntu"
-        keys_file="/home/ubuntu/.ssh/authorized_keys"
-    else
-        ssh_user="root"
-        keys_file="/root/.ssh/authorized_keys"
-    fi
-
-    echo -e $LINE"SSH"$LINE >> $OUT
-    echo "Username: ${ssh_user}" >> $OUT
-    echo "Port:     ${ssh_port}" >> $OUT
-    echo "PermitRootLogin: ${permit_root_login}" >> $OUT
-    echo "Key file: ${keys_file}" >> $OUT
-
-    echo -n "Enabling ssh access for the Cloudron support team..."
-    mkdir -p $(dirname "${keys_file}")       # .ssh does not exist sometimes
-    touch "${keys_file}"                # required for concat to work
-    if ! grep -q "${CLOUDRON_SUPPORT_PUBLIC_KEY}" "${keys_file}"; then
-        echo -e "\n${CLOUDRON_SUPPORT_PUBLIC_KEY}" >> "${keys_file}"
-        chmod 600 "${keys_file}"
-        chown "${ssh_user}" "${keys_file}"
-    fi
-
-    echo "Done"
-fi
-
 echo -n "Uploading information..."
-# for some reason not using $(cat $OUT) will not contain newlines!?
-paste_key=$(curl -X POST ${PASTEBIN}/documents --silent -d "$(cat $OUT)" | python3 -c "import sys, json; print(json.load(sys.stdin)['key'])")
+paste_key=$(curl -X POST ${PASTEBIN}/documents --silent --data-binary "@$OUT" | python3 -c "import sys, json; print(json.load(sys.stdin)['key'])")
 echo "Done"

 echo ""
@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
    exit 1
 fi

-if [[ "$(node --version)" != "v14.15.4" ]]; then
-    echo "This script requires node 14.15.4"
+if [[ "$(node --version)" != "v16.13.1" ]]; then
+    echo "This script requires node 16.13.1"
    exit 1
 fi

@@ -0,0 +1,199 @@
+#!/bin/bash
+
+# This script is run on the base ubuntu. Put things here which are managed by ubuntu
+# This script is also run after ubuntu upgrade
+
+set -euv -o pipefail
+
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+readonly arg_infraversionpath="${SOURCE_DIR}/../src"
+
+function die {
+    echo $1
+    exit 1
+}
+
+export DEBIAN_FRONTEND=noninteractive
+
+readonly ubuntu_codename=$(lsb_release -cs)
+readonly ubuntu_version=$(lsb_release -rs)
+
+# hold grub since updating it breaks on some VPS providers. also, dist-upgrade will trigger it
+apt-mark hold grub* >/dev/null
+apt-get -o Dpkg::Options::="--force-confdef" update -y
+apt-get -o Dpkg::Options::="--force-confdef" upgrade -y
+apt-mark unhold grub* >/dev/null
+
+echo "==> Installing required packages"
+
+debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
+debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
+
+# this enables automatic security upgrades (https://help.ubuntu.com/community/AutomaticSecurityUpdates)
+# resolvconf is needed for unbound to work property after disabling systemd-resolved in 18.04
+case "${ubuntu_version}" in
+16.04)
+    gpg_package="gnupg"
+    mysql_package="mysql-server-5.7"
+    ntpd_package=""
+    python_package="python2.7"
+    nginx_package="" # we use custom package for TLS v1.3 support
+    ;;
+18.04)
+    gpg_package="gpg"
+    mysql_package="mysql-server-5.7"
+    ntpd_package=""
+    python_package="python2.7"
+    nginx_package="" # we use custom package for TLS v1.3 support
+    ;;
+20.04)
+    gpg_package="gpg"
+    mysql_package="mysql-server-8.0"
+    ntpd_package="systemd-timesyncd"
+    python_package="python3.8"
+    nginx_package="nginx-full"
+    ;;
+22.04)
+    gpg_package="gpg"
+    mysql_package="mysql-server-8.0"
+    ntpd_package="systemd-timesyncd"
+    python_package="python3.10"
+    nginx_package="nginx-full"
+    ;;
+esac
+
+apt-get -y install --no-install-recommends \
+    acl \
+    apparmor \
+    build-essential \
+    cifs-utils \
+    cron \
+    curl \
+    debconf-utils \
+    dmsetup \
+    $gpg_package \
+    ipset \
+    iptables \
+    lib${python_package} \
+    linux-generic \
+    logrotate \
+    $mysql_package \
+    nfs-common \
+    $nginx_package \
+    $ntpd_package \
+    openssh-server \
+    pwgen \
+    resolvconf \
+    sshfs \
+    swaks \
+    tzdata \
+    unattended-upgrades \
+    unbound \
+    unzip \
+    xfsprogs
+
+# on some providers like scaleway the sudo file is changed and we want to keep the old one
+apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends sudo
+
+# this ensures that unattended upgades are enabled, if it was disabled during ubuntu install time (see #346)
+# debconf-set-selection of unattended-upgrades/enable_auto_updates + dpkg-reconfigure does not work
+cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
+
+apt-get install -y --no-install-recommends $python_package   # Install python which is required for npm rebuild
+
+# do not upgrade grub because it might prompt user and break this script
+echo "==> Enable memory accounting"
+apt-get -y --no-upgrade --no-install-recommends install grub2-common
+sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
+update-grub
+
+echo "==> Install collectd"
+# without this, libnotify4 will install gnome-shell
+apt-get install -y libnotify4 libcurl3-gnutls --no-install-recommends
+# https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
+if [[ "${ubuntu_version}" == "22.04" ]]; then
+    readonly launchpad="https://launchpad.net/ubuntu/+source/collectd/5.12.0-9/+build/23189375/+files"
+    cd /tmp && wget -q "${launchpad}/collectd_5.12.0-9_amd64.deb" "${launchpad}/collectd-utils_5.12.0-9_amd64.deb" "${launchpad}/collectd-core_5.12.0-9_amd64.deb" "${launchpad}/libcollectdclient1_5.12.0-9_amd64.deb"
+    cd /tmp && apt install -y --no-install-recommends ./libcollectdclient1_5.12.0-9_amd64.deb ./collectd-core_5.12.0-9_amd64.deb ./collectd_5.12.0-9_amd64.deb ./collectd-utils_5.12.0-9_amd64.deb && rm -f /tmp/collectd_*.deb
+    echo -e "\nLD_PRELOAD=/usr/lib/python3.10/config-3.10-x86_64-linux-gnu/libpython3.10.so" >> /etc/default/collectd
+else
+    if ! apt-get install -y --no-install-recommends collectd collectd-utils; then
+        # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
+        echo "Failed to install collectd, continuing anyway. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
+    fi
+
+    if [[ "${ubuntu_version}" == "20.04" ]]; then
+        echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
+    fi
+fi
+sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
+
+# some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
+echo "==> Configuring host"
+sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
+if systemctl is-active ntp; then
+    systemctl stop ntp
+    apt purge -y ntp
+fi
+timedatectl set-ntp 1
+# mysql follows the system timezone
+timedatectl set-timezone UTC
+
+echo "==> Adding sshd configuration warning"
+sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://docs.cloudron.io/security/#securing-ssh-access' -i /etc/ssh/sshd_config
+
+# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
+echo "==> Disabling motd news"
+if [[ -f "/etc/default/motd-news" ]]; then
+    sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
+fi
+
+# If privacy extensions are not disabled on server, this breaks IPv6 detection
+# https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756
+if [[ ! -f /etc/sysctl.d/99-cloudimg-ipv6.conf ]]; then
+    echo "==> Disable temporary address (IPv6)"
+    echo -e "# See https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756\nnet.ipv6.conf.all.use_tempaddr = 0\nnet.ipv6.conf.default.use_tempaddr = 0\n\n" > /etc/sysctl.d/99-cloudimg-ipv6.conf
+fi
+
+# Disable exim4 (1blu.de)
+systemctl stop exim4 || true
+systemctl disable exim4 || true
+
+# Disable bind for good measure (on online.net, kimsufi servers these are pre-installed)
+systemctl stop bind9 || true
+systemctl disable bind9 || true
+
+# on ovh images dnsmasq seems to run by default
+systemctl stop dnsmasq || true
+systemctl disable dnsmasq || true
+
+# on ssdnodes postfix seems to run by default
+systemctl stop postfix || true
+systemctl disable postfix || true
+
+# on ubuntu 18.04 and 20.04, this is the default. this requires resolvconf for DNS to work further after the disable
+systemctl stop systemd-resolved || true
+systemctl disable systemd-resolved || true
+
+# on vultr, ufw is enabled by default. we have our own firewall
+ufw disable || true
+
+# we need unbound to work as this is required for installer.sh to do any DNS requests
+echo -e "server:\n\tinterface: 127.0.0.1\n" > /etc/unbound/unbound.conf.d/cloudron-network.conf
+systemctl restart unbound
+
+# Ubuntu 22 has private home directories by default (https://discourse.ubuntu.com/t/private-home-directories-for-ubuntu-21-04-onwards/)
+sed -e 's/^HOME_MODE\([[:space:]]\+\).*$/HOME_MODE\10755/' -i /etc/login.defs
+
+# create the yellowtent user. system user has different numeric range, no age and won't show in login/gdm UI
+# the nologin will also disable su/login
+if ! id yellowtent 2>/dev/null; then
+    useradd --system --comment "Cloudron Box" --create-home --shell /usr/sbin/nologin yellowtent
+fi
+
+# add support user (no password, sudo)
+if ! id cloudron-support 2>/dev/null; then
+    useradd --system --comment "Cloudron Support (support@cloudron.io)" --create-home --no-user-group --shell /bin/bash cloudron-support
+fi
+
@@ -15,6 +15,48 @@ function log() {
  echo -e "$(date +'%Y-%m-%dT%H:%M:%S')" "==> installer: $1"
 }

+apt_ready="no"
+function prepare_apt_once() {
+    [[ "${apt_ready}" == "yes" ]] && return
+
+    log "Making sure apt is in a good state"
+
+    log "Waiting for all dpkg tasks to finish..."
+    while fuser /var/lib/dpkg/lock; do
+        sleep 1
+    done
+
+    # it's unclear what needs to be run first or whether both these command should be run. so keep trying both
+    for count in {1..3}; do
+        # alternative to apt-install -y --fix-missing ?
+        if ! dpkg --force-confold --configure -a; then
+            log "dpkg reconfigure failed (try $count)"
+            dpkg_configure="no"
+        else
+            dpkg_configure="yes"
+        fi
+
+        if ! apt update -y; then
+            log "apt update failed (try $count)"
+            apt_update="no"
+        else
+            apt_update="yes"
+        fi
+
+        [[ "${dpkg_configure}" == "yes" && "${apt_update}" == "yes" ]] && break
+
+        sleep 1
+    done
+
+    apt_ready="yes"
+
+    if [[ "${dpkg_configure}" == "yes" && "${apt_update}" == "yes" ]]; then
+        log "apt is ready"
+    else
+        log "apt is not ready but proceeding anyway"
+    fi
+}
+
 readonly user=yellowtent
 readonly box_src_dir=/home/${user}/box

@@ -27,71 +69,62 @@ readonly ubuntu_codename=$(lsb_release -cs)

 readonly is_update=$(systemctl is-active -q box && echo "yes" || echo "no")

-log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION)"
+log "Updating from $(cat $box_src_dir/VERSION 2>/dev/null) to $(cat $box_src_tmp_dir/VERSION 2>/dev/null)"

-log "updating docker"
+# https://docs.docker.com/engine/installation/linux/ubuntulinux/
+readonly docker_version=20.10.14
+if ! which docker 2>/dev/null || [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
+    log "installing/updating docker"
+
+    # create systemd drop-in file already to make sure images are with correct driver
+    mkdir -p /etc/systemd/system/docker.service.d
+    echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf

-readonly docker_version=20.10.3
-if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
    # there are 3 packages for docker - containerd, CLI and the daemon
-    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.5.11-1_amd64.deb" -o /tmp/containerd.deb
    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb

-    log "Waiting for all dpkg tasks to finish..."
-    while fuser /var/lib/dpkg/lock; do
-        sleep 1
-    done
-
-    while ! dpkg --force-confold --configure -a; do
-        log "Failed to fix packages. Retry"
-        sleep 1
-    done
-
-    # the latest docker might need newer packages
-    while ! apt update -y; do
-        log "Failed to update packages. Retry"
-        sleep 1
-    done
-
-    while ! apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb; do
-        log "Failed to install docker. Retry"
-        sleep 1
-    done
-
+    log "installing docker"
+    prepare_apt_once
+    apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
    rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
 fi

+# we want atleast nginx 1.14 for TLS v1.3 support. Ubuntu 20/22 already has nginx 1.18
+# Ubuntu 18 OpenSSL does not have TLS v1.3 support, so we use the upstream nginx packages
 readonly nginx_version=$(nginx -v 2>&1)
-if [[ "${nginx_version}" != *"1.18."* ]]; then
-    log "installing nginx 1.18"
-    $curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
-    # apt install with install deps (as opposed to dpkg -i)
-    apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
-    rm /tmp/nginx.deb
+if [[ "${ubuntu_version}" == "20.04" ]]; then
+    if [[ "${nginx_version}" == *"Ubuntu"* ]]; then
+        log "switching nginx to ubuntu package"
+        prepare_apt_once
+        apt remove -y nginx
+        apt install -y nginx-full
+    fi
+elif [[ "${ubuntu_version}" == "18.04" ]]; then
+    if [[ "${nginx_version}" != *"1.18."* ]]; then
+        log "installing/updating nginx 1.18"
+        $curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
+
+        prepare_apt_once
+
+        # apt install with install deps (as opposed to dpkg -i)
+        apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
+        rm /tmp/nginx.deb
+    fi
 fi

-if ! which mount.nfs; then
-    log "installing nfs-common"
-    apt install -y nfs-common
-fi
-
-if ! which sshfs; then
-    log "installing sshfs"
-    apt install -y sshfs
-fi
-
-log "updating node"
-readonly node_version=14.15.4
-if [[ "$(node --version)" != "v${node_version}" ]]; then
+readonly node_version=16.14.2
+if ! which node 2>/dev/null || [[ "$(node --version)" != "v${node_version}" ]]; then
+    log "installing/updating node ${node_version}"
    mkdir -p /usr/local/node-${node_version}
    $curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-${node_version}
    ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
    ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-10.18.1
+    rm -rf /usr/local/node-16.13.1
 fi

-# this is here (and not in updater.js) because rebuild requires the above node
+# note that rebuild requires the above node
 for try in `seq 1 10`; do
    # for reasons unknown, the dtrace package will fail. but rebuilding second time will work

@@ -109,15 +142,21 @@ if [[ ${try} -eq 10 ]]; then
 fi

 log "downloading new addon images"
-images=$(node -e "var i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
+images=$(node -e "let i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")

 log "\tPulling docker images: ${images}"
+if ! curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip; then
+    docker_registry=registry.ipv6.docker.com
+else
+    docker_registry=registry-1.docker.io
+fi
+
 for image in ${images}; do
-    while ! docker pull "${image}"; do           # this pulls the image using the sha256
+    while ! docker pull "${docker_registry}/${image}"; do           # this pulls the image using the sha256
        log "Could not pull ${image}"
        sleep 5
    done
-    while ! docker pull "${image%@sha256:*}"; do  # this will tag the image for readability
+    while ! docker pull "${docker_registry}/${image%@sha256:*}"; do  # this will tag the image for readability
        log "Could not pull ${image%@sha256:*}"
        sleep 5
   done
@@ -126,19 +165,25 @@ done
 log "update cloudron-syslog"
 CLOUDRON_SYSLOG_DIR=/usr/local/cloudron-syslog
 CLOUDRON_SYSLOG="${CLOUDRON_SYSLOG_DIR}/bin/cloudron-syslog"
-CLOUDRON_SYSLOG_VERSION="1.0.3"
+CLOUDRON_SYSLOG_VERSION="1.1.0"
 while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLOUDRON_SYSLOG_VERSION} ]]; do
    rm -rf "${CLOUDRON_SYSLOG_DIR}"
    mkdir -p "${CLOUDRON_SYSLOG_DIR}"
-    if npm install --unsafe-perm -g --prefix "${CLOUDRON_SYSLOG_DIR}" cloudron-syslog@${CLOUDRON_SYSLOG_VERSION}; then break; fi
+    # verbatim is not needed in node 18 since that is the default there. in node 16, ipv4 is preferred and this breaks on ipv6 only servers
+    if NODE_OPTIONS="--dns-result-order=verbatim" npm install --unsafe-perm -g --prefix "${CLOUDRON_SYSLOG_DIR}" cloudron-syslog@${CLOUDRON_SYSLOG_VERSION}; then break; fi
    log "Failed to install cloudron-syslog, trying again"
    sleep 5
 done

-if ! id "${user}" 2>/dev/null; then
-    useradd "${user}" -m
+log "creating cloudron-support user"
+if ! id cloudron-support 2>/dev/null; then
+    useradd --system --comment "Cloudron Support (support@cloudron.io)" --create-home --no-user-group --shell /bin/bash cloudron-support
 fi

+log "locking the ${user} account"
+usermod --shell /usr/sbin/nologin "${user}"
+passwd --lock "${user}"
+
 if [[ "${is_update}" == "yes" ]]; then
    log "stop box service for update"
    ${box_src_dir}/setup/stop.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly logfile="/home/yellowtent/platformdata/logs/box.log"
+
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
+echo "This will re-create all the containers. Services will go down for a bit."
+
+read -p "Do you want to proceed? (y/N) " -n 1 -r choice
+echo
+
+if [[ ! $choice =~ ^[Yy]$ ]]; then
+    exit 1
+fi
+
+echo -n "Re-creating addon containers (this takes a while) ."
+line_count=$(cat /home/yellowtent/platformdata/logs/box.log | wc -l)
+sed -e 's/"version": ".*",/"version":"48.0.0",/' -i /home/yellowtent/platformdata/INFRA_VERSION
+systemctl restart box
+
+while ! tail -n "+${line_count}" "${logfile}" | grep -q "platform is ready"; do
+    echo -n "."
+    sleep 2
+done
+
+echo -e "\nDone.\nThe Cloudron dashboard will say 'Configuring (Queued)' for each app. The apps will come up in a short while."
+
@@ -16,7 +16,7 @@ readonly HOME_DIR="/home/${USER}"
 readonly BOX_SRC_DIR="${HOME_DIR}/box"
 readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata"
 readonly APPS_DATA_DIR="${HOME_DIR}/appsdata"
-readonly BOX_DATA_DIR="${HOME_DIR}/boxdata"
+readonly BOX_DATA_DIR="${HOME_DIR}/boxdata/box"
 readonly MAIL_DATA_DIR="${HOME_DIR}/boxdata/mail"

 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -37,12 +37,17 @@ systemctl enable apparmor
 systemctl restart apparmor

 usermod ${USER} -a -G docker
-# unbound (which starts after box code) relies on this interface to exist. dockerproxy also relies on this.
-docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
+
+if ! grep -q ip6tables /etc/systemd/system/docker.service.d/cloudron.conf; then
+    log "Adding ip6tables flag to docker" # https://github.com/moby/moby/pull/41622
+    echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
+    systemctl daemon-reload
+    systemctl restart docker
+fi

 mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
-mkdir -p "${MAIL_DATA_DIR}/dkim"
+mkdir -p "${MAIL_DATA_DIR}"

 # keep these in sync with paths.js
 log "Ensuring directories"
@@ -52,8 +57,9 @@ mkdir -p "${PLATFORM_DATA_DIR}/mysql"
 mkdir -p "${PLATFORM_DATA_DIR}/postgresql"
 mkdir -p "${PLATFORM_DATA_DIR}/mongodb"
 mkdir -p "${PLATFORM_DATA_DIR}/redis"
-mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner"
-mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
+mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner" \
+         "${PLATFORM_DATA_DIR}/addons/mail/dkim"
+mkdir -p "${PLATFORM_DATA_DIR}/collectd"
 mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
 mkdir -p "${PLATFORM_DATA_DIR}/acme"
 mkdir -p "${PLATFORM_DATA_DIR}/backup"
@@ -65,15 +71,13 @@ mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
 mkdir -p "${PLATFORM_DATA_DIR}/update"
 mkdir -p "${PLATFORM_DATA_DIR}/sftp/ssh" # sftp keys
 mkdir -p "${PLATFORM_DATA_DIR}/firewall"
+mkdir -p "${PLATFORM_DATA_DIR}/sshfs"
+mkdir -p "${PLATFORM_DATA_DIR}/cifs"

 # ensure backups folder exists and is writeable
 mkdir -p /var/backups
 chmod 777 /var/backups

-# can be removed after 6.3
-[[ -f "${BOX_DATA_DIR}/updatechecker.json" ]] && mv "${BOX_DATA_DIR}/updatechecker.json" "${PLATFORM_DATA_DIR}/update/updatechecker.json"
-rm -rf "${BOX_DATA_DIR}/well-known"
-
 log "Configuring journald"
 sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
    -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
@@ -84,23 +88,19 @@ sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
 sed -e "s/^WatchdogSec=.*$/WatchdogSec=3min/" \
    -i /lib/systemd/system/systemd-journald.service

-# Give user access to system logs
-usermod -a -G systemd-journal ${USER}
-mkdir -p /var/log/journal  # in some images, this directory is not created making system log to /run/systemd instead
-chown root:systemd-journal /var/log/journal
+usermod -a -G systemd-journal ${USER} # Give user access to system logs
+if [[ ! -d /var/log/journal ]]; then # in some images, this directory is not created making system log to /run/systemd instead
+    mkdir -p /var/log/journal
+    chown root:systemd-journal /var/log/journal
+    chmod g+s /var/log/journal  # sticky bit for group propagation
+fi
 systemctl daemon-reload
 systemctl restart systemd-journald
-setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal

 # Give user access to nginx logs (uses adm group)
 usermod -a -G adm ${USER}

 log "Setting up unbound"
-# DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org)
-# We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
-# We listen on 0.0.0.0 because there is no way control ordering of docker (which creates the 172.18.0.0/16) and unbound
-# If IP6 is not enabled, dns queries seem to fail on some hosts. -s returns false if file missing or 0 size
-ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
 cp -f "${script_dir}/start/unbound.conf" /etc/unbound/unbound.conf.d/cloudron-network.conf
 # update the root anchor after a out-of-disk-space situation (see #269)
 unbound-anchor -a /var/lib/unbound/root.key
@@ -116,7 +116,7 @@ systemctl enable box
 systemctl enable cloudron-firewall
 systemctl enable --now cloudron-disable-thp

-# update firewall rules
+# update firewall rules. this must be done after docker created it's rules
 systemctl restart cloudron-firewall

 # For logrotate
@@ -129,26 +129,28 @@ systemctl restart unbound
 systemctl restart cloudron-syslog

 log "Configuring sudoers"
-rm -f /etc/sudoers.d/${USER}
-cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER}
+rm -f /etc/sudoers.d/${USER} /etc/sudoers.d/cloudron
+cp "${script_dir}/start/sudoers" /etc/sudoers.d/cloudron

 log "Configuring collectd"
-rm -rf /etc/collectd /var/log/collectd.log
+rm -rf /etc/collectd /var/log/collectd.log "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
-if [[ "${ubuntu_version}" == "20.04" ]]; then
-    # https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
-    if ! grep -q LD_PRELOAD /etc/default/collectd; then
-        echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
-    fi
-fi
 systemctl restart collectd

+log "Configuring sysctl"
+# If privacy extensions are not disabled on server, this breaks IPv6 detection
+# https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756
+if [[ ! -f /etc/sysctl.d/99-cloudimg-ipv6.conf ]]; then
+    echo "==> Disable temporary address (IPv6)"
+    echo -e "# See https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756\nnet.ipv6.conf.all.use_tempaddr = 0\nnet.ipv6.conf.default.use_tempaddr = 0\n\n" > /etc/sysctl.d/99-cloudimg-ipv6.conf
+    sysctl -p
+fi
+
 log "Configuring logrotate"
 if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then
    echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf
 fi
-rm -f "${PLATFORM_DATA_DIR}/logrotate.d/"*
 cp "${script_dir}/start/logrotate/"* "${PLATFORM_DATA_DIR}/logrotate.d/"

 # logrotate files have to be owned by root, this is here to fixup existing installations where we were resetting the owner to yellowtent
@@ -172,7 +174,7 @@ fi

 # worker_rlimit_nofile in nginx config can be max this number
 mkdir -p /etc/systemd/system/nginx.service.d
-if ! grep -q "^LimitNOFILE=" /etc/systemd/system/nginx.service.d/cloudron.conf; then
+if ! grep -q "^LimitNOFILE=" /etc/systemd/system/nginx.service.d/cloudron.conf 2>/dev/null; then
    echo -e "[Service]\nLimitNOFILE=16384\n" > /etc/systemd/system/nginx.service.d/cloudron.conf
 fi

@@ -207,7 +209,8 @@ done

 readonly mysql_root_password="password"
 mysqladmin -u root -ppassword password password # reset default root password
-if [[ "${ubuntu_version}" == "20.04" ]]; then
+readonly mysqlVersion=$(mysql -NB -u root -p${mysql_root_password} -e 'SELECT VERSION()' 2>/dev/null)
+if [[ "${mysqlVersion}" == "8.0."* ]]; then
    # mysql 8 added a new caching_sha2_password scheme which mysqljs does not support
    mysql -u root -p${mysql_root_password} -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '${mysql_root_password}';"
 fi
@@ -228,16 +231,14 @@ log "Changing ownership"
 # note, change ownership after db migrate. this allow db migrate to move files around as root and then we can fix it up here
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
 chown -R "${USER}" /etc/cloudron
-chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall"
+chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall" "${PLATFORM_DATA_DIR}/sshfs" "${PLATFORM_DATA_DIR}/cifs"
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 chown "${USER}:${USER}" "${APPS_DATA_DIR}"

-# do not chown the boxdata/mail directory; dovecot gets upset
-chown "${USER}:${USER}" "${BOX_DATA_DIR}"
-find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${MAIL_DATA_DIR}" -exec chown -R "${USER}:${USER}" {} \;
+chown "${USER}:${USER}" -R "${BOX_DATA_DIR}"
+# do not chown the boxdata/mail directory entirely; dovecot gets upset
 chown "${USER}:${USER}" "${MAIL_DATA_DIR}"
-chown "${USER}:${USER}" -R "${MAIL_DATA_DIR}/dkim" # this is owned by box currently since it generates the keys

 log "Starting Cloudron"
 systemctl start box
@@ -3,100 +3,154 @@
 set -eu -o pipefail

 echo "==> Setting up firewall"
-iptables -t filter -N CLOUDRON || true
-iptables -t filter -F CLOUDRON # empty any existing rules
+
+has_ipv6=$(cat /proc/net/if_inet6 >/dev/null 2>&1 && echo "yes" || echo "no")
+
+# wait for 120 seconds for xtables lock, checking every 1 second
+readonly iptables="iptables --wait 120 --wait-interval 1"
+readonly ip6tables="ip6tables --wait 120 --wait-interval 1"
+
+function ipxtables() {
+    $iptables "$@"
+    [[ "${has_ipv6}" == "yes" ]] && $ip6tables "$@"
+}
+
+ipxtables -t filter -N CLOUDRON || true
+ipxtables -t filter -F CLOUDRON # empty any existing rules

 # first setup any user IP block lists
 ipset create cloudron_blocklist hash:net || true
+ipset create cloudron_blocklist6 hash:net family inet6 || true
+
 /home/yellowtent/box/src/scripts/setblocklist.sh

-iptables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist src -j DROP
+$iptables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist src -j DROP
 # the DOCKER-USER chain is not cleared on docker restart
-if ! iptables -t filter -C DOCKER-USER -m set --match-set cloudron_blocklist src -j DROP; then
-    iptables -t filter -I DOCKER-USER 1 -m set --match-set cloudron_blocklist src -j DROP
+if ! $iptables -t filter -C DOCKER-USER -m set --match-set cloudron_blocklist src -j DROP; then
+    $iptables -t filter -I DOCKER-USER 1 -m set --match-set cloudron_blocklist src -j DROP
 fi

+$ip6tables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist6 src -j DROP
+# there is no DOCKER-USER chain in ip6tables, bug?
+$ip6tables -D FORWARD -m set --match-set cloudron_blocklist6 src -j DROP || true
+$ip6tables -I FORWARD 1 -m set --match-set cloudron_blocklist6 src -j DROP
+
 # allow related and establisted connections
-iptables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
-iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,443 -j ACCEPT # 202 is the alternate ssh port
+ipxtables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,443 -j ACCEPT # 202 is the alternate ssh port

 # whitelist any user ports. we used to use --dports but it has a 15 port limit (XT_MULTI_PORTS)
 ports_json="/home/yellowtent/platformdata/firewall/ports.json"
-if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(','))" 2>/dev/null); then
-    IFS=',' arr=(${allowed_tcp_ports})
-    for p in "${arr[@]}"; do
-        iptables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
+if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(' '))" 2>/dev/null); then
+    for p in $allowed_tcp_ports; do
+        ipxtables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
    done
 fi

-if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(','))" 2>/dev/null); then
-    IFS=',' arr=(${allowed_udp_ports})
-    for p in "${arr[@]}"; do
-        iptables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
+if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(' '))" 2>/dev/null); then
+    for p in $allowed_udp_ports; do
+        ipxtables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
    done
 fi

+# LDAP user directory allow list
+ipset create cloudron_ldap_allowlist hash:net || true
+ipset flush cloudron_ldap_allowlist
+
+ipset create cloudron_ldap_allowlist6 hash:net family inet6 || true
+ipset flush cloudron_ldap_allowlist6
+
+ldap_allowlist_json="/home/yellowtent/platformdata/firewall/ldap_allowlist.txt"
+# delete any existing redirect rule
+$iptables -t nat -D PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004 2>/dev/null || true
+$ip6tables -t nat -D PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004 >/dev/null || true
+if [[ -f "${ldap_allowlist_json}" ]]; then
+    # without the -n block, any last line without a new line won't be read it!
+    while read -r line || [[ -n "$line" ]]; do
+        [[ -z "${line}" ]] && continue # ignore empty lines
+        [[ "$line" =~ ^#.*$ ]] && continue # ignore lines starting with #
+        if [[ "$line" == *":"* ]]; then
+            ipset add -! cloudron_ldap_allowlist6 "${line}" # the -! ignore duplicates
+        else
+            ipset add -! cloudron_ldap_allowlist "${line}" # the -! ignore duplicates
+        fi
+    done < "${ldap_allowlist_json}"
+
+    # ldap server we expose 3004 and also redirect from standard ldaps port 636
+    $iptables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
+    $iptables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist src -p tcp --dport 3004 -j ACCEPT
+
+    $ip6tables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
+    $ip6tables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist6 src -p tcp --dport 3004 -j ACCEPT
+fi
+
 # turn and stun service
-iptables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
+ipxtables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT

-iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
-iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT
-iptables -t filter -A CLOUDRON -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
-iptables -t filter -A CLOUDRON -i lo -j ACCEPT # required for localhost connections (mysql)
+# ICMPv6 is very fundamental to IPv6 connectivity unlike ICMPv4
+$iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
+$iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
+$ip6tables -t filter -A CLOUDRON -p ipv6-icmp -j ACCEPT
+
+ipxtables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT
+$iptables -t filter -A CLOUDRON -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+ipxtables -t filter -A CLOUDRON -i lo -j ACCEPT # required for localhost connections (mysql)

 # log dropped incoming. keep this at the end of all the rules
-iptables -t filter -A CLOUDRON -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
-iptables -t filter -A CLOUDRON -j DROP
+ipxtables -t filter -A CLOUDRON -m limit --limit 2/min -j LOG --log-prefix "Packet dropped: " --log-level 7
+ipxtables -t filter -A CLOUDRON -j DROP

-if ! iptables -t filter -C INPUT -j CLOUDRON 2>/dev/null; then
-    iptables -t filter -I INPUT -j CLOUDRON
-fi
+# prepend our chain to the filter table
+$iptables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $iptables -t filter -I INPUT -j CLOUDRON
+$ip6tables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $ip6tables -t filter -I INPUT -j CLOUDRON

 # Setup rate limit chain (the recent info is at /proc/net/xt_recent)
-iptables -t filter -N CLOUDRON_RATELIMIT || true
-iptables -t filter -F CLOUDRON_RATELIMIT # empty any existing rules
+ipxtables -t filter -N CLOUDRON_RATELIMIT || true
+ipxtables -t filter -F CLOUDRON_RATELIMIT # empty any existing rules

 # log dropped incoming. keep this at the end of all the rules
-iptables -t filter -N CLOUDRON_RATELIMIT_LOG || true
-iptables -t filter -F CLOUDRON_RATELIMIT_LOG # empty any existing rules
-iptables -t filter -A CLOUDRON_RATELIMIT_LOG -m limit --limit 2/min -j LOG --log-prefix "IPTables RateLimit: " --log-level 7
-iptables -t filter -A CLOUDRON_RATELIMIT_LOG -j DROP
+ipxtables -t filter -N CLOUDRON_RATELIMIT_LOG || true
+ipxtables -t filter -F CLOUDRON_RATELIMIT_LOG # empty any existing rules
+ipxtables -t filter -A CLOUDRON_RATELIMIT_LOG -m limit --limit 2/min -j LOG --log-prefix "IPTables RateLimit: " --log-level 7
+ipxtables -t filter -A CLOUDRON_RATELIMIT_LOG -j DROP

 # http https
 for port in 80 443; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done

 # ssh
 for port in 22 202; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
-    iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
+done
+
+# ldaps
+for port in 636 3004; do
+    ipxtables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done

 # docker translates (dnat) 25, 587, 993, 4190 in the PREROUTING step
 for port in 2525 4190 9993; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
+    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
 done

-# msa, ldap, imap, sieve
-for port in 2525 3002 4190 9993; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
+# msa, ldap, imap, sieve, pop3
+for port in 2525 3002 4190 9993 9995; do
+    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
 done

 # cloudron docker network: mysql postgresql redis mongodb
 for port in 3306 5432 6379 27017; do
-    iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
+    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done

-if ! iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null; then
-    iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
-fi
+# Add the rate limit chain to input chain
+$iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
+$ip6tables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $ip6tables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT

 # Workaround issue where Docker insists on adding itself first in FORWARD table
-iptables -D FORWARD -j CLOUDRON_RATELIMIT || true
-iptables -I FORWARD 1 -j CLOUDRON_RATELIMIT
-
-echo "==> Setting up firewall done"
+ipxtables -D FORWARD -j CLOUDRON_RATELIMIT || true
+ipxtables -I FORWARD 1 -j CLOUDRON_RATELIMIT
@@ -4,26 +4,51 @@

 printf "**********************************************************************\n\n"

-if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
-    if [[ -f /tmp/.cloudron-motd-cache ]]; then
-        ip=$(cat /tmp/.cloudron-motd-cache)
-    elif ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
-        ip='<IP>'
+readonly cache_file4="/var/cache/cloudron-motd-cache4"
+readonly cache_file6="/var/cache/cloudron-motd-cache6"
+
+url4=""
+url6=""
+fallbackUrl=""
+
+function detectIp() {
+    if [[ ! -f "${cache_file4}" ]]; then
+        ip4=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
+        [[ -n "${ip4}" ]] && echo "${ip4}" > "${cache_file4}"
+    else
+        ip4=$(cat "${cache_file4}")
+    fi
+
+    if [[ ! -f "${cache_file6}" ]]; then
+        ip6=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv6.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p' || true)
+        [[ -n "${ip6}" ]] && echo "${ip6}" > "${cache_file6}"
+    else
+        ip6=$(cat "${cache_file6}")
    fi
-    echo "${ip}" > /tmp/.cloudron-motd-cache

    if [[ ! -f /etc/cloudron/SETUP_TOKEN ]]; then
-        url="https://${ip}"
+        [[ -n "${ip4}" ]] && url4="https://${ip4}"
+        [[ -n "${ip6}" ]] && url6="https://[${ip6}]"
+        [[ -z "${ip4}" && -z "${ip6}" ]] && fallbackUrl="https://<IP>"
    else
        setupToken="$(cat /etc/cloudron/SETUP_TOKEN)"
-        url="https://${ip}/?setupToken=${setupToken}"
+        [[ -n "${ip4}" ]] && url4="https://${ip4}/?setupToken=${setupToken}"
+        [[ -n "${ip6}" ]] && url6="https://[${ip6}]/?setupToken=${setupToken}"
+        [[ -z "${ip4}" && -z "${ip6}" ]] && fallbackUrl="https://<IP>?setupToken=${setupToken}"
    fi
+}
+
+if [[ -z "$(ls -A /home/yellowtent/platformdata/addons/mail/dkim)" ]]; then
+    detectIp

    printf "\t\t\tWELCOME TO CLOUDRON\n"
    printf "\t\t\t-------------------\n"

-    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit ${url} on your browser and accept the self-signed certificate to finish setup."
-    printf "Cloudron overview - https://docs.cloudron.io/ \n"
+    printf '\n\e[1;32m%-6s\e[m\n' "Visit one of the following URLs on your browser and accept the self-signed certificate to finish setup."
+    [[ -n "${url4}" ]] && printf '\e[1;32m%-6s\e[m\n' "  * ${url4}"
+    [[ -n "${url6}" ]] && printf '\e[1;32m%-6s\e[m\n' "  * ${url6}"
+    [[ -n "${fallbackUrl}" ]] && printf '\e[1;32m%-6s\e[m\n' "  * ${fallbackUrl}"
+    printf "\nCloudron overview - https://docs.cloudron.io/ \n"
    printf "Cloudron setup - https://docs.cloudron.io/installation/#setup \n"
 else
    printf "\t\t\tNOTE TO CLOUDRON ADMINS\n"
@@ -13,7 +13,7 @@
 ##############################################################################

 Hostname "localhost"
-#FQDNLookup true
+FQDNLookup false
 #BaseDir "/var/lib/collectd"
 #PluginDir "/usr/lib/collectd"
 #TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db"
@@ -187,9 +187,9 @@ LoadPlugin swap

       CalculateNum false
       CalculateSum true
-       CalculateAverage true
+       CalculateAverage false
       CalculateMinimum false
-       CalculateMaximum true
+       CalculateMaximum false
       CalculateStddev false
   </Aggregation>
 </Plugin>
@@ -211,28 +211,12 @@ LoadPlugin swap
    Interactive false

    Import "df"
-
-    Import "du"
-    <Module du>
-        <Path>
-            Instance maildata
-            Dir "/home/yellowtent/boxdata/mail"
-        </Path>
-        <Path>
-            Instance boxdata
-            Dir "/home/yellowtent/boxdata"
-            Exclude "mail"
-        </Path>
-        <Path>
-            Instance platformdata
-            Dir "/home/yellowtent/platformdata"
-        </Path>
-    </Module>
+    Import "docker-stats"
 </Plugin>

 <Plugin write_graphite>
   <Node "graphing">
-       Host "localhost"
+       Host "127.0.0.1"
       Port "2003"
       Protocol "tcp"
       LogSendErrors true
@@ -243,6 +227,3 @@ LoadPlugin swap
   </Node>
 </Plugin>

-<Include "/etc/collectd/collectd.conf.d">
-    Filter "*.conf"
-</Include>
@@ -14,7 +14,7 @@ def read():
    for d in disks:
        device = d[0]
        if 'devicemapper' in d[1] or not device.startswith('/dev/'): continue
-        instance = device[len('/dev/'):].replace('/', '_') # see #348
+        instance = device[len('/dev/'):].replace('/', '_').replace('.', '_') # see #348

        try:
            st = os.statvfs(d[1]) # handle disk removal
@@ -0,0 +1,64 @@
+import collectd,os,subprocess,json,re
+
+# https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/
+
+def parseSiSize(size):
+    units = {"B": 1, "KB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12}
+    number, unit, _ = re.split('([a-zA-Z]+)', size.upper())
+    return int(float(number)*units[unit])
+
+def parseBinarySize(size):
+    units = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "TIB": 2**40}
+    number, unit, _ = re.split('([a-zA-Z]+)', size.upper())
+    return int(float(number)*units[unit])
+
+def init():
+    collectd.info('custom docker-status plugin initialized')
+
+def read():
+    try:
+        lines = subprocess.check_output('docker stats --format  "{{ json . }}" --no-stream --no-trunc', shell=True).decode('utf-8').strip().split("\n")
+    except Exception as e:
+        collectd.info('\terror getting docker stats: %s' % (str(e)))
+        return 0
+
+    # Sample line
+    # {"BlockIO":"430kB / 676kB","CPUPerc":"0.00%","Container":"7eae5e6f4f11","ID":"7eae5e6f4f11","MemPerc":"59.15%","MemUsage":"45.55MiB / 77MiB","Name":"1062eef3-ec96-4d81-9f02-15b7dd81ccb9","NetIO":"1.5MB / 3.48MB","PIDs":"5"}
+
+    for line in lines:
+        stat = json.loads(line)
+        containerName = stat["Name"] # same as app id
+        networkData = stat["NetIO"].split("/")
+        networkRead = parseSiSize(networkData[0].strip())
+        networkWrite = parseSiSize(networkData[1].strip())
+
+        blockData = stat["BlockIO"].split("/")
+        blockRead = parseSiSize(blockData[0].strip())
+        blockWrite = parseSiSize(blockData[1].strip())
+
+        memUsageData = stat["MemUsage"].split("/")
+        memUsed = parseBinarySize(memUsageData[0].strip())
+        memMax = parseBinarySize(memUsageData[1].strip())
+
+        cpuPercData = stat["CPUPerc"].strip("%")
+        cpuPerc = float(cpuPercData)
+
+        # type comes from https://github.com/collectd/collectd/blob/master/src/types.db and https://collectd.org/wiki/index.php/Data_source
+        val = collectd.Values(type='gauge', plugin='docker-stats', plugin_instance=containerName)
+        val.dispatch(values=[networkRead], type_instance='network-read')
+        val.dispatch(values=[networkWrite], type_instance='network-write')
+        val.dispatch(values=[blockRead], type_instance='blockio-read')
+        val.dispatch(values=[blockWrite], type_instance='blockio-write')
+        val.dispatch(values=[memUsed], type_instance='mem-used')
+        val.dispatch(values=[memMax], type_instance='mem-max')
+        val.dispatch(values=[cpuPerc], type_instance='cpu-perc')
+
+        val = collectd.Values(type='counter', plugin='docker-stats', plugin_instance=containerName)
+        val.dispatch(values=[networkRead], type_instance='network-read')
+        val.dispatch(values=[networkWrite], type_instance='network-write')
+        val.dispatch(values=[blockRead], type_instance='blockio-read')
+        val.dispatch(values=[blockWrite], type_instance='blockio-write')
+
+collectd.register_init(init)
+# see Interval setting in collectd.conf for polling interval
+collectd.register_read(read)
@@ -1,105 +0,0 @@
-import collectd,os,subprocess,sys,re,time
-
-# https://www.programcreek.com/python/example/106897/collectd.register_read
-
-PATHS = [] # { name, dir, exclude }
-# there is a pattern in carbon/storage-schemas.conf which stores values every 12h for a year
-INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this
-
-# we used to pass the INTERVAL as a parameter to register_read. however, collectd write_graphite
-# takes a bit to load (tcp connection) and drops the du data. this then means that we have to wait
-# for INTERVAL secs for du data. instead, we just cache the value for INTERVAL instead
-CACHE = dict()
-CACHE_TIME = 0
-
-def du(pathinfo):
-    # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
-    dirname = pathinfo['dir']
-    cmd = 'timeout 1800 du -DsB1 "{}"'.format(dirname)
-    if pathinfo['exclude'] != '':
-        cmd += ' --exclude "{}"'.format(pathinfo['exclude'])
-
-    collectd.info('computing size with command: %s' % cmd);
-    try:
-        size = subprocess.check_output(cmd, shell=True).split()[0].decode('utf-8')
-        collectd.info('\tsize of %s is %s (time: %i)' % (dirname, size, int(time.time())))
-        return size
-    except Exception as e:
-        collectd.info('\terror getting the size of %s: %s' % (dirname, str(e)))
-        return 0
-
-def parseSize(size):
-    units = {"B": 1, "KB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12}
-    number, unit, _ = re.split('([a-zA-Z]+)', size.upper())
-    return int(float(number)*units[unit])
-
-def dockerSize():
-    # use --format '{{json .}}' to dump the string. '{{if eq .Type "Images"}}{{.Size}}{{end}}' still creates newlines
-    # https://godoc.org/github.com/docker/go-units#HumanSize is used. so it's 1000 (KB) and not 1024 (KiB)
-    cmd = 'timeout 1800 docker system df --format "{{.Size}}" | head -n1'
-    try:
-        size = subprocess.check_output(cmd, shell=True).strip().decode('utf-8')
-        collectd.info('size of docker images is %s (%s) (time: %i)' % (size, parseSize(size), int(time.time())))
-        return parseSize(size)
-    except Exception as e:
-        collectd.info('error getting docker images size : %s' % str(e))
-        return 0
-
-# configure is called for each module block. this is called before init
-def configure(config):
-    global PATHS
-
-    for child in config.children:
-        if child.key != 'Path':
-            collectd.info('du plugin: Unknown config key "%s"' % key)
-            continue
-
-        pathinfo = { 'name': '', 'dir': '', 'exclude': '' }
-        for node in child.children:
-            if node.key == 'Instance':
-                pathinfo['name'] = node.values[0]
-            elif node.key == 'Dir':
-                pathinfo['dir'] = node.values[0]
-            elif node.key == 'Exclude':
-                pathinfo['exclude'] = node.values[0]
-
-        PATHS.append(pathinfo);
-        collectd.info('du plugin: monitoring %s' % pathinfo['dir']);
-
-def init():
-    global PATHS
-    collectd.info('custom du plugin initialized with %s %s' % (PATHS, sys.version))
-
-def read():
-    global CACHE, CACHE_TIME
-
-    # read from cache if < 12 hours
-    read_cache = (time.time() - CACHE_TIME) < INTERVAL
-
-    if not read_cache:
-        CACHE_TIME = time.time()
-
-    for pathinfo in PATHS:
-        dirname = pathinfo['dir']
-        if read_cache and dirname in CACHE:
-            size = CACHE[dirname]
-        else:
-            size = du(pathinfo)
-            CACHE[dirname] = size
-
-        # type comes from https://github.com/collectd/collectd/blob/master/src/types.db
-        val = collectd.Values(type='capacity', plugin='du', plugin_instance=pathinfo['name'])
-        val.dispatch(values=[size], type_instance='usage')
-
-    if read_cache and 'docker' in CACHE:
-        size = CACHE['docker']
-    else:
-        size = dockerSize()
-        CACHE['docker'] = size
-
-    val = collectd.Values(type='capacity', plugin='du', plugin_instance='docker')
-    val.dispatch(values=[size], type_instance='usage')
-
-collectd.register_init(init)
-collectd.register_config(configure)
-collectd.register_read(read)
@@ -19,15 +19,10 @@ http {
    include       mime.types;
    default_type  application/octet-stream;

-    # the collectd config depends on this log format
-    log_format combined2 '$remote_addr - [$time_local] '
-        '"$request" $status $body_bytes_sent $request_time '
-        '"$http_referer" "$host" "$http_user_agent"';
-
    # required for long host names
    server_names_hash_bucket_size 128;

-    access_log /var/log/nginx/access.log combined2;
+    access_log /var/log/nginx/access.log combined;

    sendfile        on;

@@ -1,6 +1,9 @@
 # sudo logging breaks journalctl output with very long urls (systemd bug)
 Defaults !syslog

+Defaults!/home/yellowtent/box/src/scripts/checkvolume.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/checkvolume.sh
+
 Defaults!/home/yellowtent/box/src/scripts/clearvolume.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/clearvolume.sh

@@ -16,9 +19,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmaddondir.sh
 Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh

-Defaults!/home/yellowtent/box/src/scripts/configurecollectd.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/configurecollectd.sh
-
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh

@@ -53,9 +53,19 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/stoptask.sh
 Defaults!/home/yellowtent/box/src/scripts/setblocklist.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setblocklist.sh

+Defaults!/home/yellowtent/box/src/scripts/setldapallowlist.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setldapallowlist.sh
+
 Defaults!/home/yellowtent/box/src/scripts/addmount.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/addmount.sh

 Defaults!/home/yellowtent/box/src/scripts/rmmount.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmount.sh

+Defaults!/home/yellowtent/box/src/scripts/remountmount.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/remountmount.sh
+
+Defaults!/home/yellowtent/box/src/scripts/du.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/du.sh
+
+cloudron-support ALL=(ALL) NOPASSWD: ALL
@@ -2,7 +2,7 @@

 [Unit]
 Description=Unbound DNS Resolver
-After=network-online.target docker.service
+After=network-online.target
 Before=nss-lookup.target
 Wants=network-online.target nss-lookup.target

@@ -1,8 +1,12 @@
+# Unbound is used primarily for RBL queries (host 2.0.0.127.zen.spamhaus.org)
+# We cannot use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
+
 server:
        port: 53
        interface: 127.0.0.1
        interface: 172.18.0.1
-        do-ip6: no
+        ip-freebind: yes
+        do-ip6: yes
        access-control: 127.0.0.1 allow
        access-control: 172.18.0.1/16 allow
        cache-max-negative-ttl: 30
@@ -10,4 +14,3 @@ server:
        # enable below for logging to journalctl -u unbound
        # verbosity: 5
        # log-queries: yes
-
@@ -1,31 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    verifyToken
-};
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    safe = require('safetydance'),
-    tokens = require('./tokens.js'),
-    users = require('./users.js'),
-    util = require('util');
-
-const userGet = util.promisify(users.get);
-
-async function verifyToken(accessToken) {
-    assert.strictEqual(typeof accessToken, 'string');
-
-    const token = await tokens.getByAccessToken(accessToken);
-    if (!token) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'No such token');
-
-    const [error, user] = await safe(userGet(token.identifier));
-    if (error && error.reason === BoxError.NOT_FOUND) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not found');
-    if (error) throw error;
-
-    if (!user.active) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not active');
-
-    await safe(tokens.update(token.id, { lastUsedTime: new Date() })); // ignore any error
-
-    return user;
-}
@@ -9,11 +9,11 @@ exports = module.exports = {
 };

 const assert = require('assert'),
-    async = require('async'),
+    blobs = require('./blobs.js'),
    BoxError = require('./boxerror.js'),
    crypto = require('crypto'),
    debug = require('debug')('box:cert/acme2'),
-    domains = require('./domains.js'),
+    dns = require('./dns.js'),
    fs = require('fs'),
    os = require('os'),
    path = require('path'),
@@ -32,7 +32,7 @@ const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
 function Acme2(options) {
    assert.strictEqual(typeof options, 'object');

-    this.accountKeyPem = options.accountKeyPem; // Buffer
+    this.accountKeyPem = null; // Buffer .
    this.email = options.email;
    this.keyId = null;
    this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
@@ -47,16 +47,16 @@ function urlBase64Encode(string) {
 }

 function b64(str) {
-    var buf = Buffer.isBuffer(str) ? str : Buffer.from(str);
+    const buf = Buffer.isBuffer(str) ? str : Buffer.from(str);
    return urlBase64Encode(buf.toString('base64'));
 }

 function getModulus(pem) {
    assert(Buffer.isBuffer(pem));

-    var stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
+    const stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
    if (!stdout) return null;
-    var match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
+    const match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
    if (!match) return null;
    return Buffer.from(match[1], 'hex');
 }
@@ -88,10 +88,10 @@ Acme2.prototype.sendSignedRequest = async function (url, payload) {

    let [error, response] = await safe(superagent.get(this.directory.newNonce).timeout(30000).ok(() => true));
    if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`);
-    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code when fetching nonce : ${response.status}`);
+    if (response.status !== 204) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching nonce : ${response.status}`);

    const nonce = response.headers['Replay-Nonce'.toLowerCase()];
-    if (!nonce) throw new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response');
+    if (!nonce) throw new BoxError(BoxError.ACME_ERROR, 'No nonce in response');

    debug('sendSignedRequest: using nonce %s for url %s', nonce, url);

@@ -129,23 +129,43 @@ Acme2.prototype.updateContact = async function (registrationUri) {
    };

    const result = await this.sendSignedRequest(registrationUri, JSON.stringify(payload));
-    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
+    if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);

    debug(`updateContact: contact of user updated to ${this.email}`);
 };

-Acme2.prototype.registerUser = async function () {
+async function generateAccountKey() {
+    const acmeAccountKey = safe.child_process.execSync('openssl genrsa 4096');
+    if (!acmeAccountKey) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate acme account key: ${safe.error.message}`);
+    return acmeAccountKey;
+}
+
+Acme2.prototype.ensureAccount = async function () {
    const payload = {
        termsOfServiceAgreed: true
    };

-    debug('registerUser: registering user');
+    debug('ensureAccount: registering user');
+
+    this.accountKeyPem = await blobs.get(blobs.ACME_ACCOUNT_KEY);
+    if (!this.accountKeyPem) {
+        debug('ensureAccount: generating new account keys');
+        this.accountKeyPem = await generateAccountKey();
+        await blobs.set(blobs.ACME_ACCOUNT_KEY, this.accountKeyPem);
+    }
+
+    let result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
+    if (result.status === 403 && result.body.type === 'urn:ietf:params:acme:error:unauthorized') {
+        debug(`ensureAccount: key was revoked. ${result.status} ${JSON.stringify(result.body)}. generating new account key`);
+        this.accountKeyPem = await generateAccountKey();
+        await blobs.set(blobs.ACME_ACCOUNT_KEY, this.accountKeyPem);
+        result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
+    }

-    const result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
    // 200 if already exists. 201 for new accounts
-    if (result.status !== 200 && result.status !== 201) return new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${JSON.stringify(result.body)}`);
+    if (result.status !== 200 && result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${JSON.stringify(result.body)}`);

-    debug(`registerUser: user registered keyid: ${result.headers.location}`);
+    debug(`ensureAccount: user registered keyid: ${result.headers.location}`);

    this.keyId = result.headers.location;

@@ -166,15 +186,15 @@ Acme2.prototype.newOrder = async function (domain) {

    const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload));
    if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`);
-    if (result.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+    if (result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`);

    debug('newOrder: created order %s %j', domain, result.body);

    const order = result.body, orderUrl = result.headers.location;

-    if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid authorizations in order');
-    if (typeof order.finalize !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid finalize in order');
-    if (typeof orderUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid order location in order header');
+    if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.ACME_ERROR, 'invalid authorizations in order');
+    if (typeof order.finalize !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid finalize in order');
+    if (typeof orderUrl !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid order location in order header');

    return { order, orderUrl };
 };
@@ -184,20 +204,20 @@ Acme2.prototype.waitForOrder = async function (orderUrl) {

    debug(`waitForOrder: ${orderUrl}`);

-    return await promiseRetry({ times: 15, interval: 20000 }, async () => {
+    return await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
        debug('waitForOrder: getting status');

        const result = await this.postAsGet(orderUrl);
        if (result.status !== 200) {
            debug(`waitForOrder: invalid response code getting uri ${result.status}`);
-            throw new BoxError(BoxError.EXTERNAL_ERROR, `Bad response code: ${result.status}`);
+            throw new BoxError(BoxError.ACME_ERROR, `Bad response when waiting for order. code: ${result.status}`);
        }

        debug('waitForOrder: status is "%s %j', result.body.status, result.body);

-        if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.TRY_AGAIN, `Request is in ${result.body.status} state`);
+        if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.ACME_ERROR, `Request is in ${result.body.status} state`);
        else if (result.body.status === 'valid' && result.body.certificate) return result.body.certificate;
-        else throw new BoxError(BoxError.EXTERNAL_ERROR, `Unexpected status or invalid response: ${JSON.stringify(result.body)}`);
+        else throw new BoxError(BoxError.ACME_ERROR, `Unexpected status or invalid response when waiting for order: ${JSON.stringify(result.body)}`);
    });
 };

@@ -229,7 +249,7 @@ Acme2.prototype.notifyChallengeReady = async function (challenge) {
    };

    const result = await this.sendSignedRequest(challenge.url, JSON.stringify(payload));
-    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+    if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
 };

 Acme2.prototype.waitForChallenge = async function (challenge) {
@@ -237,20 +257,20 @@ Acme2.prototype.waitForChallenge = async function (challenge) {

    debug('waitingForChallenge: %j', challenge);

-    await promiseRetry({ times: 15, interval: 20000 }, async () => {
+    await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
        debug('waitingForChallenge: getting status');

        const result = await this.postAsGet(challenge.url);
        if (result.status !== 200) {
            debug(`waitForChallenge: invalid response code getting uri ${result.status}`);
-            throw new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode);
+            throw new BoxError(BoxError.ACME_ERROR, `Bad response code when waiting for challenge : ${result.status}`);
        }

        debug(`waitForChallenge: status is "${result.body.status}" "${JSON.stringify(result.body)}"`);

-        if (result.body.status === 'pending') throw new BoxError(BoxError.TRY_AGAIN);
+        if (result.body.status === 'pending') throw new BoxError(BoxError.ACME_ERROR, 'Challenge is in pending state');
        else if (result.body.status === 'valid') return;
-        else throw new BoxError(BoxError.EXTERNAL_ERROR, `Unexpected status: ${result.body.status}`);
+        else throw new BoxError(BoxError.ACME_ERROR, `Unexpected status when waiting for challenge: ${result.body.status}`);
    });
 };

@@ -268,7 +288,7 @@ Acme2.prototype.signCertificate = async function (domain, finalizationUrl, csrDe

    const result = await this.sendSignedRequest(finalizationUrl, JSON.stringify(payload));
    // 429 means we reached the cert limit for this domain
-    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
+    if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to sign certificate. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
 };

 Acme2.prototype.createKeyAndCsr = async function (hostname, keyFilePath, csrFilePath) {
@@ -304,7 +324,7 @@ Acme2.prototype.createKeyAndCsr = async function (hostname, keyFilePath, csrFile
    if (!csrDer) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
    if (!safe.fs.writeFileSync(csrFilePath, csrDer)) throw new BoxError(BoxError.FS_ERROR, safe.error); // bookkeeping. inspect with openssl req -text -noout -in hostname.csr -inform der

-    await safe(fs.promises.rmdir(tmpdir, { recursive: true }));
+    await safe(fs.promises.rm(tmpdir, { recursive: true, force: true }));

    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFilePath);

@@ -315,12 +335,12 @@ Acme2.prototype.downloadCertificate = async function (hostname, certUrl, certFil
    assert.strictEqual(typeof hostname, 'string');
    assert.strictEqual(typeof certUrl, 'string');

-    await promiseRetry({ times: 5, interval: 20000 }, async () => {
-        debug('downloadCertificate: downloading certificate');
+    await promiseRetry({ times: 5, interval: 20000, debug }, async () => {
+        debug(`downloadCertificate: downloading certificate of ${hostname}`);

        const result = await this.postAsGet(certUrl);
-        if (result.statusCode === 202) throw new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate');
-        if (result.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+        if (result.statusCode === 202) throw new BoxError(BoxError.ACME_ERROR, 'Retry downloading certificate');
+        if (result.statusCode !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);

        const fullChainPem = result.body; // buffer

@@ -338,7 +358,7 @@ Acme2.prototype.prepareHttpChallenge = async function (hostname, domain, authori

    debug('prepareHttpChallenge: challenges: %j', authorization);
    let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
-    if (httpChallenges.length === 0) throw new BoxError(BoxError.EXTERNAL_ERROR, 'no http challenges');
+    if (httpChallenges.length === 0) throw new BoxError(BoxError.ACME_ERROR, 'no http challenges');
    let challenge = httpChallenges[0];

    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
@@ -387,7 +407,7 @@ Acme2.prototype.prepareDnsChallenge = async function (hostname, domain, authoriz

    debug('prepareDnsChallenge: challenges: %j', authorization);
    const dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
-    if (dnsChallenges.length === 0) throw new BoxError(BoxError.EXTERNAL_ERROR, 'no dns challenges');
+    if (dnsChallenges.length === 0) throw new BoxError(BoxError.ACME_ERROR, 'no dns challenges');
    const challenge = dnsChallenges[0];

    const keyAuthorization = this.getKeyAuthorization(challenge.token);
@@ -399,17 +419,11 @@ Acme2.prototype.prepareDnsChallenge = async function (hostname, domain, authoriz

    debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);

-    return new Promise((resolve, reject) => {
-        domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-            if (error) return reject(error);
+    await dns.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);

-            domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 }, function (error) {
-                if (error) return reject(error);
+    await dns.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 });

-                resolve(challenge);
-            });
-        });
-    });
+    return challenge;
 };

 Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challenge) {
@@ -426,13 +440,7 @@ Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challeng

    debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);

-    return new Promise((resolve, reject) => {
-        domains.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-            if (error) return reject(error);
-
-            resolve(null);
-        });
-    });
+    await dns.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
 };

 Acme2.prototype.prepareChallenge = async function (hostname, domain, authorizationUrl, acmeChallengesDir) {
@@ -444,7 +452,7 @@ Acme2.prototype.prepareChallenge = async function (hostname, domain, authorizati
    debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);

    const response = await this.postAsGet(authorizationUrl);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code getting authorization : ${response.status}`);
+    if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code getting authorization : ${response.status}`);

    const authorization = response.body;

@@ -477,7 +485,7 @@ Acme2.prototype.acmeFlow = async function (hostname, domain, paths) {

    const { certFilePath, keyFilePath, csrFilePath, acmeChallengesDir } = paths;

-    await this.registerUser();
+    await this.ensureAccount();
    const { order, orderUrl } = await this.newOrder(hostname);

    for (let i = 0; i < order.authorizations.length; i++) {
@@ -501,47 +509,45 @@ Acme2.prototype.acmeFlow = async function (hostname, domain, paths) {
 };

 Acme2.prototype.loadDirectory = async function () {
-    await promiseRetry({ times: 3, interval: 20000 }, async () => {
+    await promiseRetry({ times: 3, interval: 20000, debug }, async () => {
        const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true);

-        if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code when fetching directory : ${response.status}`);
+        if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching directory : ${response.status}`);

        if (typeof response.body.newNonce !== 'string' ||
            typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`);
+            typeof response.body.newAccount !== 'string') throw new BoxError(BoxError.ACME_ERROR, `Invalid response body : ${response.body}`);

        this.directory = response.body;
    });
 };

-Acme2.prototype.getCertificate = async function (vhost, domain, paths) {
-    assert.strictEqual(typeof vhost, 'string');
+Acme2.prototype.getCertificate = async function (fqdn, domain, paths) {
+    assert.strictEqual(typeof fqdn, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof paths, 'object');

-    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
+    debug(`getCertificate: start acme flow for ${fqdn} from ${this.caDirectory}`);

-    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
-        vhost = domains.makeWildcard(vhost);
-        debug(`getCertificate: will get wildcard cert for ${vhost}`);
+    if (fqdn !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
+        fqdn = dns.makeWildcard(fqdn);
+        debug(`getCertificate: will get wildcard cert for ${fqdn}`);
    }

    await this.loadDirectory();
-    await this.acmeFlow(vhost, domain, paths);
+    await this.acmeFlow(fqdn, domain, paths);
 };

-function getCertificate(vhost, domain, paths, options, callback) {
-    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
+async function getCertificate(fqdn, domain, paths, options) {
+    assert.strictEqual(typeof fqdn, 'string'); // this can also be a wildcard domain (for alias domains)
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof paths, 'object');
    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');

-    let attempt = 1;
-    async.retry({ times: 3, interval: 0 }, function (retryCallback) {
-        debug(`getCertificate: attempt ${attempt++}`);
+    await promiseRetry({ times: 3, interval: 0, debug }, async function () {
+        debug(`getCertificate: for fqdn ${fqdn} and domain ${domain}`);

-        let acme = new Acme2(options || { });
-        acme.getCertificate(vhost, domain, paths).then(callback).catch(retryCallback);
-    }, callback);
+        const acme = new Acme2(options || { });
+        return await acme.getCertificate(fqdn, domain, paths);
+    });
 }
@@ -0,0 +1,81 @@
+'use strict';
+
+exports = module.exports =  {
+    get,
+    set,
+    unset,
+
+    getByAppId,
+    getByName,
+    unsetByAppId,
+    getAppIdByValue,
+};
+
+const assert = require('assert'),
+    database = require('./database.js');
+
+async function set(appId, addonId, env) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert(Array.isArray(env));
+
+    await unset(appId, addonId);
+    if (env.length === 0) return;
+
+    const query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
+    const args = [ ], queryArgs = [ ];
+    for (let i = 0; i < env.length; i++) {
+        args.push(appId, addonId, env[i].name, env[i].value);
+        queryArgs.push('(?, ?, ?, ?)');
+    }
+
+    await database.query(query + queryArgs.join(','), args);
+}
+
+async function unset(appId, addonId) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+
+    await database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
+}
+
+async function unsetByAppId(appId) {
+    assert.strictEqual(typeof appId, 'string');
+
+    await database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ]);
+}
+
+async function get(appId, addonId) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+
+    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
+    return results;
+}
+
+async function getByAppId(appId) {
+    assert.strictEqual(typeof appId, 'string');
+
+    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ]);
+    return results;
+}
+
+async function getAppIdByValue(addonId, namePattern, value) {
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+    assert.strictEqual(typeof value, 'string');
+
+    const results = await database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ]);
+    if (results.length === 0) return null;
+    return results[0].appId;
+}
+
+async function getByName(appId, addonId, namePattern) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+
+    const results = await database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ]);
+    if (results.length === 0) return null;
+    return results[0].value;
+}
@@ -1,597 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get,
-    add,
-    exists,
-    del,
-    update,
-    getAll,
-    getPortBindings,
-    delPortBinding,
-
-    setAddonConfig,
-    getAddonConfig,
-    getAddonConfigByAppId,
-    getAddonConfigByName,
-    unsetAddonConfig,
-    unsetAddonConfigByAppId,
-    getAppIdByAddonConfigValue,
-    getByIpAddress,
-
-    getIcons,
-
-    setHealth,
-    setTask,
-    getAppStoreIds,
-
-    // subdomain table types
-    SUBDOMAIN_TYPE_PRIMARY: 'primary',
-    SUBDOMAIN_TYPE_REDIRECT: 'redirect',
-    SUBDOMAIN_TYPE_ALIAS: 'alias',
-
-    _clear: clear
-};
-
-const assert = require('assert'),
-    async = require('async'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js'),
-    safe = require('safetydance'),
-    util = require('util');
-
-const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
-    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
-    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson',
-    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp',
-    'apps.creationTime', 'apps.updateTime', 'apps.enableMailbox', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
-    'apps.dataDir', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(',');
-
-const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
-
-function postProcess(result) {
-    assert.strictEqual(typeof result, 'object');
-
-    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
-    result.manifest = safe.JSON.parse(result.manifestJson);
-    delete result.manifestJson;
-
-    assert(result.tagsJson === null || typeof result.tagsJson === 'string');
-    result.tags = safe.JSON.parse(result.tagsJson) || [];
-    delete result.tagsJson;
-
-    assert(result.reverseProxyConfigJson === null || typeof result.reverseProxyConfigJson === 'string');
-    result.reverseProxyConfig = safe.JSON.parse(result.reverseProxyConfigJson) || {};
-    delete result.reverseProxyConfigJson;
-
-    assert(result.hostPorts === null || typeof result.hostPorts === 'string');
-    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');
-
-    result.portBindings = { };
-    let hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
-    let environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
-    let portTypes = result.portTypes === null ? [ ] : result.portTypes.split(',');
-
-    delete result.hostPorts;
-    delete result.environmentVariables;
-    delete result.portTypes;
-
-    for (let i = 0; i < environmentVariables.length; i++) {
-        result.portBindings[environmentVariables[i]] = { hostPort: parseInt(hostPorts[i], 10), type: portTypes[i] };
-    }
-
-    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
-    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
-    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
-    delete result.accessRestrictionJson;
-
-    result.sso = !!result.sso;
-    result.enableBackup = !!result.enableBackup;
-    result.enableAutomaticUpdate = !!result.enableAutomaticUpdate;
-    result.enableMailbox = !!result.enableMailbox;
-    result.proxyAuth = !!result.proxyAuth;
-    result.hasIcon = !!result.hasIcon;
-    result.hasAppStoreIcon = !!result.hasAppStoreIcon;
-
-    assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
-    result.debugMode = safe.JSON.parse(result.debugModeJson);
-    delete result.debugModeJson;
-
-    assert(result.servicesConfigJson === null || typeof result.servicesConfigJson === 'string');
-    result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
-    delete result.servicesConfigJson;
-
-    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
-    delete result.subdomains;
-    delete result.domains;
-    delete result.subdomainTypes;
-
-    result.alternateDomains = [];
-    result.aliasDomains = [];
-    for (let i = 0; i < subdomainTypes.length; i++) {
-        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
-            result.location = subdomains[i];
-            result.domain = domains[i];
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
-            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
-            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
-        }
-    }
-
-    let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
-    delete result.envNames;
-    delete result.envValues;
-    result.env = {};
-    for (let i = 0; i < envNames.length; i++) { // NOTE: envNames is [ null ] when env of an app is empty
-        if (envNames[i]) result.env[envNames[i]] = envValues[i];
-    }
-
-    let volumeIds = JSON.parse(result.volumeIds);
-    delete result.volumeIds;
-    let volumeReadOnlys = JSON.parse(result.volumeReadOnlys);
-    delete result.volumeReadOnlys;
-
-    result.mounts = volumeIds[0] === null ? [] : volumeIds.map((v, idx) => { return { volumeId: v, readOnly: !!volumeReadOnlys[idx] }; }); // NOTE: volumeIds is [null] when volumes of an app is empty
-
-    result.error = safe.JSON.parse(result.errorJson);
-    delete result.errorJson;
-
-    result.taskId = result.taskId ? String(result.taskId) : null;
-}
-
-// each query simply join apps table with another table by id. we then join the full result together
-const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
-const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
-const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
-const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
-const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
-    + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
-    + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
-    + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
-    + ` LEFT JOIN (${MOUNTS_QUERY}) AS q4 on q4.id = apps.id`;
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`${APPS_QUERY} WHERE apps.id = ?`, [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
-function getByIpAddress(ip, callback) {
-    assert.strictEqual(typeof ip, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`${APPS_QUERY} WHERE apps.containerIp = ?`, [ ip ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        results.forEach(postProcess);
-
-        callback(null, results);
-    });
-}
-
-function add(id, appStoreId, manifest, location, domain, portBindings, data, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appStoreId, 'string');
-    assert(manifest && typeof manifest === 'object');
-    assert.strictEqual(typeof manifest.version, 'string');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert(data && typeof data === 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    portBindings = portBindings || { };
-
-    var manifestJson = JSON.stringify(manifest);
-
-    const accessRestriction = data.accessRestriction || null;
-    const accessRestrictionJson = JSON.stringify(accessRestriction);
-    const memoryLimit = data.memoryLimit || 0;
-    const cpuShares = data.cpuShares || 512;
-    const installationState = data.installationState;
-    const runState = data.runState;
-    const sso = 'sso' in data ? data.sso : null;
-    const debugModeJson = data.debugMode ? JSON.stringify(data.debugMode) : null;
-    const env = data.env || {};
-    const label = data.label || null;
-    const tagsJson = data.tags ? JSON.stringify(data.tags) : null;
-    const mailboxName = data.mailboxName || null;
-    const mailboxDomain = data.mailboxDomain || null;
-    const reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null;
-    const servicesConfigJson = data.servicesConfig ? JSON.stringify(data.servicesConfig) : null;
-    const enableMailbox = data.enableMailbox || false;
-    const icon = data.icon || null;
-
-    let queries = [];
-
-    queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
-            + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
-            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson, servicesConfigJson, icon, enableMailbox ]
-    });
-
-    queries.push({
-        query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-        args: [ id, domain, location, exports.SUBDOMAIN_TYPE_PRIMARY ]
-    });
-
-    Object.keys(portBindings).forEach(function (env) {
-        queries.push({
-            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, type, appId) VALUES (?, ?, ?, ?)',
-            args: [ env, portBindings[env].hostPort, portBindings[env].type, id ]
-        });
-    });
-
-    Object.keys(env).forEach(function (name) {
-        queries.push({
-            query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
-            args: [ id, name, env[name] ]
-        });
-    });
-
-    if (data.alternateDomains) {
-        data.alternateDomains.forEach(function (d) {
-            queries.push({
-                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]
-            });
-        });
-    }
-
-    if (data.aliasDomains) {
-        data.aliasDomains.forEach(function (d) {
-            queries.push({
-                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
-            });
-        });
-    }
-
-    database.transaction(queries, function (error) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
-        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'no such domain'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function exists(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        return callback(null, result.length !== 0);
-    });
-}
-
-function getPortBindings(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        var portBindings = { };
-        for (let i = 0; i < results.length; i++) {
-            portBindings[results[i].environmentVariable] = { hostPort: results[i].hostPort, type: results[i].type };
-        }
-
-        callback(null, portBindings);
-    });
-}
-
-function getIcons(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT icon, appStoreIcon FROM apps WHERE id = ?', [ id ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, { icon: results[0].icon, appStoreIcon: results[0].appStoreIcon });
-    });
-}
-
-function delPortBinding(hostPort, type, callback) {
-    assert.strictEqual(typeof hostPort, 'number');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM appPortBindings WHERE hostPort=? AND type=?', [ hostPort, type ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var queries = [
-        { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
-        { query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
-    ];
-
-    database.transaction(queries, function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[5].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        database.query.bind(null, 'DELETE FROM subdomains'),
-        database.query.bind(null, 'DELETE FROM appPortBindings'),
-        database.query.bind(null, 'DELETE FROM appAddonConfigs'),
-        database.query.bind(null, 'DELETE FROM appEnvVars'),
-        database.query.bind(null, 'DELETE FROM apps')
-    ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        return callback(null);
-    });
-}
-
-function update(id, app, callback) {
-    // ts is useful as a versioning mechanism (for example, icon changed). update the timestamp explicity in code instead of db.
-    // this way health and healthTime can be updated without changing ts
-    app.ts = new Date();
-    updateWithConstraints(id, app, '', callback);
-}
-
-function updateWithConstraints(id, app, constraints, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof constraints, 'string');
-    assert.strictEqual(typeof callback, 'function');
-    assert(!('portBindings' in app) || typeof app.portBindings === 'object');
-    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
-    assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
-    assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
-    assert(!('tags' in app) || Array.isArray(app.tags));
-    assert(!('env' in app) || typeof app.env === 'object');
-
-    var queries = [ ];
-
-    if ('portBindings' in app) {
-        var portBindings = app.portBindings || { };
-        // replace entries by app id
-        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
-        Object.keys(portBindings).forEach(function (env) {
-            var values = [ portBindings[env].hostPort, portBindings[env].type, env, id ];
-            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, type, environmentVariable, appId) VALUES(?, ?, ?, ?)', args: values });
-        });
-    }
-
-    if ('env' in app) {
-        queries.push({ query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] });
-
-        Object.keys(app.env).forEach(function (name) {
-            queries.push({
-                query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
-                args: [ id, name, app.env[name] ]
-            });
-        });
-    }
-
-    if ('location' in app && 'domain' in app) { // must be updated together as they are unique together
-        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ]}); // all locations of an app must be updated together
-        queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, app.domain, app.location, exports.SUBDOMAIN_TYPE_PRIMARY ]});
-
-        if ('alternateDomains' in app) {
-            app.alternateDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
-            });
-        }
-
-        if ('aliasDomains' in app) {
-            app.aliasDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
-            });
-        }
-    }
-
-    if ('mounts' in app) {
-        queries.push({ query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ]});
-        app.mounts.forEach(function (m) {
-            queries.push({ query: 'INSERT INTO appMounts (appId, volumeId, readOnly) VALUES (?, ?, ?)', args: [ id, m.volumeId, m.readOnly ]});
-        });
-    }
-
-    var fields = [ ], values = [ ];
-    for (let p in app) {
-        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig') {
-            fields.push(`${p}Json = ?`);
-            values.push(JSON.stringify(app[p]));
-        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
-            fields.push(p + ' = ?');
-            values.push(app[p]);
-        }
-    }
-
-    if (values.length !== 0) {
-        values.push(id);
-        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
-    }
-
-    database.transaction(queries, function (error, results) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[results.length - 1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        return callback(null);
-    });
-}
-
-function setHealth(appId, health, healthTime, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof health, 'string');
-    assert(util.types.isDate(healthTime));
-    assert.strictEqual(typeof callback, 'function');
-
-    const values = { health, healthTime };
-
-    updateWithConstraints(appId, values, '', callback);
-}
-
-function setTask(appId, values, options, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof values, 'object');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    values.ts = new Date();
-
-    if (!options.requireNullTaskId) return updateWithConstraints(appId, values, '', callback);
-
-    if (options.requiredState === null) {
-        updateWithConstraints(appId, values, 'AND taskId IS NULL', callback);
-    } else {
-        updateWithConstraints(appId, values, `AND taskId IS NULL AND installationState = "${options.requiredState}"`, callback);
-    }
-}
-
-function getAppStoreIds(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT id, appStoreId FROM apps', function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function setAddonConfig(appId, addonId, env, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert(Array.isArray(env));
-    assert.strictEqual(typeof callback, 'function');
-
-    unsetAddonConfig(appId, addonId, function (error) {
-        if (error) return callback(error);
-
-        if (env.length === 0) return callback(null);
-
-        var query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
-        var args = [ ], queryArgs = [ ];
-        for (var i = 0; i < env.length; i++) {
-            args.push(appId, addonId, env[i].name, env[i].value);
-            queryArgs.push('(?, ?, ?, ?)');
-        }
-
-        database.query(query + queryArgs.join(','), args, function (error) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            return callback(null);
-        });
-    });
-}
-
-function unsetAddonConfig(appId, addonId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function unsetAddonConfigByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function getAddonConfig(appId, addonId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAddonConfigByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAppIdByAddonConfigValue(addonId, namePattern, value, callback) {
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-    assert.strictEqual(typeof value, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null, results[0].appId);
-    });
-}
-
-function getAddonConfigByName(appId, addonId, namePattern, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null, results[0].value);
-    });
-}
@@ -1,10 +1,8 @@
 'use strict';

-const appdb = require('./appdb.js'),
-    apps = require('./apps.js'),
+const apps = require('./apps.js'),
    assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:apphealthmonitor'),
@@ -17,17 +15,15 @@ exports = module.exports = {
    run
 };

-const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
 const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes

 const OOM_EVENT_LIMIT = 60 * 60 * 1000; // will only raise 1 oom event every hour
 let gStartTime = null; // time when apphealthmonitor was started
 let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago

-function setHealth(app, health, callback) {
+async function setHealth(app, health) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof health, 'string');
-    assert.strictEqual(typeof callback, 'function');

    // app starts out with null health
    // if it became healthy, we update immediately. this is required for ui to say "running" etc
@@ -42,79 +38,85 @@ function setHealth(app, health, callback) {
            debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);

            // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, auditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app: app });
        }
    } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
        if (lastHealth === apps.HEALTH_HEALTHY) {
            debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);

            // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, auditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app: app });
        }
    } else {
        debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);
-        return callback(null);
+        return;
    }

-    appdb.setHealth(app.id, health, healthTime, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null); // app uninstalled?
-        if (error) return callback(error);
+    const [error] = await safe(apps.setHealth(app.id, health, healthTime));
+    if (error && error.reason === BoxError.NOT_FOUND) return; // app uninstalled?
+    if (error) throw error;

-        app.health = health;
-        app.healthTime = healthTime;
-
-        callback(null);
-    });
+    app.health = health;
+    app.healthTime = healthTime;
 }


 // callback is called with error for fatal errors and not if health check failed
-function checkAppHealth(app, callback) {
+async function checkAppHealth(app, options) {
    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
+    assert.strictEqual(typeof options, 'object');

-    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) {
-        return callback(null);
-    }
+    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) return;

    const manifest = app.manifest;

-    docker.inspect(app.containerId, function (error, data) {
-        if (error || !data || !data.State) return setHealth(app, apps.HEALTH_ERROR, callback);
-        if (data.State.Running !== true) return setHealth(app, apps.HEALTH_DEAD, callback);
+    let healthCheckUrl, host;
+    if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) {
+        healthCheckUrl = app.upstreamUri;
+        host = new URL(app.upstreamUri).host; // includes port
+    } else {
+        const [error, data] = await safe(docker.inspect(app.containerId));
+        if (error || !data || !data.State) return await setHealth(app, apps.HEALTH_ERROR);
+        if (data.State.Running !== true) return await setHealth(app, apps.HEALTH_DEAD);

        // non-appstore apps may not have healthCheckPath
-        if (!manifest.healthCheckPath) return setHealth(app, apps.HEALTH_HEALTHY, callback);
+        if (!manifest.healthCheckPath) return await setHealth(app, apps.HEALTH_HEALTHY);

-        const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
-        superagent
-            .get(healthCheckUrl)
-            .set('Host', app.fqdn) // required for some apache configs with rewrite rules
-            .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
-            .redirects(0)
-            .timeout(HEALTHCHECK_INTERVAL)
-            .end(function (error, res) {
-                if (error && !error.response) {
-                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
-                } else if (res.statusCode > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
-                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
-                } else {
-                    setHealth(app, apps.HEALTH_HEALTHY, callback);
-                }
-            });
-    });
+        healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
+        host = app.fqdn;
+    }
+
+    const [healthCheckError, response] = await safe(superagent
+        .get(healthCheckUrl)
+        .disableTLSCerts() // for app proxy
+        .set('Host', host) // required for some apache configs with rewrite rules
+        .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
+        .redirects(0)
+        .ok(() => true)
+        .timeout(options.timeout * 1000));
+
+    if (healthCheckError) {
+        await apps.appendLogLine(app, `=> Healtheck error: ${healthCheckError}`);
+        await setHealth(app, apps.HEALTH_UNHEALTHY);
+    } else if (response.status > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
+        await apps.appendLogLine(app, `=> Healtheck error got response status ${response.status}`);
+        await setHealth(app, apps.HEALTH_UNHEALTHY);
+    } else {
+        await setHealth(app, apps.HEALTH_HEALTHY);
+    }
 }

-function getContainerInfo(containerId, callback) {
-    docker.inspect(containerId, function (error, result) {
-        if (error) return callback(error);
+async function getContainerInfo(containerId) {
+    const result = await docker.inspect(containerId);

-        const appId = safe.query(result, 'Config.Labels.appId', null);
+    const appId = safe.query(result, 'Config.Labels.appId', null);
+    if (appId) return { app: await apps.get(appId) }; // don't get by container id as this can be an exec container

-        if (!appId) return callback(null, null /* app */, { name: result.Name.slice(1) }); // addon . Name has a '/' in the beginning for some reason
-
-        apps.get(appId, callback); // don't get by container id as this can be an exec container
-    });
+    if (result.Name.startsWith('/redis-')) {
+        return { app: await apps.get(result.Name.slice('/redis-'.length)), addonName: 'redis' };
+    } else {
+        return { addonName: result.Name.slice(1) }; // addon . Name has a '/' in the beginning for some reason
+    }
 }

 /*
@@ -122,81 +124,67 @@ function getContainerInfo(containerId, callback) {
        docker run -ti -m 100M cloudron/base:3.0.0 /bin/bash
        stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
-function processDockerEvents(intervalSecs, callback) {
-    assert.strictEqual(typeof intervalSecs, 'number');
-    assert.strictEqual(typeof callback, 'function');
+async function processDockerEvents(options) {
+    assert.strictEqual(typeof options, 'object');

-    const since = ((new Date().getTime() / 1000) - intervalSecs).toFixed(0);
+    const since = ((new Date().getTime() / 1000) - options.intervalSecs).toFixed(0);
    const until = ((new Date().getTime() / 1000) - 1).toFixed(0);

-    docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
-        if (error) return callback(error);
+    const stream = await docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) });
+    stream.setEncoding('utf8');
+    stream.on('data', async function (data) { // this is actually ldjson, we only process the first line for now
+        const event = safe.JSON.parse(data);
+        if (!event) return;
+        const containerId = String(event.id);

-        stream.setEncoding('utf8');
-        stream.on('data', function (data) {
-            const event = JSON.parse(data);
-            const containerId = String(event.id);
+        const [error, info] = await safe(getContainerInfo(containerId));
+        const program = error ? containerId : (info.addonName || info.app.fqdn);
+        const now = Date.now();
+        const notifyUser = !info?.app?.debugMode && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);

-            getContainerInfo(containerId, function (error, app, addon) {
-                const program = error ? containerId : (app ? app.fqdn : addon.name);
-                const now = Date.now();
-                const notifyUser = !(app && app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
+        debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);

-                debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
+        if (notifyUser) {
+            await eventlog.add(eventlog.ACTION_APP_OOM, AuditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null });

-                // do not send mails for dev apps
-                if (notifyUser) {
-                    // app can be null for addon containers
-                    eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event, containerId, addon: addon || null, app: app || null });
-
-                    gLastOomMailTime = now;
-                }
-            });
-        });
-
-        stream.on('error', function (error) {
-            debug('Error reading docker events', error);
-            callback();
-        });
-
-        stream.on('end', callback);
-
-        // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
-        setTimeout(stream.destroy.bind(stream), 3000); // https://github.com/apocas/dockerode/issues/179
+            gLastOomMailTime = now;
+        }
    });
+
+    stream.on('error', function (error) {
+        debug('Error reading docker events', error);
+    });
+
+    stream.on('end', function () {
+        // debug('Event stream ended');
+    });
+
+    // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
+    setTimeout(stream.destroy.bind(stream), options.timeout); // https://github.com/apocas/dockerode/issues/179
 }

-function processApp(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function processApp(options) {
+    assert.strictEqual(typeof options, 'object');

-    apps.getAll(function (error, allApps) {
-        if (error) return callback(error);
+    const allApps = await apps.list();

-        async.each(allApps, checkAppHealth, function (error) {
-            const alive = allApps
-                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
+    const healthChecks = allApps.map((app) => checkAppHealth(app, options)); // start healthcheck in parallel

-            debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.` + (error ? ` ${error.reason}` : ''));
+    await Promise.allSettled(healthChecks); // wait for all promises to finish

-            callback(null);
-        });
-    });
+    const alive = allApps
+        .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
+
+    debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.`);
 }

-function run(intervalSecs, callback) {
+async function run(intervalSecs) {
    assert.strictEqual(typeof intervalSecs, 'number');
-    assert.strictEqual(typeof callback, 'function');

    if (constants.TEST) return;

    if (!gStartTime) gStartTime = new Date();

-    async.series([
-        processApp, // this is first because docker.getEvents seems to get 'stuck' sometimes
-        processDockerEvents.bind(null, intervalSecs)
-    ], function (error) {
-        if (error) debug(`run: could not check app health. ${error.message}`);
-
-        callback();
-    });
+    await processApp({ timeout: (intervalSecs - 3) * 1000 });
+    await processDockerEvents({ intervalSecs, timeout: 3000 });
 }
@@ -0,0 +1,210 @@
+'use strict';
+
+exports = module.exports = {
+    list,
+    listByUser,
+    add,
+    get,
+    update,
+    remove,
+    getIcon
+};
+
+const assert = require('assert'),
+    apps = require('./apps.js'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    debug = require('debug')('box:applinks'),
+    jsdom = require('jsdom'),
+    safe = require('safetydance'),
+    superagent = require('superagent'),
+    uuid = require('uuid'),
+    validator = require('validator');
+
+const APPLINKS_FIELDS= [ 'id', 'accessRestrictionJson', 'creationTime', 'updateTime', 'ts', 'label', 'tagsJson', 'icon', 'upstreamUri' ].join(',');
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    assert(result.tagsJson === null || typeof result.tagsJson === 'string');
+    result.tags = safe.JSON.parse(result.tagsJson) || [];
+    delete result.tagsJson;
+
+    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
+    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
+    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
+    delete result.accessRestrictionJson;
+
+    result.ts = new Date(result.ts).getTime();
+
+    result.icon = result.icon ? result.icon : null;
+
+}
+
+function validateUpstreamUri(upstreamUri) {
+    assert.strictEqual(typeof upstreamUri, 'string');
+
+    if (!upstreamUri) return new BoxError(BoxError.BAD_FIELD, 'upstreamUri cannot be empty');
+
+    if (!upstreamUri.includes('://')) return new BoxError(BoxError.BAD_FIELD, 'upstreamUri has no schema');
+
+    const uri = safe(() => new URL(upstreamUri));
+    if (!uri) return new BoxError(BoxError.BAD_FIELD, `upstreamUri is invalid: ${safe.error.message}`);
+    if (uri.protocol !== 'http:' && uri.protocol !== 'https:') return new BoxError(BoxError.BAD_FIELD, 'upstreamUri has an unsupported scheme');
+
+    return null;
+}
+
+async function list() {
+    const results = await database.query(`SELECT ${APPLINKS_FIELDS} FROM applinks ORDER BY upstreamUri`);
+
+    results.forEach(postProcess);
+
+    return results;
+}
+
+async function listByUser(user) {
+    assert.strictEqual(typeof user, 'object');
+
+    const result = await list();
+    return result.filter((app) => apps.canAccess(app, user));
+}
+
+async function detectMetaInfo(applink) {
+    assert.strictEqual(typeof applink, 'object');
+
+    const [error, response] = await safe(superagent.get(applink.upstreamUri).set('User-Agent', 'Mozilla'));
+    if (error || !response.text) throw new BoxError(BoxError.BAD_FIELD, 'cannot fetch upstream uri for favicon and label');
+
+    if (applink.favicon && applink.label) return;
+
+    // set redirected URI if any for favicon url
+    const redirectUri = (response.redirects && response.redirects.length) ? response.redirects[0] : null;
+
+    const dom = new jsdom.JSDOM(response.text);
+    if (!applink.icon) {
+        let favicon = '';
+        if (dom.window.document.querySelector('link[rel="apple-touch-icon"]')) favicon = dom.window.document.querySelector('link[rel="apple-touch-icon"]').href ;
+        if (!favicon.endsWith('.png') && dom.window.document.querySelector('meta[name="msapplication-TileImage"]')) favicon = dom.window.document.querySelector('meta[name="msapplication-TileImage"]').content ;
+        if (!favicon.endsWith('.png') && dom.window.document.querySelector('link[rel="shortcut icon"]')) favicon = dom.window.document.querySelector('link[rel="shortcut icon"]').href ;
+        if (!favicon.endsWith('.png') && dom.window.document.querySelector('link[rel="icon"]')) {
+            let iconElements = dom.window.document.querySelectorAll('link[rel="icon"]');
+            if (iconElements.length) {
+                favicon = iconElements[0].href; // choose first one for a start
+
+                // check if we have sizes attributes and then choose the largest one
+                iconElements = Array.from(iconElements).filter(function (e) {
+                    return e.attributes.getNamedItem('sizes') && e.attributes.getNamedItem('sizes').value;
+                }).sort(function (a, b) {
+                    return parseInt(b.attributes.getNamedItem('sizes').value.split('x')[0]) - parseInt(a.attributes.getNamedItem('sizes').value.split('x')[0]);
+                });
+                if (iconElements.length) favicon = iconElements[0].href;
+            }
+        }
+        if (!favicon.endsWith('.png') && dom.window.document.querySelector('meta[itemprop="image"]')) favicon = dom.window.document.querySelector('meta[itemprop="image"]').content;
+
+        if (favicon) {
+            if (favicon.startsWith('/')) favicon = (redirectUri || applink.upstreamUri) + favicon;
+
+            debug(`detectMetaInfo: found icon: ${favicon}`);
+
+            const [error, response] = await safe(superagent.get(favicon));
+            if (error) console.error(`Failed to fetch icon ${favicon}: `, error);
+            else if (response.ok && response.headers['content-type'] === 'image/png') applink.icon = response.body;
+            else console.error(`Failed to fetch icon ${favicon}: statusCode=${response.status}`);
+        } else {
+            console.error(`Unable to find a suitable icon for ${applink.upstreamUri}`);
+        }
+    }
+
+    if (!applink.label) {
+        if (dom.window.document.querySelector('meta[property="og:title"]')) applink.label = dom.window.document.querySelector('meta[property="og:title"]').content;
+        else if (dom.window.document.querySelector('meta[property="og:site_name"]')) applink.label = dom.window.document.querySelector('meta[property="og:site_name"]').content;
+        else if (dom.window.document.title) applink.label = dom.window.document.title;
+    }
+}
+
+async function add(applink) {
+    assert.strictEqual(typeof applink, 'object');
+    assert.strictEqual(typeof applink.upstreamUri, 'string');
+
+    debug(`add: ${applink.upstreamUri}`, applink);
+
+    let error = validateUpstreamUri(applink.upstreamUri);
+    if (error) throw error;
+
+    if (applink.icon) {
+        if (!validator.isBase64(applink.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
+        applink.icon = Buffer.from(applink.icon, 'base64');
+    }
+
+    await detectMetaInfo(applink);
+
+    const data = {
+        id: uuid.v4(),
+        accessRestrictionJson: applink.accessRestriction ? JSON.stringify(applink.accessRestriction) : null,
+        label: applink.label || '',
+        tagsJson: applink.tags ? JSON.stringify(applink.tags) : null,
+        icon: applink.icon || null,
+        upstreamUri: applink.upstreamUri
+    };
+
+    const query = 'INSERT INTO applinks (id, accessRestrictionJson, label, tagsJson, icon, upstreamUri) VALUES (?, ?, ?, ?, ?, ?)';
+    const args = [ data.id, data.accessRestrictionJson, data.label, data.tagsJson, data.icon, data.upstreamUri ];
+
+    [error] = await safe(database.query(query, args));
+    if (error) throw error;
+
+    return data.id;
+}
+
+async function get(applinkId) {
+    assert.strictEqual(typeof applinkId, 'string');
+
+    const result = await database.query(`SELECT ${APPLINKS_FIELDS} FROM applinks WHERE id = ?`, [ applinkId ]);
+    if (result.length === 0) return null;
+
+    postProcess(result[0]);
+
+    return result[0];
+}
+
+async function update(applinkId, applink) {
+    assert.strictEqual(typeof applinkId, 'string');
+    assert.strictEqual(typeof applink, 'object');
+    assert.strictEqual(typeof applink.upstreamUri, 'string');
+
+    debug(`update: ${applink.upstreamUri}`, applink);
+
+    let error = validateUpstreamUri(applink.upstreamUri);
+    if (error) throw error;
+
+    if (applink.icon) {
+        if (!validator.isBase64(applink.icon)) throw new BoxError(BoxError.BAD_FIELD, 'icon is not base64');
+        applink.icon = Buffer.from(applink.icon, 'base64');
+    }
+
+    await detectMetaInfo(applink);
+
+    const query = 'UPDATE applinks SET label=?, icon=?, upstreamUri=?, tagsJson=?, accessRestrictionJson=? WHERE id = ?';
+    const args = [ applink.label, applink.icon || null, applink.upstreamUri, applink.tags ? JSON.stringify(applink.tags) : null, applink.accessRestriction ? JSON.stringify(applink.accessRestriction) : null, applinkId ];
+
+    const result = await database.query(query, args);
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
+}
+
+async function remove(applinkId) {
+    assert.strictEqual(typeof applinkId, 'string');
+
+    const result = await database.query('DELETE FROM applinks WHERE id = ?', [ applinkId ]);
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
+}
+
+async function getIcon(applinkId) {
+    assert.strictEqual(typeof applinkId, 'string');
+
+    const applink = await get(applinkId);
+    if (!applink) throw new BoxError(BoxError.NOT_FOUND, 'Applink not found');
+
+    return applink.icon;
+}
@@ -0,0 +1,88 @@
+'use strict';
+
+exports = module.exports = {
+    get,
+    add,
+    list,
+    del,
+
+    removePrivateFields
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    crypto = require('crypto'),
+    database = require('./database.js'),
+    hat = require('./hat.js'),
+    safe = require('safetydance'),
+    uuid = require('uuid'),
+    _ = require('underscore');
+
+const APP_PASSWORD_FIELDS = [ 'id', 'name', 'userId', 'identifier', 'hashedPassword', 'creationTime' ].join(',');
+
+function validateAppPasswordName(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 char');
+    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'name too long');
+
+    return null;
+}
+
+function removePrivateFields(appPassword) {
+    return _.pick(appPassword, 'id', 'name', 'userId', 'identifier', 'creationTime');
+}
+
+async function get(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE id = ?', [ id ]);
+    if (result.length === 0) return null;
+    return result[0];
+}
+
+async function add(userId, identifier, name) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof name, 'string');
+
+    let error = validateAppPasswordName(name);
+    if (error) throw error;
+
+    if (identifier.length < 1) throw new BoxError(BoxError.BAD_FIELD, 'identifier must be atleast 1 char');
+
+    const password = hat(16 * 4);
+    const hashedPassword = crypto.createHash('sha256').update(password).digest('base64');
+
+    const appPassword = {
+        id: 'uid-' + uuid.v4(),
+        name,
+        userId,
+        identifier,
+        password,
+        hashedPassword
+    };
+
+    const query = 'INSERT INTO appPasswords (id, userId, identifier, name, hashedPassword) VALUES (?, ?, ?, ?, ?)';
+    const args = [ appPassword.id, appPassword.userId, appPassword.identifier, appPassword.name, appPassword.hashedPassword ];
+
+    [error] = await safe(database.query(query, args));
+    if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('appPasswords_name_userId_identifier') !== -1) throw new BoxError(BoxError.ALREADY_EXISTS, 'name/app combination already exists');
+    if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.indexOf('userId')) throw new BoxError(BoxError.NOT_FOUND, 'user not found');
+    if (error) throw error;
+
+    return { id: appPassword.id, password: appPassword.password };
+}
+
+async function list(userId) {
+    assert.strictEqual(typeof userId, 'string');
+
+    return await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE userId = ?', [ userId ]);
+}
+
+async function del(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query('DELETE FROM appPasswords WHERE id = ?', [ id ]);
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'password not found');
+}
@@ -13,19 +13,21 @@ exports = module.exports = {
    purchaseApp,
    unpurchaseApp,

-    getUserToken,
+    getWebToken,
    getSubscription,
    isFreePlan,

    getAppUpdate,
    getBoxUpdate,

-    createTicket
+    createTicket,
+
+    // exported for tests
+    _unregister: unregister
 };

-var apps = require('./apps.js'),
+const apps = require('./apps.js'),
    assert = require('assert'),
-    async = require('async'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:appstore'),
@@ -35,6 +37,7 @@ var apps = require('./apps.js'),
    safe = require('safetydance'),
    semver = require('semver'),
    settings = require('./settings.js'),
+    sysinfo = require('./sysinfo.js'),
    superagent = require('superagent'),
    support = require('./support.js'),
    util = require('util');
@@ -50,7 +53,7 @@ let gFeatures = {
    privateDockerRegistry: false,
    branding: false,
    support: false,
-    directoryConfig: false,
+    profileConfig: false,
    mailboxMaxCount: 5,
    emailPremium: false
 };
@@ -74,99 +77,66 @@ function isAppAllowed(appstoreId, listingConfig) {
    return true;
 }

-function getCloudronToken(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    settings.getCloudronToken(function (error, token) {
-        if (error) return callback(error);
-        if (!token) return callback(new BoxError(BoxError.LICENSE_ERROR, 'Missing token'));
-
-        callback(null, token);
-    });
-}
-
-function login(email, password, totpToken, callback) {
+async function login(email, password, totpToken) {
    assert.strictEqual(typeof email, 'string');
    assert.strictEqual(typeof password, 'string');
    assert.strictEqual(typeof totpToken, 'string');
-    assert.strictEqual(typeof callback, 'function');

-    var data = {
-        email: email,
-        password: password,
-        totpToken: totpToken
-    };
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/login`)
+        .send({ email, password, totpToken })
+        .timeout(30 * 1000)
+        .ok(() => true));

-    const url = settings.apiServerOrigin() + '/api/v1/login';
-    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${result.statusCode}`));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. status code: ${response.status}`);
+    if (!response.body.accessToken) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. invalid response: ${response.text}`);

-        callback(null, result.body); // { userId, accessToken }
-    });
+    return response.body; // { userId, accessToken }
 }

-function registerUser(email, password, callback) {
+async function registerUser(email, password) {
    assert.strictEqual(typeof email, 'string');
    assert.strictEqual(typeof password, 'string');
-    assert.strictEqual(typeof callback, 'function');

-    var data = {
-        email: email,
-        password: password,
-    };
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/register_user`)
+        .send({ email, password, utmSource: 'cloudron-dashboard' })
+        .timeout(30 * 1000)
+        .ok(() => true));

-    const url = settings.apiServerOrigin() + '/api/v1/register_user';
-    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${result.statusCode}`));
-
-        callback(null);
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, 'Registration error: account already exists');
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Registration error. invalid response: ${response.status}`);
 }

-function getUserToken(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function getWebToken() {
+    if (settings.isDemo()) throw new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode');

-    if (settings.isDemo()) return callback(new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode'));
+    const token = await settings.getAppstoreWebToken();
+    if (!token) throw new BoxError(BoxError.NOT_FOUND); // user will have to re-login with password somehow

-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
-
-        const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
-
-        superagent.post(url).send({}).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${result.status}`));
-
-            callback(null, result.body.accessToken);
-        });
-    });
+    return token;
 }

-function getSubscription(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function getSubscription() {
+    const token = await settings.getAppstoreApiToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');

-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/subscription`)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));

-        const url = settings.apiServerOrigin() + '/api/v1/subscription';
-        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR));
-            if (result.statusCode === 502) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 502) throw new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`);

-            // update the features cache
-            gFeatures = result.body.features;
-            safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
+    // update the features cache
+    gFeatures = response.body.features;
+    safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');

-            callback(null, result.body);
-        });
-    });
+    return response.body;
 }

 function isFreePlan(subscription) {
@@ -174,225 +144,209 @@ function isFreePlan(subscription) {
 }

 // See app.js install it will create a db record first but remove it again if appstore purchase fails
-function purchaseApp(data, callback) {
+async function purchaseApp(data) {
    assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId, appId }
    assert(data.appstoreId || data.manifestId);
    assert.strictEqual(typeof data.appId, 'string');
-    assert.strictEqual(typeof callback, 'function');

-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getAppstoreApiToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');

-        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps`;
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/cloudronapps`)
+        .send(data)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));

-        superagent.post(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); // appstoreId does not exist
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 402) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            // 200 if already purchased, 201 is newly purchased
-            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', result.status, result.body)));
-
-            callback(null);
-        });
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND); // appstoreId does not exist
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    // 200 if already purchased, 201 is newly purchased
+    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', response.status, response.body));
 }

-function unpurchaseApp(appId, data, callback) {
+async function unpurchaseApp(appId, data) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId }
    assert(data.appstoreId || data.manifestId);
-    assert.strictEqual(typeof callback, 'function');

-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getAppstoreApiToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');

-        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
+    const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;

-        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 404) return callback(null);   // was never purchased
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
+    let [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));

-            superagent.del(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
-                if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 404) return;   // was never purchased
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed to get app. status:${response.status}`);

-                callback(null);
-            });
-        });
-    });
+    [error, response] = await safe(superagent.del(url)
+        .send(data)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
+
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed. status:${response.status}`);
 }

-function getBoxUpdate(options, callback) {
+async function getBoxUpdate(options) {
    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');

-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getAppstoreApiToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');

-        const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
+    const query = {
+        accessToken: token,
+        boxVersion: constants.VERSION,
+        automatic: options.automatic
+    };

-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            automatic: options.automatic
-        };
+    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/boxupdate`)
+        .query(query)
+        .timeout(30 * 1000)
+        .ok(() => true));

-        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 204) return callback(null, null); // no update
-            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 204) return; // no update
+    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));

-            var updateInfo = result.body;
+    const updateInfo = response.body;

-            if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', result.statusCode, result.text)));
-            }
-
-            // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
-            if (!updateInfo.version || typeof updateInfo.version !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', result.statusCode, result.text)));
-
-            callback(null, updateInfo);
-        });
-    });
-}
-
-function getAppUpdate(app, options, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
-
-        const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            appId: app.appStoreId,
-            appVersion: app.manifest.version,
-            automatic: options.automatic
-        };
-
-        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 204) return callback(null); // no update
-            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
-
-            const updateInfo = result.body;
-
-            // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
-            const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
-
-            // do some sanity checks
-            if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
-                debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', result.statusCode, result.text)));
-            }
-
-            updateInfo.unstable = !!updateInfo.unstable;
-
-            // { id, creationDate, manifest, unstable }
-            callback(null, updateInfo);
-        });
-    });
-}
-
-function registerCloudron(data, callback) {
-    assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const url = `${settings.apiServerOrigin()}/api/v1/register_cloudron`;
-
-    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${result.statusCode} ${error.message}`));
-
-        // cloudronId, token, licenseKey
-        if (!result.body.cloudronId) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id'));
-        if (!result.body.cloudronToken) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token'));
-        if (!result.body.licenseKey) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no license'));
-
-        async.series([
-            settings.setCloudronId.bind(null, result.body.cloudronId),
-            settings.setCloudronToken.bind(null, result.body.cloudronToken),
-            settings.setLicenseKey.bind(null, result.body.licenseKey),
-        ], function (error) {
-            if (error) return callback(error);
-
-            debug(`registerCloudron: Cloudron registered with id ${result.body.cloudronId}`);
-
-            callback();
-        });
-    });
-}
-
-function updateCloudron(data, callback) {
-    assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    getCloudronToken(function (error, token) {
-        if (error && error.reason === BoxError.LICENSE_ERROR) return callback(null); // missing token. not registered yet
-        if (error) return callback(error);
-
-        const url = `${settings.apiServerOrigin()}/api/v1/update_cloudron`;
-        const query = {
-            accessToken: token
-        };
-
-        superagent.post(url).query(query).send(data).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
-
-            debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
-
-            callback();
-        });
-    });
-}
-
-function registerWithLoginCredentials(options, callback) {
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    function maybeSignup(done) {
-        if (!options.signup) return done();
-
-        registerUser(options.email, options.password, done);
+    if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
+        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', response.status, response.text));
    }

-    getCloudronToken(function (error, token) {
-        if (token) return callback(new BoxError(BoxError.CONFLICT, 'Cloudron is already registered'));
+    // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
+    if (!updateInfo.version || typeof updateInfo.version !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
+    if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
+    if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', response.status, response.text));
+    if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', response.status, response.text));
+    if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', response.status, response.text));
+    if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', response.status, response.text));

-        maybeSignup(function (error) {
-            if (error) return callback(error);
-
-            login(options.email, options.password, options.totpToken || '', function (error, result) {
-                if (error) return callback(error);
-
-                registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
-            });
-        });
-    });
+    return updateInfo;
 }

-function createTicket(info, auditSource, callback) {
+async function getAppUpdate(app, options) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+
+    const token = await settings.getAppstoreApiToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+
+    const query = {
+        accessToken: token,
+        boxVersion: constants.VERSION,
+        appId: app.appStoreId,
+        appVersion: app.manifest.version,
+        automatic: options.automatic
+    };
+
+    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/appupdate`)
+        .query(query)
+        .timeout(30 * 1000)
+        .ok(() => true));
+
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 204) return; // no update
+    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+
+    const updateInfo = response.body;
+
+    // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
+    const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
+
+    // do some sanity checks
+    if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
+        debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
+        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', response.status, response.text));
+    }
+
+    updateInfo.unstable = !!updateInfo.unstable;
+
+    // { id, creationDate, manifest, unstable }
+    return updateInfo;
+}
+
+async function registerCloudron(data) {
+    assert.strictEqual(typeof data, 'object');
+
+    const { domain, accessToken, version, existingApps } = data;
+
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/register_cloudron`)
+        .send({ domain, accessToken, version, existingApps })
+        .timeout(30 * 1000)
+        .ok(() => true));
+
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${response.statusCode} ${error.message}`);
+
+    // cloudronId, token
+    if (!response.body.cloudronId) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id');
+    if (!response.body.cloudronToken) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token');
+
+    await settings.setCloudronId(response.body.cloudronId);
+    await settings.setAppstoreApiToken(response.body.cloudronToken);
+    await settings.setAppstoreWebToken(accessToken);
+
+    debug(`registerCloudron: Cloudron registered with id ${response.body.cloudronId}`);
+}
+
+async function updateCloudron(data) {
+    assert.strictEqual(typeof data, 'object');
+
+    const { domain } = data;
+
+    const token = await settings.getAppstoreApiToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+
+    const query = {
+        accessToken: token
+    };
+
+    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/update_cloudron`)
+        .query(query)
+        .send({ domain })
+        .timeout(30 * 1000)
+        .ok(() => true));
+
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+
+    debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
+}
+
+async function registerWithLoginCredentials(options) {
+    assert.strictEqual(typeof options, 'object');
+
+    if (options.signup) await registerUser(options.email, options.password);
+
+    const result = await login(options.email, options.password, options.totpToken || '');
+    await registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION });
+
+    for (const app of await apps.list()) {
+        await purchaseApp({ appId: app.id, appstoreId: app.appStoreId, manifestId: app.manifest.id || 'customapp' });
+    }
+}
+
+async function unregister() {
+    await settings.setCloudronId('');
+    await settings.setAppstoreApiToken('');
+    await settings.setAppstoreWebToken('');
+}
+
+async function createTicket(info, auditSource) {
    assert.strictEqual(typeof info, 'object');
    assert.strictEqual(typeof info.email, 'string');
    assert.strictEqual(typeof info.displayName, 'string');
@@ -400,128 +354,96 @@ function createTicket(info, auditSource, callback) {
    assert.strictEqual(typeof info.subject, 'string');
    assert.strictEqual(typeof info.description, 'string');
    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');

-    function collectAppInfoIfNeeded(callback) {
-        if (!info.appId) return callback();
-        apps.get(info.appId, callback);
+    const token = await settings.getAppstoreApiToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+
+    if (info.enableSshSupport) {
+        await safe(support.enableRemoteSupport(true, auditSource));
+        info.ipv4 = await sysinfo.getServerIPv4();
    }

-    function enableSshIfNeeded(callback) {
-        if (!info.enableSshSupport) return callback();
+    info.app = info.appId ? await apps.get(info.appId) : null;
+    info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets

-        support.enableRemoteSupport(true, auditSource, function (error) {
-            // ensure we can at least get the ticket through
-            if (error) debug('Unable to enable SSH support.', error);
+    const request = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true);

-            callback();
-        });
+    // either send as JSON through body or as multipart, depending on attachments
+    if (info.app) {
+        request.field('infoJSON', JSON.stringify(info));
+
+        const logPaths = await apps.getLogPaths(info.app);
+        for (const logPath of logPaths) {
+            const logs = safe.child_process.execSync(`tail --lines=1000 ${logPath}`);
+            if (logs) request.attach(path.basename(logPath), logs, path.basename(logPath));
+        }
+    } else {
+        request.send(info);
    }

-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const [error, response] = await safe(request);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));

-        enableSshIfNeeded(function (error) {
-            if (error) return callback(error);
+    await eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);

-            collectAppInfoIfNeeded(function (error, app) {
-                if (error) return callback(error);
-                if (app) info.app = app;
-
-                info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
-
-                var req = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
-                    .query({ accessToken: token })
-                    .timeout(30 * 1000);
-
-                // either send as JSON through body or as multipart, depending on attachments
-                if (info.app) {
-                    req.field('infoJSON', JSON.stringify(info));
-
-                    apps.getLocalLogfilePaths(info.app).forEach(function (filePath) {
-                        var logs = safe.child_process.execSync(`tail --lines=1000 ${filePath}`);
-                        if (logs) req.attach(path.basename(filePath), logs, path.basename(filePath));
-                    });
-                } else {
-                    req.send(info);
-                }
-
-                req.end(function (error, result) {
-                    if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                    if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                    if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                    if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
-
-                    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
-
-                    callback(null, { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
-                });
-            });
-        });
-    });
+    return { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` };
 }

-function getApps(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function getApps() {
+    const token = await settings.getAppstoreApiToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');

-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const unstable = await settings.getUnstableAppsConfig();

-        settings.getUnstableAppsConfig(function (error, unstable) {
-            if (error) return callback(error);
+    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/apps`)
+        .query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable })
+        .timeout(30 * 1000)
+        .ok(() => true));

-            const url = `${settings.apiServerOrigin()}/api/v1/apps`;
-            superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable }).timeout(30 * 1000).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
-                if (!result.body.apps) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 403 || response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', response.status, response.body));
+    if (!response.body.apps) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));

-                settings.getAppstoreListingConfig(function (error, listingConfig) {
-                    if (error) return callback(error);
-
-                    const filteredApps = result.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
-
-                    callback(null, filteredApps);
-                });
-            });
-        });
-    });
+    const listingConfig = await settings.getAppstoreListingConfig();
+    const filteredApps = response.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
+    return filteredApps;
 }

-function getAppVersion(appId, version, callback) {
+async function getAppVersion(appId, version) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof version, 'string');
-    assert.strictEqual(typeof callback, 'function');

-    settings.getAppstoreListingConfig(function (error, listingConfig) {
-        if (error) return callback(error);
+    const listingConfig = await settings.getAppstoreListingConfig();

-        if (!isAppAllowed(appId, listingConfig)) return callback(new BoxError(BoxError.FEATURE_DISABLED));
+    if (!isAppAllowed(appId, listingConfig)) throw new BoxError(BoxError.FEATURE_DISABLED);

-        getCloudronToken(function (error, token) {
-            if (error) return callback(error);
+    const token = await settings.getAppstoreApiToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');

-            let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
-            if (version !== 'latest') url += `/versions/${version}`;
+    let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
+    if (version !== 'latest') url += `/versions/${version}`;

-            superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
-                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
+    const [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));

-                callback(null, result.body);
-            });
-        });
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 403 || response.statusCode === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', response.status, response.body));
+
+    return response.body;
 }

-function getApp(appId, callback) {
+async function getApp(appId) {
    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');

-    getAppVersion(appId, 'latest', callback);
+    return await getAppVersion(appId, 'latest');
 }
@@ -4,7 +4,7 @@ exports = module.exports = {
    scheduleTask
 };

-let assert = require('assert'),
+const assert = require('assert'),
    BoxError = require('./boxerror.js'),
    debug = require('debug')('box:apptaskmanager'),
    fs = require('fs'),
@@ -13,7 +13,6 @@ let assert = require('assert'),
    path = require('path'),
    paths = require('./paths.js'),
    scheduler = require('./scheduler.js'),
-    services = require('./services.js'),
    tasks = require('./tasks.js');

 let gActiveTasks = { }; // indexed by app id
@@ -21,12 +20,11 @@ let gPendingTasks = [ ];
 let gInitialized = false;

 const TASK_CONCURRENCY = 3;
-const NOOP_CALLBACK = function (error) { if (error) debug(error); };

 function waitText(lockOperation) {
    if (lockOperation === locker.OP_BOX_UPDATE) return 'Waiting for Cloudron to finish updating. See the Settings view';
    if (lockOperation === locker.OP_PLATFORM_START) return 'Waiting for Cloudron to initialize';
-    if (lockOperation === locker.OP_FULL_BACKUP) return 'Wait for Cloudron to finish backup. See the Backups view';
+    if (lockOperation === locker.OP_FULL_BACKUP) return 'Waiting for Cloudron to finish backup. See the Backups view';

    return ''; // cannot happen
 }
@@ -37,31 +35,31 @@ function initializeSync() {
 }

 // callback is called when task is finished
-function scheduleTask(appId, taskId, options, callback) {
+function scheduleTask(appId, taskId, options, onFinished) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof taskId, 'string');
    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
+    assert.strictEqual(typeof onFinished, 'function');

    if (!gInitialized) initializeSync();

    if (appId in gActiveTasks) {
-        return callback(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
+        return onFinished(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
    }

    if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
        debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, options, callback });
+        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' });
+        gPendingTasks.push({ appId, taskId, options, onFinished });
        return;
    }

-    var lockError = locker.recursiveLock(locker.OP_APPTASK);
+    const lockError = locker.recursiveLock(locker.OP_APPTASK);

    if (lockError) {
        debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, options, callback });
+        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) });
+        gPendingTasks.push({ appId, taskId, options, onFinished });
        return;
    }

@@ -74,7 +72,7 @@ function scheduleTask(appId, taskId, options, callback) {
    scheduler.suspendJobs(appId);

    tasks.startTask(taskId, Object.assign(options, { logFile }), function (error, result) {
-        callback(error, result);
+        onFinished(error, result);

        delete gActiveTasks[appId];
        locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
@@ -84,15 +82,10 @@ function scheduleTask(appId, taskId, options, callback) {
 }

 function startNextTask() {
-    if (gPendingTasks.length === 0) {
-        // rebuild sftp when task queue is empty. this minimizes risk of sftp rebuild overlapping with other app tasks
-        services.rebuildService('sftp', error => { if (error) debug('Unable to rebuild sftp:', error); });
-        return;
-    }
+    if (gPendingTasks.length === 0) return;

    assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);

    const t = gPendingTasks.shift();
-    scheduleTask(t.appId, t.taskId, t.options, t.callback);
+    scheduleTask(t.appId, t.taskId, t.options, t.onFinished);
 }
-
@@ -1,15 +1,24 @@
 'use strict';

-exports = module.exports = {
-    CRON:  { userId: null, username: 'cron' },
-    HEALTH_MONITOR: { userId: null, username: 'healthmonitor' },
-    EXTERNAL_LDAP_TASK: { userId: null, username: 'externalldap' },
-    EXTERNAL_LDAP_AUTO_CREATE: { userId: null, username: 'externalldap' },
+class AuditSource {
+    constructor(username, userId, ip) {
+        this.username = username;
+        this.userId = userId || null;
+        this.ip = ip || null;
+    }

-    fromRequest
-};
-
-function fromRequest(req) {
-    const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-    return { ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+    static fromRequest(req) {
+        const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+        return new AuditSource(req.user?.username, req.user?.id, ip);
+    }
 }
+
+// these can be static variables but see https://stackoverflow.com/questions/60046847/eslint-does-not-allow-static-class-properties#comment122122927_60464446
+AuditSource.CRON =  new AuditSource('cron');
+AuditSource.HEALTH_MONITOR = new AuditSource('healthmonitor');
+AuditSource.EXTERNAL_LDAP_TASK = new AuditSource('externalldap');
+AuditSource.EXTERNAL_LDAP_AUTO_CREATE = new AuditSource('externalldap');
+AuditSource.APPTASK = new AuditSource('apptask');
+AuditSource.PLATFORM = new AuditSource('platform');
+
+exports = module.exports = AuditSource;
@@ -19,7 +19,13 @@
      <username>%EMAILADDRESS%</username>
      <addThisServer>true</addThisServer>
    </outgoingServer>
-
+    <incomingServer type="pop3">
+      <hostname><%= mailFqdn %></hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <username>%EMAILADDRESS%</username>
+      <authentication>password-cleartext</authentication>
+    </incomingServer>
    <documentation url="http://cloudron.io/email/#autodiscover">
       <descr lang="en">Cloudron Email</descr>
    </documentation>
@@ -0,0 +1,299 @@
+'use strict';
+
+const BoxError = require('./boxerror.js');
+
+exports = module.exports = {
+    run,
+
+    _applyBackupRetentionPolicy: applyBackupRetentionPolicy
+};
+
+const apps = require('./apps.js'),
+    assert = require('assert'),
+    backupFormat = require('./backupformat.js'),
+    backups = require('./backups.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:backupcleaner'),
+    moment = require('moment'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    settings = require('./settings.js'),
+    storage = require('./storage.js'),
+    _ = require('underscore');
+
+function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
+    assert(Array.isArray(allBackups));
+    assert.strictEqual(typeof policy, 'object');
+    assert(Array.isArray(referencedBackupIds));
+
+    const now = new Date();
+
+    for (const backup of allBackups) {
+        if (backup.state === backups.BACKUP_STATE_ERROR) {
+            backup.discardReason = 'error';
+        } else if (backup.state === backups.BACKUP_STATE_CREATING) {
+            if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
+            else backup.discardReason = 'creating-too-long';
+        } else if (referencedBackupIds.includes(backup.id)) {
+            backup.keepReason = 'reference';
+        } else if ((backup.preserveSecs === -1) || ((now - backup.creationTime) < (backup.preserveSecs * 1000))) {
+            backup.keepReason = 'preserveSecs';
+        } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) {
+            backup.keepReason = 'keepWithinSecs';
+        }
+    }
+
+    const KEEP_FORMATS = {
+        keepDaily: 'Y-M-D',
+        keepWeekly: 'Y-W',
+        keepMonthly: 'Y-M',
+        keepYearly: 'Y'
+    };
+
+    for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) {
+        if (!(format in policy)) continue;
+
+        const n = policy[format]; // we want to keep "n" backups of format
+        if (!n) continue; // disabled rule
+
+        let lastPeriod = null, keptSoFar = 0;
+        for (const backup of allBackups) {
+            if (backup.discardReason) continue; // already discarded for some reason
+            if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason
+            const period = moment(backup.creationTime).format(KEEP_FORMATS[format]);
+            if (period === lastPeriod) continue; // already kept for this period
+
+            lastPeriod = period;
+            backup.keepReason = backup.keepReason ? `${backup.keepReason}+${format}` : format;
+            if (++keptSoFar === n) break;
+        }
+    }
+
+    if (policy.keepLatest) {
+        let latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL);
+        if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest';
+    }
+
+    for (const backup of allBackups) {
+        debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
+    }
+}
+
+async function removeBackup(backupConfig, backup, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof backup, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const backupFilePath = backupFormat.api(backup.format).getBackupFilePath(backupConfig, backup.remotePath);
+
+    let removeError;
+    if (backup.format ==='tgz') {
+        progressCallback({ message: `${backup.remotePath}: Removing ${backupFilePath}`});
+        [removeError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFilePath));
+    } else {
+        progressCallback({ message: `${backup.remotePath}: Removing directory ${backupFilePath}`});
+        [removeError] = await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath, progressCallback));
+    }
+
+    if (removeError) {
+        debug('removeBackup: error removing backup %j : %s', backup, removeError.message);
+        return;
+    }
+
+    // prune empty directory if possible
+    const [pruneError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath)));
+    if (pruneError) debug('removeBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), pruneError.message);
+
+    const [delError] = await safe(backups.del(backup.id));
+    if (delError) debug(`removeBackup: error removing ${backup.id} from database`, delError);
+    else debug(`removeBackup: removed ${backup.remotePath}`);
+}
+
+async function cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert(Array.isArray(referencedBackupIds));
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const removedAppBackupPaths = [];
+
+    const allApps = await apps.list();
+    const allAppIds = allApps.map(a => a.id);
+
+    const appBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000);
+
+    // collate the backups by app id. note that the app could already have been uninstalled
+    let appBackupsById = {};
+    for (const appBackup of appBackups) {
+        if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = [];
+        appBackupsById[appBackup.identifier].push(appBackup);
+    }
+
+    // apply backup policy per app. keep latest backup only for existing apps
+    let appBackupsToRemove = [];
+    for (const appId of Object.keys(appBackupsById)) {
+        applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedBackupIds);
+        appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
+    }
+
+    for (const appBackup of appBackupsToRemove) {
+        await progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
+        removedAppBackupPaths.push(appBackup.remotePath);
+        await removeBackup(backupConfig, appBackup, progressCallback); // never errors
+    }
+
+    debug('cleanupAppBackups: done');
+
+    return removedAppBackupPaths;
+}
+
+async function cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert(Array.isArray(referencedBackupIds));
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const removedMailBackupPaths = [];
+
+    const mailBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_MAIL, 1, 1000);
+
+    applyBackupRetentionPolicy(mailBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), referencedBackupIds);
+
+    for (const mailBackup of mailBackups) {
+        if (mailBackup.keepReason) continue;
+        await progressCallback({ message: `Removing mail backup ${mailBackup.remotePath}`});
+        removedMailBackupPaths.push(mailBackup.remotePath);
+        await removeBackup(backupConfig, mailBackup, progressCallback); // never errors
+    }
+
+    debug('cleanupMailBackups: done');
+
+    return removedMailBackupPaths;
+}
+
+async function cleanupBoxBackups(backupConfig, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    let referencedBackupIds = [], removedBoxBackupPaths = [];
+
+    const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
+
+    applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */);
+
+    for (const boxBackup of boxBackups) {
+        if (boxBackup.keepReason) {
+            referencedBackupIds = referencedBackupIds.concat(boxBackup.dependsOn);
+            continue;
+        }
+
+        await progressCallback({ message: `Removing box backup ${boxBackup.remotePath}`});
+
+        removedBoxBackupPaths.push(boxBackup.remotePath);
+        await removeBackup(backupConfig, boxBackup, progressCallback);
+    }
+
+    debug('cleanupBoxBackups: done');
+
+    return { removedBoxBackupPaths, referencedBackupIds };
+}
+
+// cleans up the database by checking if backup existsing in the remote
+async function cleanupMissingBackups(backupConfig, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const perPage = 1000;
+    const missingBackupPaths = [];
+
+    if (constants.TEST) return missingBackupPaths;
+
+    let page = 1, result = [];
+    do {
+        result = await backups.list(page, perPage);
+
+        for (const backup of result) {
+            let backupFilePath = backupFormat.api(backup.format).getBackupFilePath(backupConfig, backup.remotePath);
+            if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
+
+            const [existsError, exists] = await safe(storage.api(backupConfig.provider).exists(backupConfig, backupFilePath));
+            if (existsError || exists) continue;
+
+            await progressCallback({ message: `Removing missing backup ${backup.remotePath}`});
+
+            const [delError] = await safe(backups.del(backup.id));
+            if (delError) debug(`cleanupMissingBackups: error removing ${backup.id} from database`, delError);
+
+            missingBackupPaths.push(backup.remotePath);
+        }
+
+        ++ page;
+    } while (result.length === perPage);
+
+    debug('cleanupMissingBackups: done');
+
+    return missingBackupPaths;
+}
+
+// removes the snapshots of apps that have been uninstalled
+async function cleanupSnapshots(backupConfig) {
+    assert.strictEqual(typeof backupConfig, 'object');
+
+    const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
+    const info = safe.JSON.parse(contents);
+    if (!info) return;
+
+    delete info.box;
+
+    const progressCallback = (progress) => { debug(`cleanupSnapshots: ${progress.message}`); };
+
+    for (const appId of Object.keys(info)) {
+        const app = await apps.get(appId);
+        if (app) continue; // app is still installed
+
+        if (info[appId].format ==='tgz') {
+            await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`)), { debug });
+        } else {
+            await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`), progressCallback), { debug });
+        }
+
+        safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
+        safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`));
+
+        await safe(backups.setSnapshotInfo(appId, null /* info */), { debug });
+        debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`);
+    }
+
+    debug('cleanupSnapshots: done');
+}
+
+async function run(progressCallback) {
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const backupConfig = await settings.getBackupConfig();
+
+    const status = await storage.api(backupConfig.provider).getBackupProviderStatus(backupConfig);
+    debug(`clean: mount point status is ${JSON.stringify(status)}`);
+    if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
+
+    if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
+        debug('cleanup: keeping all backups');
+        return {};
+    }
+
+    await progressCallback({ percent: 10, message: 'Cleaning box backups' });
+    const { removedBoxBackupPaths, referencedBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback); // references is app or mail backup ids
+
+    await progressCallback({ percent: 20, message: 'Cleaning mail backups' });
+    const removedMailBackupPaths = await cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback);
+
+    await progressCallback({ percent: 40, message: 'Cleaning app backups' });
+    const removedAppBackupPaths = await cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback);
+
+    await progressCallback({ percent: 70, message: 'Checking storage backend and removing stale entries in database' });
+    const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback);
+
+    await progressCallback({ percent: 90, message: 'Cleaning snapshots' });
+    await cleanupSnapshots(backupConfig);
+
+    return { removedBoxBackupPaths, removedMailBackupPaths, removedAppBackupPaths, missingBackupPaths };
+}
@@ -1,176 +0,0 @@
-'use strict';
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js'),
-    safe = require('safetydance');
-
-const BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
-
-exports = module.exports = {
-    add,
-
-    getByTypePaged,
-    getByIdentifierPaged,
-    getByIdentifierAndStatePaged,
-
-    get,
-    del,
-    update,
-    list,
-
-    _clear: clear
-};
-
-function postProcess(result) {
-    assert.strictEqual(typeof result, 'object');
-
-    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
-
-    result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
-    delete result.manifestJson;
-}
-
-function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof state, 'string');
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, state, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function getByTypePaged(type, page, perPage, callback) {
-    assert.strictEqual(typeof type, 'string');
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ type, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function getByIdentifierPaged(identifier, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function list(page, perPage, callback) {
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups ORDER BY creationTime DESC LIMIT ?,?',
-        [ (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
-        [ id ], function (error, result) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-            if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
-
-            postProcess(result[0]);
-
-            callback(null, result[0]);
-        });
-}
-
-function add(id, data, callback) {
-    assert(data && typeof data === 'object');
-    assert.strictEqual(typeof id, 'string');
-    assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number');
-    assert.strictEqual(typeof data.packageVersion, 'string');
-    assert.strictEqual(typeof data.type, 'string');
-    assert.strictEqual(typeof data.identifier, 'string');
-    assert.strictEqual(typeof data.state, 'string');
-    assert(Array.isArray(data.dependsOn));
-    assert.strictEqual(typeof data.manifest, 'object');
-    assert.strictEqual(typeof data.format, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var creationTime = data.creationTime || new Date(); // allow tests to set the time
-    var manifestJson = JSON.stringify(data.manifest);
-
-    database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ],
-        function (error) {
-            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            callback(null);
-        });
-}
-
-function update(id, backup, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof backup, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var fields = [ ], values = [ ];
-    for (var p in backup) {
-        fields.push(p + ' = ?');
-        values.push(backup[p]);
-    }
-    values.push(id);
-
-    database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('TRUNCATE TABLE backups', [], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        callback(null);
-    });
-}
@@ -0,0 +1,12 @@
+'use strict';
+
+exports = module.exports = {
+    api
+};
+
+function api(format) {
+    switch (format) {
+    case 'tgz': return require('./backupformat/tgz.js');
+    case 'rsync': return require('./backupformat/rsync.js');
+    }
+}
@@ -0,0 +1,245 @@
+'use strict';
+
+exports = module.exports = {
+    getBackupFilePath,
+    download,
+    upload,
+
+    _saveFsMetadata: saveFsMetadata,
+    _restoreFsMetadata: restoreFsMetadata
+};
+
+const assert = require('assert'),
+    async = require('async'),
+    BoxError = require('../boxerror.js'),
+    DataLayout = require('../datalayout.js'),
+    debug = require('debug')('box:backupformat/rsync'),
+    fs = require('fs'),
+    hush = require('../hush.js'),
+    once = require('../once.js'),
+    path = require('path'),
+    safe = require('safetydance'),
+    storage = require('../storage.js'),
+    syncer = require('../syncer.js'),
+    util = require('util');
+
+function getBackupFilePath(backupConfig, remotePath) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+
+    const rootPath = storage.api(backupConfig.provider).getBackupRootPath(backupConfig);
+    return path.join(rootPath, remotePath);
+}
+
+function sync(backupConfig, remotePath, dataLayout, progressCallback, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB
+    const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10);
+    const removeDir = util.callbackify(storage.api(backupConfig.provider).removeDir);
+    const remove = util.callbackify(storage.api(backupConfig.provider).remove);
+
+    syncer.sync(dataLayout, function processTask(task, iteratorCallback) {
+        debug('sync: processing task: %j', task);
+        // the empty task.path is special to signify the directory
+        const destPath = task.path && backupConfig.encryptedFilenames ? hush.encryptFilePath(task.path, backupConfig.encryption) : task.path;
+        const backupFilePath = path.join(getBackupFilePath(backupConfig, remotePath), destPath);
+
+        if (task.operation === 'removedir') {
+            debug(`Removing directory ${backupFilePath}`);
+            return removeDir(backupConfig, backupFilePath, progressCallback, iteratorCallback);
+        } else if (task.operation === 'remove') {
+            debug(`Removing ${backupFilePath}`);
+            return remove(backupConfig, backupFilePath, iteratorCallback);
+        }
+
+        let retryCount = 0;
+        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+            retryCallback = once(retryCallback); // protect again upload() erroring much later after read stream error
+
+            ++retryCount;
+            if (task.operation === 'add') {
+                progressCallback({ message: `Adding ${task.path}` + (retryCount > 1 ?  ` (Try ${retryCount})` : '') });
+                debug(`Adding ${task.path} position ${task.position} try ${retryCount}`);
+                const stream = hush.createReadStream(dataLayout.toLocalPath('./' + task.path), backupConfig.encryption);
+                stream.on('error', (error) => retryCallback(error.message.includes('ENOENT') ? null : error)); // ignore error if file disappears
+                stream.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0MBps looks wrong
+                    progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}MBps` }); // 0M@0MBps looks wrong
+                });
+                // only create the destination path when we have confirmation that the source is available. otherwise, we end up with
+                // files owned as 'root' and the cp later will fail
+                stream.on('open', function () {
+                    storage.api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, function (error) {
+                        debug(error ? `Error uploading ${task.path} try ${retryCount}: ${error.message}` : `Uploaded ${task.path}`);
+                        retryCallback(error);
+                    });
+                });
+            }
+        }, iteratorCallback);
+    }, concurrency, function (error) {
+        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+
+        callback();
+    });
+}
+
+// this is not part of 'snapshotting' because we need root access to traverse
+async function saveFsMetadata(dataLayout, metadataFile) {
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof metadataFile, 'string');
+
+    // contains paths prefixed with './'
+    const metadata = {
+        emptyDirs: [],
+        execFiles: [],
+        symlinks: []
+    };
+
+    // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
+    for (let lp of dataLayout.localPaths()) {
+        const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (emptyDirs === null) throw new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`);
+        if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
+
+        const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (execFiles === null) throw new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`);
+        if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
+
+        const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (symlinks === null) throw new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`);
+        if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => {
+            const target = safe.fs.readlinkSync(sl);
+            return { path: dataLayout.toRemotePath(sl), target };
+        }));
+    }
+
+    if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) throw new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`);
+}
+
+async function restoreFsMetadata(dataLayout, metadataFile) {
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof metadataFile, 'string');
+
+    debug(`Recreating empty directories in ${dataLayout.toString()}`);
+
+    const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
+    if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
+    const metadata = safe.JSON.parse(metadataJson);
+    if (metadata === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message);
+
+    for (const emptyDir of metadata.emptyDirs) {
+        const [mkdirError] = await safe(fs.promises.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }));
+        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to create path: ${mkdirError.message}`);
+    }
+
+    for (const execFile of metadata.execFiles) {
+        const [chmodError] = await safe(fs.promises.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8)));
+        if (chmodError) throw new BoxError(BoxError.FS_ERROR, `unable to chmod: ${chmodError.message}`);
+    }
+
+    for (const symlink of (metadata.symlinks || [])) {
+        if (!symlink.target) continue;
+        // the path may not exist if we had a directory full of symlinks
+        const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }));
+        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink (mkdir): ${mkdirError.message}`);
+        const [symlinkError] = await safe(fs.promises.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file'));
+        if (symlinkError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink: ${symlinkError.message}`);
+    }
+}
+
+function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
+
+    function downloadFile(entry, done) {
+        let relativePath = path.relative(backupFilePath, entry.fullPath);
+        if (backupConfig.encryptedFilenames) {
+            const { error, result } = hush.decryptFilePath(relativePath, backupConfig.encryption);
+            if (error) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file'));
+            relativePath = result;
+        }
+        const destFilePath = dataLayout.toLocalPath('./' + relativePath);
+
+        fs.mkdir(path.dirname(destFilePath), { recursive: true }, function (error) {
+            if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));
+
+            async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+                storage.api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) {
+                    if (error) {
+                        progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                        return retryCallback(error);
+                    }
+
+                    let destStream = hush.createWriteStream(destFilePath, backupConfig.encryption);
+
+                    // protect against multiple errors. must destroy the write stream so that a previous retry does not write
+                    let closeAndRetry = once((error) => {
+                        if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
+                        else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` });
+                        sourceStream.destroy();
+                        destStream.destroy();
+                        retryCallback(error);
+                    });
+
+                    destStream.on('progress', function (progress) {
+                        const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                        if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0MBps looks wrong
+                        progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}MBps` });
+                    });
+                    destStream.on('error', closeAndRetry);
+
+                    sourceStream.on('error', closeAndRetry);
+
+                    progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
+
+                    sourceStream.pipe(destStream, { end: true }).on('done', closeAndRetry);
+                });
+            }, done);
+        });
+    }
+
+    storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
+        // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
+        const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
+
+        async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
+    }, callback);
+}
+
+async function download(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
+
+    const backupFilePath = getBackupFilePath(backupConfig, remotePath);
+    const downloadDirAsync = util.promisify(downloadDir);
+
+    await downloadDirAsync(backupConfig, backupFilePath, dataLayout, progressCallback);
+    await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
+}
+
+async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof dataLayout, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const syncAsync = util.promisify(sync);
+
+    await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
+    await syncAsync(backupConfig, remotePath, dataLayout, progressCallback);
+}
@@ -0,0 +1,195 @@
+'use strict';
+
+exports = module.exports = {
+    getBackupFilePath,
+    download,
+    upload
+};
+
+const assert = require('assert'),
+    async = require('async'),
+    BoxError = require('../boxerror.js'),
+    DataLayout = require('../datalayout.js'),
+    debug = require('debug')('box:backupformat/tgz'),
+    { DecryptStream, EncryptStream } = require('../hush.js'),
+    once = require('../once.js'),
+    path = require('path'),
+    progressStream = require('progress-stream'),
+    storage = require('../storage.js'),
+    tar = require('tar-fs'),
+    zlib = require('zlib');
+
+function getBackupFilePath(backupConfig, remotePath) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+
+    const rootPath = storage.api(backupConfig.provider).getBackupRootPath(backupConfig);
+
+    const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz';
+    return path.join(rootPath, remotePath + fileType);
+}
+
+function tarPack(dataLayout, encryption) {
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const pack = tar.pack('/', {
+        dereference: false, // pack the symlink and not what it points to
+        entries: dataLayout.localPaths(),
+        ignoreStatError: (path, err) => {
+            debug(`tarPack: error stat'ing ${path} - ${err.code}`);
+            return err.code === 'ENOENT'; // ignore if file or dir got removed (probably some temporary file)
+        },
+        map: function(header) {
+            header.name = dataLayout.toRemotePath(header.name);
+            // the tar pax format allows us to encode filenames > 100 and size > 8GB (see #640)
+            // https://www.systutorials.com/docs/linux/man/5-star/
+            if (header.size > 8589934590 || header.name > 99) header.pax = { size: header.size };
+            return header;
+        },
+        strict: false // do not error for unknown types (skip fifo, char/block devices)
+    });
+
+    const gzip = zlib.createGzip({});
+    const ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds
+
+    pack.on('error', function (error) {
+        debug('tarPack: tar stream error.', error);
+        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    gzip.on('error', function (error) {
+        debug('tarPack: gzip stream error.', error);
+        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    if (encryption) {
+        const encryptStream = new EncryptStream(encryption);
+        encryptStream.on('error', function (error) {
+            debug('tarPack: encrypt stream error.', error);
+            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+        });
+
+        pack.pipe(gzip).pipe(encryptStream).pipe(ps);
+    } else {
+        pack.pipe(gzip).pipe(ps);
+    }
+
+    return ps;
+}
+
+function tarExtract(inStream, dataLayout, encryption) {
+    assert.strictEqual(typeof inStream, 'object');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof encryption, 'object');
+
+    const gunzip = zlib.createGunzip({});
+    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
+    const extract = tar.extract('/', {
+        map: function (header) {
+            header.name = dataLayout.toLocalPath(header.name);
+            return header;
+        },
+        dmode: 500 // ensure directory is writable
+    });
+
+    const emitError = once((error) => {
+        inStream.destroy();
+        ps.emit('error', error);
+    });
+
+    inStream.on('error', function (error) {
+        debug('tarExtract: input stream error.', error);
+        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    gunzip.on('error', function (error) {
+        debug('tarExtract: gunzip stream error.', error);
+        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    extract.on('error', function (error) {
+        debug('tarExtract: extract stream error.', error);
+        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+    });
+
+    extract.on('finish', function () {
+        debug('tarExtract: done.');
+        // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not extract
+        ps.emit('done');
+    });
+
+    if (encryption) {
+        const decrypt = new DecryptStream(encryption);
+        decrypt.on('error', function (error) {
+            debug('tarExtract: decrypt stream error.', error);
+            emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`));
+        });
+        inStream.pipe(ps).pipe(decrypt).pipe(gunzip).pipe(extract);
+    } else {
+        inStream.pipe(ps).pipe(gunzip).pipe(extract);
+    }
+
+    return ps;
+}
+
+async function download(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
+
+    const backupFilePath = getBackupFilePath(backupConfig, remotePath);
+
+    return new Promise((resolve, reject) => {
+        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+            progressCallback({ message: `Downloading backup ${remotePath}` });
+
+            storage.api(backupConfig.provider).download(backupConfig, backupFilePath, function (error, sourceStream) {
+                if (error) return retryCallback(error);
+
+                const ps = tarExtract(sourceStream, dataLayout, backupConfig.encryption);
+
+                ps.on('progress', function (progress) {
+                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                    if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong
+                    progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` });
+                });
+                ps.on('error', retryCallback);
+                ps.on('done', retryCallback);
+            });
+        }, (error) => {
+            if (error) return reject(error);
+            resolve();
+        });
+    });
+}
+
+async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof dataLayout, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    return new Promise((resolve, reject) => {
+        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+            retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error
+
+            const tarStream = tarPack(dataLayout, backupConfig.encryption);
+
+            tarStream.on('progress', function (progress) {
+                const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
+                if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0MBps looks wrong
+                progressCallback({ message: `Uploading backup ${transferred}M@${speed}MBps` });
+            });
+            tarStream.on('error', retryCallback); // already returns BoxError
+
+            storage.api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, remotePath), tarStream, retryCallback);
+        }, (error) => {
+            if (error) return reject(error);
+            resolve();
+        });
+    });
+}
@@ -0,0 +1,528 @@
+'use strict';
+
+exports = module.exports = {
+    fullBackup,
+
+    restore,
+
+    backupApp,
+    downloadApp,
+
+    backupMail,
+    downloadMail,
+
+    upload,
+};
+
+const apps = require('./apps.js'),
+    assert = require('assert'),
+    backupFormat = require('./backupformat.js'),
+    backups = require('./backups.js'),
+    BoxError = require('./boxerror.js'),
+    constants = require('./constants.js'),
+    DataLayout = require('./datalayout.js'),
+    database = require('./database.js'),
+    debug = require('debug')('box:backuptask'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    services = require('./services.js'),
+    settings = require('./settings.js'),
+    shell = require('./shell.js'),
+    storage = require('./storage.js');
+
+const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');
+
+function canBackupApp(app) {
+    // only backup apps that are installed or specific pending states
+
+    // stopped apps cannot be backed up because addons might be down (redis)
+    if (app.runState === apps.RSTATE_STOPPED) return false;
+
+    // we used to check the health here but that doesn't work for stopped apps. it's better to just fail
+    // and inform the user if the backup fails and the app addons have not been setup yet.
+    return  app.installationState === apps.ISTATE_INSTALLED ||
+            app.installationState === apps.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === apps.ISTATE_PENDING_BACKUP ||  // called from apptask
+            app.installationState === apps.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// binary units (non SI) 1024 based
+function prettyBytes(bytes) {
+    assert.strictEqual(typeof bytes, 'number');
+
+    const i = Math.floor(Math.log(bytes) / Math.log(1024)),
+        sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+
+    return (bytes / Math.pow(1024, i)).toFixed(2) * 1 + '' + sizes[i];
+}
+
+async function checkPreconditions(backupConfig, dataLayout) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    // check mount status before uploading
+    const status = await storage.api(backupConfig.provider).getBackupProviderStatus(backupConfig);
+    debug(`upload: mount point status is ${JSON.stringify(status)}`);
+    if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
+
+    // check availabe size. this requires root for df to work
+    const df = await storage.api(backupConfig.provider).getAvailableSize(backupConfig);
+    let used = 0;
+    for (const localPath of dataLayout.localPaths()) {
+        debug(`checkPreconditions: getting disk usage of ${localPath}`);
+        const result = safe.child_process.execSync(`du -Dsb ${localPath}`, { encoding: 'utf8' });
+        if (!result) throw new BoxError(BoxError.FS_ERROR, `du error: ${safe.error.message}`);
+        used += parseInt(result, 10);
+    }
+
+    debug(`checkPreconditions: ${used} bytes`);
+
+    const needed = 0.6 * used + (1024 * 1024 * 1024); // check if there is atleast 1GB left afterwards. aim for 60% because rsync/tgz won't need full 100%
+    if (df.available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${prettyBytes(needed)} Available: ${prettyBytes(df.available)}`);
+}
+
+// this function is called via backupupload (since it needs root to traverse app's directory)
+async function upload(remotePath, format, dataLayoutString, progressCallback) {
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof format, 'string');
+    assert.strictEqual(typeof dataLayoutString, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`upload: path ${remotePath} format ${format} dataLayout ${dataLayoutString}`);
+
+    const dataLayout = DataLayout.fromString(dataLayoutString);
+    const backupConfig = await settings.getBackupConfig();
+
+    await checkPreconditions(backupConfig, dataLayout);
+
+    await backupFormat.api(format).upload(backupConfig, remotePath, dataLayout, progressCallback);
+}
+
+async function download(backupConfig, remotePath, format, dataLayout, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof format, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`download: Downloading ${remotePath} of format ${format} to ${dataLayout.toString()}`);
+
+    await backupFormat.api(format).download(backupConfig, remotePath, dataLayout, progressCallback);
+}
+
+async function restore(backupConfig, remotePath, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
+    if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
+    const dataLayout = new DataLayout(boxDataDir, []);
+
+    await download(backupConfig, remotePath, backupConfig.format, dataLayout, progressCallback);
+
+    debug('restore: download completed, importing database');
+
+    await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`);
+
+    debug('restore: database imported');
+
+    await settings.initCache();
+}
+
+async function downloadApp(app, restoreConfig, progressCallback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof restoreConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
+    if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
+    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);
+
+    const startTime = new Date();
+    const backupConfig = restoreConfig.backupConfig || await settings.getBackupConfig();
+
+    await download(backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
+    debug('downloadApp: time: %s', (new Date() - startTime)/1000);
+}
+
+async function runBackupUpload(uploadConfig, progressCallback) {
+    assert.strictEqual(typeof uploadConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const { remotePath, backupConfig, dataLayout, progressTag } = uploadConfig;
+    assert.strictEqual(typeof remotePath, 'string');
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressTag, 'string');
+    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
+
+    // https://stackoverflow.com/questions/48387040/node-js-recommended-max-old-space-size
+    const envCopy = Object.assign({}, process.env);
+    if (backupConfig.memoryLimit && backupConfig.memoryLimit >= 2*1024*1024*1024) {
+        const heapSize = Math.min((backupConfig.memoryLimit/1024/1024) - 256, 8192);
+        debug(`runBackupUpload: adjusting heap size to ${heapSize}M`);
+        envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`;
+    }
+
+    let result = ''; // the script communicates error result as a string
+    function onMessage(progress) { // this is { message } or { result }
+        if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
+        debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
+        result = progress.result;
+    }
+
+    const [error] = await safe(shell.promises.sudo(`backup-${remotePath}`, [ BACKUP_UPLOAD_CMD, remotePath, backupConfig.format, dataLayout.toString() ], { env: envCopy, preserveEnv: true, ipc: true, onMessage }));
+    if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
+        throw new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed');
+    } else if (error && error.code === 50) { // exited with error
+        throw new BoxError(BoxError.EXTERNAL_ERROR, result);
+    }
+}
+
+async function snapshotBox(progressCallback) {
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    progressCallback({ message: 'Snapshotting box' });
+
+    const startTime = new Date();
+
+    await database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`);
+    debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`);
+}
+
+async function uploadBoxSnapshot(backupConfig, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    await snapshotBox(progressCallback);
+
+    const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
+    if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
+
+    const uploadConfig = {
+        remotePath: 'snapshot/box',
+        backupConfig,
+        dataLayout: new DataLayout(boxDataDir, []),
+        progressTag: 'box'
+    };
+
+    progressCallback({ message: 'Uploading box snapshot' });
+
+    const startTime = new Date();
+
+    await runBackupUpload(uploadConfig, progressCallback);
+
+    debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);
+
+    await backups.setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format });
+}
+
+async function copy(backupConfig, srcRemotePath, destRemotePath, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof srcRemotePath, 'string');
+    assert.strictEqual(typeof destRemotePath, 'string');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const { provider, format } = backupConfig;
+    const oldFilePath = backupFormat.api(format).getBackupFilePath(backupConfig, srcRemotePath);
+    const newFilePath = backupFormat.api(format).getBackupFilePath(backupConfig, destRemotePath);
+
+    const startTime = new Date();
+    await safe(storage.api(provider).copy(backupConfig, oldFilePath, newFilePath, progressCallback));
+    debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
+}
+
+async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert(Array.isArray(dependsOn));
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const remotePath = `${tag}/box_v${constants.VERSION}`;
+    const format = backupConfig.format;
+
+    debug(`rotateBoxBackup: rotating to id ${remotePath}`);
+
+    const data = {
+        remotePath,
+        encryptionVersion: backupConfig.encryption ? 2 : null,
+        packageVersion: constants.VERSION,
+        type: backups.BACKUP_TYPE_BOX,
+        state: backups.BACKUP_STATE_CREATING,
+        identifier: backups.BACKUP_IDENTIFIER_BOX,
+        dependsOn,
+        manifest: null,
+        format,
+        preserveSecs: options.preserveSecs || 0
+    };
+
+    const id = await backups.add(data);
+    const [error] = await safe(copy(backupConfig, 'snapshot/box', remotePath, progressCallback));
+    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
+    await backups.setState(id, state);
+    if (error) throw error;
+
+    return id;
+}
+
+async function backupBox(dependsOn, tag, options, progressCallback) {
+    assert(Array.isArray(dependsOn));
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const backupConfig = await settings.getBackupConfig();
+
+    await uploadBoxSnapshot(backupConfig, progressCallback);
+    return await rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback);
+}
+
+async function rotateAppBackup(backupConfig, app, tag, options, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const snapshotInfo = backups.getSnapshotInfo(app.id);
+
+    const manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat
+    const remotePath = `${tag}/app_${app.fqdn}_v${manifest.version}`;
+    const format = backupConfig.format;
+
+    debug(`rotateAppBackup: rotating ${app.fqdn} to path ${remotePath}`);
+
+    const data = {
+        remotePath,
+        encryptionVersion: backupConfig.encryption ? 2 : null,
+        packageVersion: manifest.version,
+        type: backups.BACKUP_TYPE_APP,
+        state: backups.BACKUP_STATE_CREATING,
+        identifier: app.id,
+        dependsOn: [],
+        manifest,
+        format,
+        preserveSecs: options.preserveSecs || 0
+    };
+
+    const id = await backups.add(data);
+    const [error] = await safe(copy(backupConfig,  `snapshot/app_${app.id}`, remotePath, progressCallback));
+    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
+    await backups.setState(id, state);
+    if (error) throw error;
+
+    return id;
+}
+
+async function backupApp(app, options, progressCallback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    if (options.snapshotOnly) return await snapshotApp(app, progressCallback);
+
+    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
+
+    debug(`backupApp: backing up ${app.fqdn} with tag ${tag}`);
+
+    return await backupAppWithTag(app, tag, options, progressCallback);
+}
+
+async function snapshotApp(app, progressCallback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const startTime = new Date();
+    progressCallback({ message: `Snapshotting app ${app.fqdn}` });
+
+    await apps.backupConfig(app);
+    await services.backupAddons(app, app.manifest.addons);
+
+    debug(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`);
+}
+
+async function uploadAppSnapshot(backupConfig, app, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    await snapshotApp(app, progressCallback);
+
+    const remotePath = `snapshot/app_${app.id}`;
+    const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
+    if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`);
+
+    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);
+
+    progressCallback({ message: `Uploading app snapshot ${app.fqdn}`});
+
+    const uploadConfig = {
+        remotePath,
+        backupConfig,
+        dataLayout,
+        progressTag: app.fqdn
+    };
+
+    const startTime = new Date();
+
+    await runBackupUpload(uploadConfig, progressCallback);
+
+    debug(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`);
+
+    await backups.setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), manifest: app.manifest, format: backupConfig.format });
+}
+
+async function backupAppWithTag(app, tag, options, progressCallback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    if (!canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup
+        const results = await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1);
+        if (results.length === 0) return null; // no backup to re-use
+
+        return results[0].id;
+    }
+
+    const backupConfig = await settings.getBackupConfig();
+
+    await uploadAppSnapshot(backupConfig, app, progressCallback);
+    return await rotateAppBackup(backupConfig, app, tag, options, progressCallback);
+}
+
+async function uploadMailSnapshot(backupConfig, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
+    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);
+
+    const uploadConfig = {
+        remotePath: 'snapshot/mail',
+        backupConfig,
+        dataLayout: new DataLayout(mailDataDir, []),
+        progressTag: 'mail'
+    };
+
+    progressCallback({ message: 'Uploading mail snapshot' });
+
+    const startTime = new Date();
+
+    await runBackupUpload(uploadConfig, progressCallback);
+
+    debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`);
+
+    await backups.setSnapshotInfo('mail', { timestamp: new Date().toISOString(), format: backupConfig.format });
+}
+
+async function rotateMailBackup(backupConfig, tag, options, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const remotePath = `${tag}/mail_v${constants.VERSION}`;
+    const format = backupConfig.format;
+
+    debug(`rotateMailBackup: rotating to ${remotePath}`);
+
+    const data = {
+        remotePath,
+        encryptionVersion: backupConfig.encryption ? 2 : null,
+        packageVersion: constants.VERSION,
+        type: backups.BACKUP_TYPE_MAIL,
+        state: backups.BACKUP_STATE_CREATING,
+        identifier: backups.BACKUP_IDENTIFIER_MAIL,
+        dependsOn: [],
+        manifest: null,
+        format,
+        preserveSecs: options.preserveSecs || 0
+    };
+
+    const id = await backups.add(data);
+    const [error] = await safe(copy(backupConfig, 'snapshot/mail', remotePath, progressCallback));
+    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
+    await backups.setState(id, state);
+    if (error) throw error;
+
+    return id;
+}
+
+async function backupMailWithTag(tag, options, progressCallback) {
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`backupMailWithTag: backing up mail with tag ${tag}`);
+
+    const backupConfig = await settings.getBackupConfig();
+    await uploadMailSnapshot(backupConfig, progressCallback);
+    return await rotateMailBackup(backupConfig, tag, options, progressCallback);
+}
+
+async function backupMail(options, progressCallback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
+    debug(`backupMail: backing up mail with tag ${tag}`);
+    return await backupMailWithTag(tag, options, progressCallback);
+}
+
+async function downloadMail(restoreConfig, progressCallback) {
+    assert.strictEqual(typeof restoreConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
+    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);
+    const dataLayout = new DataLayout(mailDataDir, []);
+
+    const startTime = new Date();
+
+    await download(restoreConfig.backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
+    debug('downloadMail: time: %s', (new Date() - startTime)/1000);
+}
+
+// this function is called from external process. calling process is expected to have a lock
+async function fullBackup(options, progressCallback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); // unique tag under which all apps/mail/box backs up
+
+    const allApps = await apps.list();
+
+    let percent = 1;
+    let step = 100/(allApps.length+3);
+
+    const appBackupIds = [];
+    for (let i = 0; i < allApps.length; i++) {
+        const app = allApps[i];
+        progressCallback({ percent: percent, message: `Backing up ${app.fqdn} (${i+1}/${allApps.length})` });
+        percent += step;
+
+        if (!app.enableBackup) {
+            debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`);
+            continue; // nothing to backup
+        }
+
+        const startTime = new Date();
+        const appBackupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
+        debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
+        if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up
+    }
+
+    progressCallback({ percent, message: 'Backing up mail' });
+    percent += step;
+    const mailBackupId = await backupMailWithTag(tag, options, (progress) => progressCallback({ percent, message: progress.message }));
+
+    progressCallback({ percent, message: 'Backing up system data' });
+    percent += step;
+
+    const dependsOn = appBackupIds.concat(mailBackupId);
+    const backupId = await backupBox(dependsOn, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
+    return backupId;
+}
@@ -4,16 +4,16 @@

 exports = module.exports = {
    get,
+    getString,
    set,
+    setString,
    del,

-    initSecrets,
-
    ACME_ACCOUNT_KEY: 'acme_account_key',
    ADDON_TURN_SECRET: 'addon_turn_secret',
-    DHPARAMS: 'dhparams',
    SFTP_PUBLIC_KEY: 'sftp_public_key',
    SFTP_PRIVATE_KEY: 'sftp_private_key',
+    PROXY_AUTH_TOKEN_SECRET: 'proxy_auth_token_secret',

    CERT_PREFIX: 'cert',

@@ -21,13 +21,7 @@ exports = module.exports = {
 };

 const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    constants = require('./constants.js'),
-    crypto = require('crypto'),
-    database = require('./database.js'),
-    debug = require('debug')('box:blobs'),
-    paths = require('./paths.js'),
-    safe = require('safetydance');
+    database = require('./database.js');

 const BLOBS_FIELDS = [ 'id', 'value' ].join(',');

@@ -39,6 +33,14 @@ async function get(id) {
    return result[0].value;
 }

+async function getString(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query(`SELECT ${BLOBS_FIELDS} FROM blobs WHERE id = ?`, [ id ]);
+    if (result.length === 0) return null;
+    return result[0].value.toString('utf8');
+}
+
 async function set(id, value) {
    assert.strictEqual(typeof id, 'string');
    assert(value === null || Buffer.isBuffer(value));
@@ -46,6 +48,13 @@ async function set(id, value) {
    await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, value ]);
 }

+async function setString(id, value) {
+    assert.strictEqual(typeof id, 'string');
+    assert(value === null || typeof value === 'string');
+
+    await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, Buffer.from(value) ]);
+}
+
 async function del(id) {
    await database.query('DELETE FROM blobs WHERE id=?', [ id ]);
 }
@@ -53,50 +62,3 @@ async function del(id) {
 async function clear() {
    await database.query('DELETE FROM blobs');
 }
-
-async function initSecrets() {
-    let acmeAccountKey = await get(exports.ACME_ACCOUNT_KEY);
-    if (!acmeAccountKey) {
-        acmeAccountKey = safe.child_process.execSync('openssl genrsa 4096');
-        if (!acmeAccountKey) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate acme account key: ${safe.error.message}`);
-        await set(exports.ACME_ACCOUNT_KEY, acmeAccountKey);
-    }
-
-    let turnSecret = await get(exports.ADDON_TURN_SECRET);
-    if (!turnSecret) {
-        turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter
-        await set(exports.ADDON_TURN_SECRET, Buffer.from(turnSecret));
-    }
-
-    if (!constants.TEST) {
-        let dhparams = await get(exports.DHPARAMS);
-        if (!dhparams) {
-            debug('initSecrets: generating dhparams.pem. this takes forever');
-            dhparams = safe.child_process.execSync('openssl dhparam 2048');
-            if (!dhparams) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
-            if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
-            await set(exports.DHPARAMS, dhparams);
-        } else if (!safe.fs.existsSync(paths.DHPARAMS_FILE)) {
-            if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
-        }
-    }
-
-    let sftpPrivateKey = await get(exports.SFTP_PRIVATE_KEY);
-    let sftpPublicKey = await get(exports.SFTP_PUBLIC_KEY);
-
-    if (!sftpPrivateKey || !sftpPublicKey) {
-        debug('initSecrets: generate sftp keys');
-        if (constants.TEST) {
-            safe.fs.unlinkSync(paths.SFTP_PUBLIC_KEY_FILE);
-            safe.fs.unlinkSync(paths.SFTP_PRIVATE_KEY_FILE);
-        }
-        if (!safe.child_process.execSync(`ssh-keygen -m PEM -t rsa -f "${paths.SFTP_KEYS_DIR}/ssh_host_rsa_key" -q -N ""`)) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate sftp ssh keys: ${safe.error.message}`);
-        sftpPublicKey = safe.fs.readFileSync(paths.SFTP_PUBLIC_KEY_FILE);
-        await set(exports.SFTP_PUBLIC_KEY, sftpPublicKey);
-        sftpPrivateKey = safe.fs.readFileSync(paths.SFTP_PRIVATE_KEY_FILE);
-        await set(exports.SFTP_PRIVATE_KEY, sftpPrivateKey);
-    } else if (!safe.fs.existsSync(paths.SFTP_PUBLIC_KEY_FILE) || !safe.fs.existsSync(paths.SFTP_PRIVATE_KEY_FILE)) {
-        if (!safe.fs.writeFileSync(paths.SFTP_PUBLIC_KEY_FILE, sftpPublicKey)) throw new BoxError(BoxError.FS_ERROR, `Could not save sftp public key: ${safe.error.message}`);
-        if (!safe.fs.writeFileSync(paths.SFTP_PRIVATE_KEY_FILE, sftpPrivateKey)) throw new BoxError(BoxError.FS_ERROR, `Could not save sftp private key: ${safe.error.message}`);
-    }
-}
@@ -33,6 +33,7 @@ function BoxError(reason, errorOrMessage, override) {
 }
 util.inherits(BoxError, Error);
 BoxError.ACCESS_DENIED = 'Access Denied';
+BoxError.ACME_ERROR = 'Acme Error';
 BoxError.ADDONS_ERROR = 'Addons Error';
 BoxError.ALREADY_EXISTS = 'Already Exists';
 BoxError.BAD_FIELD = 'Bad Field';
@@ -51,7 +52,7 @@ BoxError.INACTIVE = 'Inactive'; // service/volume/mount
 BoxError.INTERNAL_ERROR = 'Internal Error';
 BoxError.INVALID_CREDENTIALS = 'Invalid Credentials';
 BoxError.IPTABLES_ERROR = 'IPTables Error';
-BoxError.LICENSE_ERROR = 'License Error';
+BoxError.LICENSE_ERROR = 'License Error'; // billing or subscription expired
 BoxError.LOGROTATE_ERROR = 'Logrotate Error';
 BoxError.MAIL_ERROR = 'Mail Error';
 BoxError.MOUNT_ERROR = 'Mount Error';
@@ -60,6 +61,7 @@ BoxError.NGINX_ERROR = 'Nginx Error';
 BoxError.NOT_FOUND = 'Not found';
 BoxError.NOT_IMPLEMENTED = 'Not implemented';
 BoxError.NOT_SIGNED = 'Not Signed';
+BoxError.NOT_SUPPORTED = 'Not Supported';
 BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.PLAN_LIMIT = 'Plan Limit';
 BoxError.SPAWN_ERROR = 'Spawn Error';
@@ -85,10 +87,12 @@ BoxError.toHttpError = function (error) {
    case BoxError.ALREADY_EXISTS:
    case BoxError.BAD_STATE:
    case BoxError.CONFLICT:
+    case BoxError.NOT_SUPPORTED:
        return new HttpError(409, error);
    case BoxError.INVALID_CREDENTIALS:
        return new HttpError(412, error);
    case BoxError.EXTERNAL_ERROR:
+    case BoxError.ACME_ERROR:
    case BoxError.NETWORK_ERROR:
    case BoxError.FS_ERROR:
    case BoxError.MOUNT_ERROR:
@@ -1,6 +1,6 @@
 'use strict';

-let assert = require('assert'),
+const assert = require('assert'),
    fs = require('fs'),
    path = require('path');

@@ -11,7 +11,7 @@ exports = module.exports = {
 function getChanges(version) {
    assert.strictEqual(typeof version, 'string');

-    let changelog = [ ];
+    const changelog = [];
    const lines = fs.readFileSync(path.join(__dirname, '../CHANGES'), 'utf8').split('\n');

    version = version.replace(/[+-].*/, ''); // strip prerelease
@@ -19,20 +19,23 @@ exports = module.exports = {
    renewCerts,
    syncDnsRecords,

+    updateDiskUsage,
+
    runSystemChecks
 };

 const apps = require('./apps.js'),
    appstore = require('./appstore.js'),
    assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
-    backups = require('./backups.js'),
+    AuditSource = require('./auditsource.js'),
    BoxError = require('./boxerror.js'),
    branding = require('./branding.js'),
    constants = require('./constants.js'),
    cron = require('./cron.js'),
    debug = require('debug')('box:cloudron'),
+    delay = require('./delay.js'),
+    dns = require('./dns.js'),
+    dockerProxy = require('./dockerproxy.js'),
    domains = require('./domains.js'),
    eventlog = require('./eventlog.js'),
    fs = require('fs'),
@@ -54,202 +57,164 @@ const apps = require('./apps.js'),

 const REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');

-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 async function initialize() {
-    runStartupTasks();
+    safe(runStartupTasks(), { debug }); // background

    await notifyUpdate();
 }

-function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        cron.stopJobs,
-        platform.stopAllTasks
-    ], callback);
+async function uninitialize() {
+    await cron.stopJobs();
+    await dockerProxy.stop();
+    await platform.stopAllTasks();
 }

-function onActivated(options, callback) {
+async function onActivated(options) {
    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');

    debug('onActivated: running post activation tasks');

    // Starting the platform after a user is available means:
    // 1. mail bounces can now be sent to the cloudron owner
    // 2. the restore code path can run without sudo (since mail/ is non-root)
-    async.series([
-        platform.start.bind(null, options),
-        cron.startJobs,
-        // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
-        // the UI some time to query the dashboard domain in the restore code path
-        (done) => setTimeout(() => reverseProxy.writeDefaultConfig({ activated :true }, done), 30000)
-    ], callback);
+    await platform.start(options);
+    await cron.startJobs();
+    await dockerProxy.start(); // this relies on the 'cloudron' docker network interface to be available
+
+    // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
+    // the UI some time to query the dashboard domain in the restore code path
+    await delay(30000);
+    await reverseProxy.writeDefaultConfig({ activated :true });
 }

 async function notifyUpdate() {
    const version = safe.fs.readFileSync(paths.VERSION_FILE, 'utf8');
    if (version === constants.VERSION) return;

-    await eventlog.add(eventlog.ACTION_UPDATE_FINISH, auditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
+    if (!version) {
+        await eventlog.add(eventlog.ACTION_INSTALL_FINISH, AuditSource.CRON, { version: constants.VERSION });
+    } else {
+        await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
+        const [error] = await safe(tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }));
+        if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist
+    }

-    return new Promise((resolve, reject) => {
-        tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }, function (error) {
-            if (error && error.reason !== BoxError.NOT_FOUND) return reject(error); // when hotfixing, task may not exist
-
-            safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
-
-            resolve();
-        });
-    });
+    safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
 }

 // each of these tasks can fail. we will add some routes to fix/re-run them
-function runStartupTasks() {
-    const tasks = [
-        // stop all the systemd tasks
-        platform.stopAllTasks,
+async function runStartupTasks() {
+    const tasks = [];

-        // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
-        function (callback) {
-            settings.getBackupConfig(function (error, backupConfig) {
-                if (error) return callback(error);
+    // stop all the systemd tasks
+    tasks.push(platform.stopAllTasks);

-                backups.configureCollectd(backupConfig, callback);
-            });
-        },
+    // always generate webadmin config since we have no versioning mechanism for the ejs
+    tasks.push(async function () {
+        if (!settings.dashboardDomain()) return;

-        // always generate webadmin config since we have no versioning mechanism for the ejs
-        function (callback) {
-            if (!settings.dashboardDomain()) return callback();
-
-            reverseProxy.writeDashboardConfig(settings.dashboardDomain(), callback);
-        },
+        const domainObject = await domains.get(settings.dashboardDomain());
+        await reverseProxy.writeDashboardConfig(domainObject);
+    });

+    tasks.push(async function () {
        // check activation state and start the platform
-        function (callback) {
-            users.isActivated(function (error, activated) {
-                if (error) return callback(error);
+        const activated = await users.isActivated();

-                // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
-                // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
-                // we remove the config as a simple security measure to not expose IP <-> domain
-                if (!activated) {
-                    debug('runStartupTasks: not activated. generating IP based redirection config');
-                    return reverseProxy.writeDefaultConfig({ activated: false }, callback);
-                }
-
-                onActivated({}, callback);
-            });
+        // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
+        // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
+        // we remove the config as a simple security measure to not expose IP <-> domain
+        if (!activated) {
+            debug('runStartupTasks: not activated. generating IP based redirection config');
+            return await reverseProxy.writeDefaultConfig({ activated: false });
        }
-    ];
+
+        await onActivated({});
+    });

    // we used to run tasks in parallel but simultaneous nginx reloads was causing issues
-    async.series(async.reflectAll(tasks), function (error, results) {
-        results.forEach((result, idx) => {
-            if (result.error) debug(`Startup task at index ${idx} failed: ${result.error.message}`);
-        });
-    });
+    for (let i = 0; i < tasks.length; i++) {
+        const [error] = await safe(tasks[i]());
+        if (error) debug(`Startup task at index ${i} failed: ${error.message} ${error.stack}`);
+    }
 }

-function getConfig(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function getConfig() {
    const release = safe.fs.readFileSync('/etc/lsb-release', 'utf-8');
-    if (release === null) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
+    if (release === null) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    const ubuntuVersion = release.match(/DISTRIB_DESCRIPTION="(.*)"/)[1];

-    settings.getAll(function (error, allSettings) {
-        if (error) return callback(error);
+    const allSettings = await settings.list();

-        // be picky about what we send out here since this is sent for 'normal' users as well
-        callback(null, {
-            apiServerOrigin: settings.apiServerOrigin(),
-            webServerOrigin: settings.webServerOrigin(),
-            adminDomain: settings.dashboardDomain(),
-            adminFqdn: settings.dashboardFqdn(),
-            mailFqdn: settings.mailFqdn(),
-            version: constants.VERSION,
-            ubuntuVersion,
-            isDemo: settings.isDemo(),
-            cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
-            footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
-            features: appstore.getFeatures(),
-            profileLocked: allSettings[settings.DIRECTORY_CONFIG_KEY].lockUserProfiles,
-            mandatory2FA: allSettings[settings.DIRECTORY_CONFIG_KEY].mandatory2FA
-        });
-    });
+    // be picky about what we send out here since this is sent for 'normal' users as well
+    return {
+        apiServerOrigin: settings.apiServerOrigin(),
+        webServerOrigin: settings.webServerOrigin(),
+        consoleServerOrigin: settings.consoleServerOrigin(),
+        adminDomain: settings.dashboardDomain(),
+        adminFqdn: settings.dashboardFqdn(),
+        mailFqdn: settings.mailFqdn(),
+        version: constants.VERSION,
+        ubuntuVersion,
+        isDemo: settings.isDemo(),
+        cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
+        footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
+        features: appstore.getFeatures(),
+        profileLocked: allSettings[settings.PROFILE_CONFIG_KEY].lockUserProfiles,
+        mandatory2FA: allSettings[settings.PROFILE_CONFIG_KEY].mandatory2FA
+    };
 }

-function reboot(callback) {
-    notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '', function (error) {
-        if (error) debug('reboot: failed to clear reboot notification.', error);
+async function reboot() {
+    await notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '');

-        shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
-    });
+    const [error] = await safe(shell.promises.sudo('reboot', [ REBOOT_CMD ], {}));
+    if (error) debug('reboot: could not reboot', error);
 }

-function isRebootRequired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function isRebootRequired() {
    // https://serverfault.com/questions/92932/how-does-ubuntu-keep-track-of-the-system-restart-required-flag-in-motd
-    callback(null, fs.existsSync('/var/run/reboot-required'));
+    return fs.existsSync('/var/run/reboot-required');
 }

-// called from cron.js
-function runSystemChecks(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function runSystemChecks() {
    debug('runSystemChecks: checking status');

-    async.parallel([
-        checkMailStatus,
-        checkRebootRequired,
-        checkUbuntuVersion
-    ], callback);
+    const checks = [
+        checkMailStatus(),
+        checkRebootRequired(),
+        checkUbuntuVersion()
+    ];
+
+    await Promise.allSettled(checks);
 }

-function checkMailStatus(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    mail.checkConfiguration(function (error, message) {
-        if (error) return callback(error);
-
-        notifications.alert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly', message, callback);
-    });
+async function checkMailStatus() {
+    const message = await mail.checkConfiguration();
+    await notifications.alert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly', message);
 }

-function checkRebootRequired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    isRebootRequired(function (error, rebootRequired) {
-        if (error) return callback(error);
-
-        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '', callback);
-    });
+async function checkRebootRequired() {
+    const rebootRequired = await isRebootRequired();
+    await notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '');
 }

-function checkUbuntuVersion(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function checkUbuntuVersion() {
    const isXenial = fs.readFileSync('/etc/lsb-release', 'utf-8').includes('16.04');
-    if (!isXenial) return callback();
+    if (!isXenial) return;

-    notifications.alert(notifications.ALERT_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 16.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-18/ to upgrade to Ubuntu 18 at the earliest.', callback);
+    await notifications.alert(notifications.ALERT_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 16.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-18/ to upgrade to Ubuntu 18 at the earliest.');
 }

-function getLogs(unit, options, callback) {
+async function getLogs(unit, options) {
    assert.strictEqual(typeof unit, 'string');
    assert(options && typeof options === 'object');
-    assert.strictEqual(typeof callback, 'function');

    assert.strictEqual(typeof options.lines, 'number');
    assert.strictEqual(typeof options.format, 'string');
    assert.strictEqual(typeof options.follow, 'boolean');

-    var lines = options.lines === -1 ? '+1' : options.lines,
+    const lines = options.lines === -1 ? '+1' : options.lines,
        format = options.format || 'json',
        follow = options.follow;

@@ -261,15 +226,15 @@ function getLogs(unit, options, callback) {
    // need to handle box.log without subdir
    if (unit === 'box') args.push(path.join(paths.LOG_DIR, 'box.log'));
    else if (unit.startsWith('crash-')) args.push(path.join(paths.CRASH_LOG_DIR, unit.slice(6) + '.log'));
-    else return callback(new BoxError(BoxError.BAD_FIELD, 'No such unit', { field: 'unit' }));
+    else throw new BoxError(BoxError.BAD_FIELD, `No such unit '${unit}'`);

-    var cp = spawn('/usr/bin/tail', args);
+    const cp = spawn('/usr/bin/tail', args);

-    var transformStream = split(function mapper(line) {
+    const transformStream = split(function mapper(line) {
        if (format !== 'json') return line + '\n';

-        var data = line.split(' '); // logs are <ISOtimestamp> <msg>
-        var timestamp = (new Date(data[0])).getTime();
+        const data = line.split(' '); // logs are <ISOtimestamp> <msg>
+        let timestamp = (new Date(data[0])).getTime();
        if (isNaN(timestamp)) timestamp = 0;

        return JSON.stringify({
@@ -283,142 +248,107 @@ function getLogs(unit, options, callback) {

    cp.stdout.pipe(transformStream);

-    return callback(null, transformStream);
+    return transformStream;
 }

-function prepareDashboardDomain(domain, auditSource, callback) {
+async function prepareDashboardDomain(domain, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');

    debug(`prepareDashboardDomain: ${domain}`);

-    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+    if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');

-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await domains.get(domain);
+    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');

-        const fqdn = domains.fqdn(constants.DASHBOARD_LOCATION, domainObject);
+    const fqdn = dns.fqdn(constants.DASHBOARD_SUBDOMAIN, domainObject);

-        apps.getAll(function (error, result) {
-            if (error) return callback(error);
+    const result = await apps.list();
+    if (result.some(app => app.fqdn === fqdn)) throw new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app');

-            const conflict = result.filter(app => app.fqdn === fqdn);
-            if (conflict.length) return callback(new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app'));
+    const taskId = await tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_SUBDOMAIN, domain, auditSource ]);

-            tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_LOCATION, domain, auditSource ], function (error, taskId) {
-                if (error) return callback(error);
+    tasks.startTask(taskId, {});

-                tasks.startTask(taskId, {}, NOOP_CALLBACK);
-
-                callback(null, taskId);
-            });
-        });
-    });
+    return taskId;
 }

 // call this only pre activation since it won't start mail server
-function setDashboardDomain(domain, auditSource, callback) {
+async function setDashboardDomain(domain, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');

    debug(`setDashboardDomain: ${domain}`);

-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await domains.get(domain);
+    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');

-        reverseProxy.writeDashboardConfig(domain, function (error) {
-            if (error) return callback(error);
+    await reverseProxy.writeDashboardConfig(domainObject);
+    const fqdn = dns.fqdn(constants.DASHBOARD_SUBDOMAIN, domainObject);

-            const fqdn = domains.fqdn(constants.DASHBOARD_LOCATION, domainObject);
+    await settings.setDashboardLocation(domain, fqdn);

-            settings.setDashboardLocation(domain, fqdn, function (error) {
-                if (error) return callback(error);
+    await safe(appstore.updateCloudron({ domain }), { debug });

-                appstore.updateCloudron({ domain }, NOOP_CALLBACK);
-
-                eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain, fqdn });
-
-                callback(null);
-            });
-        });
-    });
+    await eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain, fqdn });
 }

 // call this only post activation because it will restart mail server
-function updateDashboardDomain(domain, auditSource, callback) {
+async function updateDashboardDomain(domain, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');

    debug(`updateDashboardDomain: ${domain}`);

-    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+    if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');

-    setDashboardDomain(domain, auditSource, function (error) {
-        if (error) return callback(error);
+    await setDashboardDomain(domain, auditSource);

-        services.rebuildService('turn', NOOP_CALLBACK); // to update the realm variable
-
-        callback(null);
-    });
+    safe(services.rebuildService('turn', auditSource), { debug }); // to update the realm variable
 }

-function renewCerts(options, auditSource, callback) {
+async function renewCerts(options, auditSource) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');

-    tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ], function (error, taskId) {
-        if (error) return callback(error);
-
-        tasks.startTask(taskId, {}, NOOP_CALLBACK);
-
-        callback(null, taskId);
-    });
+    const taskId = await tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ]);
+    tasks.startTask(taskId, {});
+    return taskId;
 }

-function setupDnsAndCert(subdomain, domain, auditSource, progressCallback, callback) {
+async function setupDnsAndCert(subdomain, domain, auditSource, progressCallback) {
    assert.strictEqual(typeof subdomain, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');

-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await domains.get(domain);
+    const dashboardFqdn = dns.fqdn(subdomain, domainObject);

-        const dashboardFqdn = domains.fqdn(subdomain, domainObject);
+    const ipv4 = await sysinfo.getServerIPv4();
+    const ipv6 = await sysinfo.getServerIPv6();

-        sysinfo.getServerIp(function (error, ip) {
-            if (error) return callback(error);
-
-            async.series([
-                (done) => { progressCallback({ message: `Updating DNS of ${dashboardFqdn}` }); done(); },
-                domains.upsertDnsRecords.bind(null, subdomain, domain, 'A', [ ip ]),
-                (done) => { progressCallback({ message: `Waiting for DNS of ${dashboardFqdn}` }); done(); },
-                domains.waitForDnsRecord.bind(null, subdomain, domain, 'A', ip, { interval: 30000, times: 50000 }),
-                (done) => { progressCallback({ message: `Getting certificate of ${dashboardFqdn}` }); done(); },
-                reverseProxy.ensureCertificate.bind(null, domains.fqdn(subdomain, domainObject), domain, auditSource)
-            ], function (error) {
-                if (error) return callback(error);
-
-                callback(null);
-            });
-        });
-    });
+    progressCallback({ percent: 20, message: `Updating DNS of ${dashboardFqdn}` });
+    await dns.upsertDnsRecords(subdomain, domain, 'A', [ ipv4 ]);
+    if (ipv6) await dns.upsertDnsRecords(subdomain, domain, 'AAAA', [ ipv6 ]);
+    progressCallback({ percent: 40, message: `Waiting for DNS of ${dashboardFqdn}` });
+    await dns.waitForDnsRecord(subdomain, domain, 'A', ipv4, { interval: 30000, times: 50000 });
+    if (ipv6) await dns.waitForDnsRecord(subdomain, domain, 'AAAA', ipv6, { interval: 30000, times: 50000 });
+    progressCallback({ percent: 60, message: `Getting certificate of ${dashboardFqdn}` });
+    await reverseProxy.ensureCertificate(dns.fqdn(subdomain, domainObject), domain, auditSource);
 }

-function syncDnsRecords(options, callback) {
+async function syncDnsRecords(options) {
    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');

-    tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ], function (error, taskId) {
-        if (error) return callback(error);
-
-        tasks.startTask(taskId, {}, NOOP_CALLBACK);
-
-        callback(null, taskId);
-    });
+    const taskId = await tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ]);
+    tasks.startTask(taskId, {});
+    return taskId;
+}
+
+async function updateDiskUsage() {
+    const taskId = await tasks.add(tasks.TASK_UPDATE_DISK_USAGE, []);
+    tasks.startTask(taskId, {});
+    return taskId;
 }
@@ -1,55 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    addProfile,
-    removeProfile
-};
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    debug = require('debug')('collectd'),
-    fs = require('fs'),
-    path = require('path'),
-    paths = require('./paths.js'),
-    safe = require('safetydance'),
-    shell = require('./shell.js');
-
-const CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh');
-
-function addProfile(name, profile, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof profile, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    const configFilePath = path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`);
-
-    // skip restarting collectd if the profile already exists with the same contents
-    const currentProfile = safe.fs.readFileSync(configFilePath, 'utf8') || '';
-    if (currentProfile === profile) return callback(null);
-
-    fs.writeFile(configFilePath, profile, function (error) {
-        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
-
-        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
-
-            callback(null);
-        });
-    });
-
-}
-
-function removeProfile(name, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`), function (error) {
-        if (error && error.code !== 'ENOENT') debug('Error removing collectd profile', error);
-
-        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
-
-            callback(null);
-        });
-    });
-}
@@ -1,42 +0,0 @@
-LoadPlugin "table"
-<Plugin table>
-  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.stat">
-    Instance "<%= appId %>-memory"
-    Separator " \\n"
-    <Result>
-      Type gauge
-      InstancesFrom 0
-      ValuesFrom 1
-    </Result>
-  </Table>
-
-  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.max_usage_in_bytes">
-    Instance "<%= appId %>-memory"
-    Separator "\\n"
-    <Result>
-      Type gauge
-      InstancePrefix "max_usage_in_bytes"
-      ValuesFrom 0
-    </Result>
-  </Table>
-
-  <Table "/sys/fs/cgroup/cpuacct/docker/<%= containerId %>/cpuacct.stat">
-    Instance "<%= appId %>-cpu"
-    Separator " \\n"
-    <Result>
-      Type gauge
-      InstancesFrom 0
-      ValuesFrom 1
-    </Result>
-  </Table>
-</Plugin>
-
-<Plugin python>
-  <Module du>
-    <Path>
-        Instance "<%= appId %>"
-        Dir "<%= appDataDir %>"
-    </Path>
-  </Module>
-</Plugin>
-
@@ -1,14 +1,14 @@
 'use strict';

-let fs = require('fs'),
+const fs = require('fs'),
    path = require('path');

 const CLOUDRON = process.env.BOX_ENV === 'cloudron',
    TEST = process.env.BOX_ENV === 'test';

 exports = module.exports = {
-    SMTP_LOCATION: 'smtp',
-    IMAP_LOCATION: 'imap',
+    SMTP_SUBDOMAIN: 'smtp',
+    IMAP_SUBDOMAIN: 'imap',

    // These are combined into one array because users and groups become mailboxes
    RESERVED_NAMES: [
@@ -22,13 +22,14 @@ exports = module.exports = {
        'admins', 'users'         // ldap code uses 'users' pseudo group
    ],

-    DASHBOARD_LOCATION: 'my',
+    DASHBOARD_SUBDOMAIN: 'my',

    PORT: CLOUDRON ? 3000 : 5454,
    INTERNAL_SMTP_PORT: 2525, // this value comes from the mail container
    AUTHWALL_PORT: 3001,
    LDAP_PORT: 3002,
    DOCKER_PROXY_PORT: 3003,
+    USER_DIRECTORY_LDAPS_PORT: 3004, // user directory LDAP with TLS rerouting in iptables, public port is 636

    NGINX_DEFAULT_CONFIG_FILE_NAME: 'default.conf',

@@ -46,18 +47,33 @@ exports = module.exports = {
        'io.github.sickchill.cloudronapp',
        'to.couchpota.cloudronapp'
    ],
+    DEMO_APP_LIMIT: 20,
+
+    PROXY_APP_APPSTORE_ID: 'io.cloudron.builtin.appproxy',

    AUTOUPDATE_PATTERN_NEVER: 'never',

+    // the db field is a blob so we make this explicit
+    AVATAR_NONE: Buffer.from('', 'utf8'),
+    AVATAR_GRAVATAR: Buffer.from('gravatar', 'utf8'),
+    AVATAR_CUSTOM: Buffer.from('custom', 'utf8'),   // this is not used here just for reference. The field will contain a byte buffer instead of the type string
+
    SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8), // also used in dashboard client.js

    CLOUDRON: CLOUDRON,
    TEST: TEST,

+    PORT25_CHECK_SERVER: 'port25check.cloudron.io',
+
+    FORUM_URL: 'https://forum.cloudron.io',
+
+    SUPPORT_USERNAME: 'cloudron-support',
    SUPPORT_EMAIL: 'support@cloudron.io',

+    USER_DIRECTORY_LDAP_DN: 'cn=admin,ou=system,dc=cloudron',
+
    FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',

-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '6.0.1-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '7.2.0-test'
 };

@@ -5,7 +5,8 @@ exports = module.exports = {
 };

 const assert = require('assert'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
+    child_process = require('child_process'),
    eventlog = require('./eventlog.js'),
    safe = require('safetydance'),
    path = require('path'),
@@ -17,43 +18,36 @@ const COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
 const CRASH_LOG_TIMESTAMP_OFFSET = 1000 * 60 * 60; // 60 min
 const CRASH_LOG_TIMESTAMP_FILE = '/tmp/crashlog.timestamp';

-function collectLogs(unitName, callback) {
+async function collectLogs(unitName) {
    assert.strictEqual(typeof unitName, 'string');
-    assert.strictEqual(typeof callback, 'function');

-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
-    if (!logs) return callback(safe.error);
-
-    callback(null, logs);
+    const logs = child_process.execSync(`sudo ${COLLECT_LOGS_CMD} ${unitName}`, { encoding: 'utf8' });
+    return logs;
 }

-function sendFailureLogs(unitName, callback) {
+async function sendFailureLogs(unitName) {
    assert.strictEqual(typeof unitName, 'string');
-    assert.strictEqual(typeof callback, 'function');

    // check if we already sent a mail in the last CRASH_LOG_TIME_OFFSET window
    const timestamp = safe.fs.readFileSync(CRASH_LOG_TIMESTAMP_FILE, 'utf8');
    if (timestamp && (parseInt(timestamp) + CRASH_LOG_TIMESTAMP_OFFSET) > Date.now()) {
        console.log('Crash log already sent within window');
-        return callback();
+        return;
    }

-    collectLogs(unitName, async function (error, logs) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            logs = util.format('Failed to collect logs.', error);
-        }
+    let [error, logs] = await safe(collectLogs(unitName));
+    if (error) {
+        console.error('Failed to collect logs.', error);
+        logs = util.format('Failed to collect logs.', error);
+    }

-        const crashId = `${new Date().toISOString()}`;
-        console.log(`Creating crash log for ${unitName} with id ${crashId}`);
+    const crashId = `${new Date().toISOString()}`;
+    console.log(`Creating crash log for ${unitName} with id ${crashId}`);

-        if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, `${crashId}.log`), logs)) console.log(`Failed to stash logs to ${crashId}.log:`, safe.error);
+    if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, `${crashId}.log`), logs)) console.log(`Failed to stash logs to ${crashId}.log:`, safe.error);

-        [error] = await safe(eventlog.add(eventlog.ACTION_PROCESS_CRASH, auditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }));
-        if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);
+    [error] = await safe(eventlog.add(eventlog.ACTION_PROCESS_CRASH, AuditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }));
+    if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);

-        safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
-
-        callback();
-    });
+    safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
 }
@@ -19,8 +19,7 @@ exports = module.exports = {
 const appHealthMonitor = require('./apphealthmonitor.js'),
    apps = require('./apps.js'),
    assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
    backups = require('./backups.js'),
    cloudron = require('./cloudron.js'),
    constants = require('./constants.js'),
@@ -29,6 +28,8 @@ const appHealthMonitor = require('./apphealthmonitor.js'),
    dyndns = require('./dyndns.js'),
    eventlog = require('./eventlog.js'),
    janitor = require('./janitor.js'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
    scheduler = require('./scheduler.js'),
    settings = require('./settings.js'),
    system = require('./system.js'),
@@ -52,8 +53,6 @@ const gJobs = {
    appHealthMonitor: null
 };

-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 // cron format
 // Seconds: 0-59
 // Minutes: 0-59
@@ -62,87 +61,106 @@ const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 // Months: 0-11
 // Day of Week: 0-6

-function startJobs(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function getCronSeed() {
+    let hour = null;
+    let minute = null;

-    debug('startJobs: starting cron jobs');
+    const seedData = safe.fs.readFileSync(paths.CRON_SEED_FILE, 'utf8') || '';
+    const parts = seedData.split(':');
+    if (parts.length === 2) {
+        hour = parseInt(parts[0]) || null;
+        minute = parseInt(parts[1]) || null;
+    }
+
+    if ((hour == null || hour < 0 || hour > 23) || (minute == null || minute < 0 || minute > 60)) {
+        hour = Math.floor(24 * Math.random());
+        minute = Math.floor(60 * Math.random());
+
+        debug(`getCronSeed: writing new cron seed file with ${hour}:${minute} to ${paths.CRON_SEED_FILE}`);
+
+        safe.fs.writeFileSync(paths.CRON_SEED_FILE, `${hour}:${minute}`);
+    }
+
+    return { hour, minute };
+}
+
+async function startJobs() {
+    const { hour, minute } = getCronSeed();
+
+    debug(`startJobs: starting cron jobs with hour ${hour} and minute ${minute}`);

-    const randomTick = Math.floor(60*Math.random());
    gJobs.systemChecks = new CronJob({
-        cronTime: '00 30 2 * * *', // once a day. if you change this interval, change the notification messages with correct duration
-        onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
+        cronTime: `00 ${minute} 2 * * *`, // once a day. if you change this interval, change the notification messages with correct duration
+        onTick: async () => await safe(cloudron.runSystemChecks(), { debug }),
        start: true
    });

    gJobs.diskSpaceChecker = new CronJob({
        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
+        onTick: async () => await safe(system.checkDiskSpace(), { debug }),
        start: true
    });

    // this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
    gJobs.updateCheckerJob = new CronJob({
-        cronTime: `${randomTick} ${randomTick} 1,5,9,13,17,21,23 * * *`,
-        onTick: () => updateChecker.checkForUpdates({ automatic: true }, NOOP_CALLBACK),
+        cronTime: `00 ${minute} 1,5,9,13,17,21,23 * * *`,
+        onTick: async () => await safe(updateChecker.checkForUpdates({ automatic: true }), { debug }),
        start: true
    });

    gJobs.cleanupTokens = new CronJob({
        cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: janitor.cleanupTokens,
+        onTick: async () => await safe(janitor.cleanupTokens(), { debug }),
        start: true
    });

    gJobs.cleanupBackups = new CronJob({
        cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN,
-        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        onTick: async () => await safe(backups.startCleanupTask(AuditSource.CRON), { debug }),
        start: true
    });

    gJobs.cleanupEventlog = new CronJob({
        cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: eventlog.cleanup.bind(null, new Date(Date.now() - 60 * 60 * 24 * 10 * 1000)), // 10 days ago
+        onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 60 * 60 * 24 * 60 * 1000) }), { debug }), // 60 days ago
        start: true
    });

    gJobs.dockerVolumeCleaner = new CronJob({
        cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: janitor.cleanupDockerVolumes,
+        onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug }),
        start: true
    });

    gJobs.schedulerSync = new CronJob({
        cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
-        onTick: scheduler.sync,
+        onTick: async () => await safe(scheduler.sync(), { debug }),
        start: true
    });

+    // randomized per Cloudron based on hourlySeed
    gJobs.certificateRenew = new CronJob({
-        cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
+        cronTime: `00 10 ${hour} * * *`,
+        onTick: async () => await safe(cloudron.renewCerts({}, AuditSource.CRON), { debug }),
        start: true
    });

    gJobs.appHealthMonitor = new CronJob({
        cronTime: '*/10 * * * * *', // every 10 seconds
-        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
+        onTick: async () => await safe(appHealthMonitor.run(10), { debug }), // 10 is the max run time
        start: true
    });

-    settings.getAll(function (error, allSettings) {
-        if (error) return callback(error);
+    const allSettings = await settings.list();

-        const tz = allSettings[settings.TIME_ZONE_KEY];
-        backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
-        autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz);
-        dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
-
-        callback();
-    });
+    const tz = allSettings[settings.TIME_ZONE_KEY];
+    backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
+    autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz);
+    dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
 }

 // eslint-disable-next-line no-unused-vars
-function handleSettingsChanged(key, value) {
+async function handleSettingsChanged(key, value) {
    assert.strictEqual(typeof key, 'string');
    // value is a variant

@@ -152,10 +170,8 @@ function handleSettingsChanged(key, value) {
    case settings.AUTOUPDATE_PATTERN_KEY:
    case settings.DYNAMIC_DNS_KEY:
        debug('handleSettingsChanged: recreating all jobs');
-        async.series([
-            stopJobs,
-            startJobs
-        ], NOOP_CALLBACK);
+        await stopJobs();
+        await startJobs();
        break;
    default:
        break;
@@ -172,7 +188,7 @@ function backupConfigChanged(value, tz) {

    gJobs.backup = new CronJob({
        cronTime: value.schedulePattern,
-        onTick: backups.startBackupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        onTick: async () => await safe(backups.startBackupTask(AuditSource.CRON), { debug }),
        start: true,
        timeZone: tz
    });
@@ -190,19 +206,21 @@ function autoupdatePatternChanged(pattern, tz) {

    gJobs.autoUpdater = new CronJob({
        cronTime: pattern,
-        onTick: function() {
+        onTick: async function() {
            const updateInfo = updateChecker.getUpdateInfo();
            // do box before app updates. for the off chance that the box logic fixes some app update logic issue
            if (updateInfo.box && !updateInfo.box.unstable) {
                debug('Starting box autoupdate to %j', updateInfo.box);
-                updater.updateToLatest({ skipBackup: false }, auditSource.CRON, NOOP_CALLBACK);
+                const [error] = await safe(updater.updateToLatest({ skipBackup: false }, AuditSource.CRON));
+                if (error) debug(`Failed to box autoupdate: ${error.message}`);
                return;
            }

            const appUpdateInfo = _.omit(updateInfo, 'box');
            if (Object.keys(appUpdateInfo).length > 0) {
                debug('Starting app update to %j', appUpdateInfo);
-                apps.autoupdateApps(appUpdateInfo, auditSource.CRON, NOOP_CALLBACK);
+                const [error] = await safe(apps.autoupdateApps(appUpdateInfo, AuditSource.CRON));
+                if (error) debug(`Failed to app autoupdate: ${error.message}`);
            } else {
                debug('No app auto updates available');
            }
@@ -221,7 +239,7 @@ function dynamicDnsChanged(enabled) {
    if (enabled) {
        gJobs.dynamicDns = new CronJob({
            cronTime: '5 * * * * *',    // we only update the records if the ip has changed.
-            onTick: dyndns.sync.bind(null, auditSource.CRON, NOOP_CALLBACK),
+            onTick: async () => await safe(dyndns.sync(AuditSource.CRON), { debug }),
            start: true
        });
    } else {
@@ -230,14 +248,10 @@ function dynamicDnsChanged(enabled) {
    }
 }

-function stopJobs(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    for (var job in gJobs) {
+async function stopJobs() {
+    for (const job in gJobs) {
        if (!gJobs[job]) continue;
        gJobs[job].stop();
        gJobs[job] = null;
    }
-
-    callback();
 }
@@ -15,14 +15,14 @@ exports = module.exports = {
 const assert = require('assert'),
    async = require('async'),
    BoxError = require('./boxerror.js'),
-    child_process = require('child_process'),
    constants = require('./constants.js'),
    debug = require('debug')('box:database'),
    mysql = require('mysql'),
-    once = require('once'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
    util = require('util');

-var gConnectionPool = null;
+let gConnectionPool = null;

 const gDatabase = {
    hostname: '127.0.0.1',
@@ -32,10 +32,8 @@ const gDatabase = {
    name: 'box'
 };

-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    if (gConnectionPool !== null) return callback(null);
+async function initialize() {
+    if (gConnectionPool !== null) return;

    if (constants.TEST) {
        // see setupTest script how the mysql-server is run
@@ -45,6 +43,8 @@ function initialize(callback) {
    // https://github.com/mysqljs/mysql#pool-options
    gConnectionPool  = mysql.createPool({
        connectionLimit: 5,
+        acquireTimeout: 60000,
+        connectTimeout: 60000,
        host: gDatabase.hostname,
        user: gDatabase.username,
        password: gDatabase.password,
@@ -61,60 +61,53 @@ function initialize(callback) {
        // note the pool also has an 'acquire' event but that is called whenever we do a getConnection()
        connection.on('error', (error) => debug(`Connection ${connection.threadId} error: ${error.message} ${error.code}`));

-        connection.query('USE ' + gDatabase.name);
+        connection.query(`USE ${gDatabase.name}`);
        connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
+        connection.query('SET SESSION group_concat_max_len = 65536'); // GROUP_CONCAT has only 1024 default
    });
-
-    callback(null);
 }

-function uninitialize(callback) {
-    if (!gConnectionPool) return callback(null);
+async function uninitialize() {
+    if (!gConnectionPool) return;

-    gConnectionPool.end(callback);
+    gConnectionPool.end();
    gConnectionPool = null;
 }

-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var cmd = util.format('mysql --host="%s" --user="%s" --password="%s" -Nse "SHOW TABLES" %s | grep -v "^migrations$" | while read table; do mysql --host="%s" --user="%s" --password="%s" -e "SET FOREIGN_KEY_CHECKS = 0; TRUNCATE TABLE $table" %s; done',
+async function clear() {
+    const cmd = util.format('mysql --host="%s" --user="%s" --password="%s" -Nse "SHOW TABLES" %s | grep -v "^migrations$" | while read table; do mysql --host="%s" --user="%s" --password="%s" -e "SET FOREIGN_KEY_CHECKS = 0; TRUNCATE TABLE $table" %s; done',
        gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name,
        gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name);

-    child_process.exec(cmd, callback);
+    await shell.promises.exec('clear_database', cmd);
 }

-function query() {
+async function query() {
    assert.notStrictEqual(gConnectionPool, null);

    return new Promise((resolve, reject) => {
        let args = Array.prototype.slice.call(arguments);
-        const callback = typeof args[args.length - 1] === 'function' ? args.pop() : null;

        args.push(function queryCallback(error, result) {
-            if (error) return callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+            if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage || null }));

-            callback ? callback(null, result) : resolve(result);
+            resolve(result);
        });

        gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
    });
 }

-function transaction(queries) {
+async function transaction(queries) {
    assert(Array.isArray(queries));

-    const args = Array.prototype.slice.call(arguments);
-    const callback = typeof args[args.length - 1] === 'function' ? once(args.pop()) : null;
-
    return new Promise((resolve, reject) => {
        gConnectionPool.getConnection(function (error, connection) {
-            if (error) return callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+            if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));

            const releaseConnection = (error) => {
                connection.release();
-                callback ? callback(error) : reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+                reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage || null }));
            };

            connection.beginTransaction(function (error) {
@@ -130,7 +123,7 @@ function transaction(queries) {

                        connection.release();

-                        callback ? callback(null, results) : resolve(results);
+                        resolve(results);
                    });
                });
            });
@@ -138,27 +131,27 @@ function transaction(queries) {
    });
 }

-function importFromFile(file, callback) {
+async function importFromFile(file) {
    assert.strictEqual(typeof file, 'string');
-    assert.strictEqual(typeof callback, 'function');

-    var cmd = `/usr/bin/mysql -h "${gDatabase.hostname}" -u ${gDatabase.username} -p${gDatabase.password} ${gDatabase.name} < ${file}`;
+    const cmd = `/usr/bin/mysql -h "${gDatabase.hostname}" -u ${gDatabase.username} -p${gDatabase.password} ${gDatabase.name} < ${file}`;

-    async.series([
-        query.bind(null, 'CREATE DATABASE IF NOT EXISTS box'),
-        child_process.exec.bind(null, cmd)
-    ], callback);
+    await query('CREATE DATABASE IF NOT EXISTS box');
+    const [error] = await safe(shell.promises.exec('importFromFile', cmd));
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 }

-function exportToFile(file, callback) {
+async function exportToFile(file) {
    assert.strictEqual(typeof file, 'string');
-    assert.strictEqual(typeof callback, 'function');

-    // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
-    // this option must not be set in production cloudrons which still use the old mysqldump
-    const disableColStats = (constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
+    // latest mysqldump enables column stats by default which is not present in 5.7 util
+    const mysqlDumpHelp = safe.child_process.execSync('/usr/bin/mysqldump --help', { encoding: 'utf8' });
+    if (!mysqlDumpHelp) throw new BoxError(BoxError.DATABASE_ERROR, safe.error);
+    const hasColStats = mysqlDumpHelp.includes('column-statistics');
+    const colStats = hasColStats ? '--column-statistics=0' : '';

-    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${disableColStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
+    const cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${colStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;

-    child_process.exec(cmd, callback);
+    const [error] = await safe(shell.promises.exec('exportToFile', cmd));
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 }
@@ -1,6 +1,6 @@
 'use strict';

-let assert = require('assert'),
+const assert = require('assert'),
    path = require('path');

 class DataLayout {
@@ -0,0 +1,13 @@
+'use strict';
+
+exports = module.exports = delay;
+
+const assert = require('assert');
+
+function delay(msecs) {
+    assert.strictEqual(typeof msecs, 'number');
+
+    return new Promise(function (resolve) {
+        setTimeout(resolve, msecs);
+    });
+}
@@ -0,0 +1,40 @@
+'use strict';
+
+exports = module.exports = {
+    resolve,
+};
+
+const assert = require('assert'),
+    constants = require('./constants.js'),
+    dns = require('dns'),
+    safe = require('safetydance'),
+    _ = require('underscore');
+
+// a note on TXT records. It doesn't have quotes ("") at the DNS level. Those quotes
+// are added for DNS server software to enclose spaces. Such quotes may also be returned
+// by the DNS REST API of some providers
+async function resolve(hostname, rrtype, options) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof rrtype, 'string');
+    assert(options && typeof options === 'object');
+
+    const defaultOptions = { server: '127.0.0.1', timeout: 5000 }; // unbound runs on 127.0.0.1
+    const resolver = new dns.promises.Resolver();
+    options = _.extend({ }, defaultOptions, options);
+
+    // Only use unbound on a Cloudron
+    if (constants.CLOUDRON) resolver.setServers([ options.server ]);
+
+    // should callback with ECANCELLED but looks like we might hit https://github.com/nodejs/node/issues/14814
+    const timerId = setTimeout(resolver.cancel.bind(resolver), options.timeout || 5000);
+
+    const [error, result] = await safe(resolver.resolve(hostname, rrtype));
+    clearTimeout(timerId);
+
+    if (error && error.code === 'ECANCELLED') error.code = 'TIMEOUT';
+    if (error) throw error;
+
+    // when you query a random record, it errors with ENOTFOUND. But, if you query a record which has a different type
+    // we sometimes get empty array and sometimes ENODATA. for TXT records, result is 2d array of strings
+    return result;
+}
@@ -0,0 +1,371 @@
+'use strict';
+
+exports = module.exports = {
+    start,
+    stop,
+
+    validateConfig,
+    applyConfig
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:directoryserver'),
+    dns = require('./dns.js'),
+    domains = require('./domains.js'),
+    eventlog = require('./eventlog.js'),
+    fs = require('fs'),
+    groups = require('./groups.js'),
+    ldap = require('ldapjs'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    reverseproxy = require('./reverseproxy.js'),
+    safe = require('safetydance'),
+    settings = require('./settings.js'),
+    speakeasy = require('speakeasy'),
+    shell = require('./shell.js'),
+    users = require('./users.js'),
+    util = require('util'),
+    validator = require('validator');
+
+let gServer = null;
+
+const NOOP = function () {};
+
+const SET_LDAP_ALLOWLIST_CMD = path.join(__dirname, 'scripts/setldapallowlist.sh');
+
+async function validateConfig(config) {
+    const { enabled, secret, allowlist } = config;
+
+    if (!enabled) return;
+
+    if (!secret) throw new BoxError(BoxError.BAD_FIELD, 'secret cannot be empty');
+
+    let gotOne = false;
+    for (const line of allowlist.split('\n')) {
+        if (!line || line.startsWith('#')) continue;
+        const rangeOrIP = line.trim();
+        // this checks for IPv4 and IPv6
+        if (!validator.isIP(rangeOrIP) && !validator.isIPRange(rangeOrIP)) throw new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} is not a valid IP or range`);
+        gotOne = true;
+    }
+
+    // only allow if we at least have one allowed IP/range
+    if (!gotOne) throw new BoxError(BoxError.BAD_FIELD, 'allowlist must at least contain one IP or range');
+}
+
+async function applyConfig(config) {
+    assert.strictEqual(typeof config, 'object');
+
+    // this is done only because it's easier for the shell script and the firewall service to get the value
+    if (config.enabled) {
+        if (!safe.fs.writeFileSync(paths.LDAP_ALLOWLIST_FILE, config.allowlist + '\n', 'utf8')) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
+    } else {
+        safe.fs.unlinkSync(paths.LDAP_ALLOWLIST_FILE);
+    }
+
+    const [error] = await safe(shell.promises.sudo('setLdapAllowlist', [ SET_LDAP_ALLOWLIST_CMD ], {}));
+    if (error) throw new BoxError(BoxError.IPTABLES_ERROR, `Error setting ldap allowlist: ${error.message}`);
+
+    if (config.enabled) await start(); else await stop();
+}
+
+// helper function to deal with pagination
+function finalSend(results, req, res, next) {
+    let min = 0;
+    let max = results.length;
+    let cookie = null;
+    let pageSize = 0;
+
+    // check if this is a paging request, if so get the cookie for session info
+    req.controls.forEach(function (control) {
+        if (control.type === ldap.PagedResultsControl.OID) {
+            pageSize = control.value.size;
+            cookie = control.value.cookie;
+        }
+    });
+
+    function sendPagedResults(start, end) {
+        start = (start < min) ? min : start;
+        end = (end > max || end < min) ? max : end;
+        let i;
+
+        for (i = start; i < end; i++) {
+            res.send(results[i]);
+        }
+
+        return i;
+    }
+
+    if (cookie && Buffer.isBuffer(cookie)) {
+        // we have pagination
+        let first = min;
+        if (cookie.length !== 0) {
+            first = parseInt(cookie.toString(), 10);
+        }
+        const last = sendPagedResults(first, first + pageSize);
+
+        let resultCookie;
+        if (last < max) {
+            resultCookie = Buffer.from(last.toString());
+        } else {
+            resultCookie = Buffer.from('');
+        }
+
+        res.controls.push(new ldap.PagedResultsControl({
+            value: {
+                size: pageSize, // correctness not required here
+                cookie: resultCookie
+            }
+        }));
+    } else {
+        // no pagination simply send all
+        results.forEach(function (result) {
+            res.send(result);
+        });
+    }
+
+    // all done
+    res.end();
+    next();
+}
+
+async function authorize(req, res, next) {
+    debug('authorize: ', req.connection.ldap.bindDN.toString());
+
+    // this is for connection attempts without previous bind
+    if (req.connection.ldap.bindDN.equals('cn=anonymous')) return next(new ldap.InsufficientAccessRightsError());
+
+    // we only allow this one DN to pass
+    if (!req.connection.ldap.bindDN.equals(constants.USER_DIRECTORY_LDAP_DN)) return next(new ldap.InsufficientAccessRightsError());
+
+    return next();
+}
+
+// https://ldapwiki.com/wiki/RootDSE / RFC 4512 - ldapsearch -x -h "${CLOUDRON_LDAP_SERVER}" -p "${CLOUDRON_LDAP_PORT}" -b  "" -s base
+// ldapjs seems to call this handler for everything when search === ''
+async function maybeRootDSE(req, res, next) {
+    debug(`maybeRootDSE: requested with scope:${req.scope} dn:${req.dn.toString()}`);
+
+    if (req.scope !== 'base') return next(new ldap.NoSuchObjectError()); // per the spec, rootDSE search require base scope
+    if (!req.dn || req.dn.toString() !== '') return next(new ldap.NoSuchObjectError());
+
+    res.send({
+        dn: '',
+        attributes: {
+            objectclass: [ 'RootDSE', 'top', 'OpenLDAProotDSE' ],
+            supportedLDAPVersion: '3',
+            vendorName: 'Cloudron LDAP',
+            vendorVersion: '1.0.0'
+        }
+    });
+    res.end();
+}
+
+async function userSearch(req, res, next) {
+    debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    const [error, allUsers] = await safe(users.list());
+    if (error) return next(new ldap.OperationsError(error.toString()));
+
+    const [groupsError, allGroups] = await safe(groups.listWithMembers());
+    if (groupsError) return next(new ldap.OperationsError(error.toString()));
+
+    let results = [];
+
+    // send user objects
+    for (const user of allUsers) {
+        // skip entries with empty username. Some apps like owncloud can't deal with this
+        if (!user.username) continue;
+
+        const dn = ldap.parseDN(`cn=${user.id},ou=users,dc=cloudron`);
+
+        const displayName = user.displayName || user.username || ''; // displayName can be empty and username can be null
+        const nameParts = displayName.split(' ');
+        const firstName = nameParts[0];
+        const lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
+
+        const obj = {
+            dn: dn.toString(),
+            attributes: {
+                objectclass: ['user', 'inetorgperson', 'person' ],
+                objectcategory: 'person',
+                cn: displayName,
+                uid: user.id,
+                entryuuid: user.id, // to support OpenLDAP clients
+                mail: user.email,
+                mailAlternateAddress: user.fallbackEmail,
+                displayname: displayName,
+                givenName: firstName,
+                username: user.username,
+                samaccountname: user.username,      // to support ActiveDirectory clients
+                memberof: allGroups.filter(function (g) { return g.userIds.indexOf(user.id) !== -1; }).map(function (g) { return g.name; })
+            }
+        };
+
+        if (user.twoFactorAuthenticationEnabled) obj.attributes.twoFactorAuthenticationEnabled = true;
+
+        // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
+        // which is required to have atleast one character if present
+        if (lastName.length !== 0) obj.attributes.sn = lastName;
+
+        // ensure all filter values are also lowercase
+        const lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+            results.push(obj);
+        }
+    }
+
+    finalSend(results, req, res, next);
+}
+
+async function groupSearch(req, res, next) {
+    debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    const [error, allUsers] = await safe(users.list());
+    if (error) return next(new ldap.OperationsError(error.toString()));
+
+    const results = [];
+
+    let [errorGroups, allGroups] = await safe(groups.listWithMembers());
+    if (errorGroups) return next(new ldap.OperationsError(errorGroups.toString()));
+
+    for (const group of allGroups) {
+        const dn = ldap.parseDN(`cn=${group.name},ou=groups,dc=cloudron`);
+        const members = group.userIds.filter(function (uid) { return allUsers.map(function (u) { return u.id; }).indexOf(uid) !== -1; });
+
+        const obj = {
+            dn: dn.toString(),
+            attributes: {
+                objectclass: ['group'],
+                cn: group.name,
+                gidnumber: group.id,
+                memberuid: members
+            }
+        };
+
+        // ensure all filter values are also lowercase
+        const lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+            results.push(obj);
+        }
+    }
+
+    finalSend(results, req, res, next);
+}
+
+// Will attach req.user if successful
+async function userAuth(req, res, next) {
+    // extract the common name which might have different attribute names
+    const cnAttributeName = Object.keys(req.dn.rdns[0].attrs)[0];
+    const commonName = req.dn.rdns[0].attrs[cnAttributeName].value;
+    if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+    const TOTPTOKEN_ATTRIBUTE_NAME = 'totptoken'; // This has to be in-sync with externalldap.js
+    const totpToken = req.dn.rdns[0].attrs[TOTPTOKEN_ATTRIBUTE_NAME] ? req.dn.rdns[0].attrs[TOTPTOKEN_ATTRIBUTE_NAME].value : null;
+
+    let verifyFunc;
+    if (cnAttributeName === 'mail') {
+        verifyFunc = users.verifyWithEmail;
+    } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
+        verifyFunc = users.verifyWithEmail;
+    } else if (commonName.indexOf('uid-') === 0) {
+        verifyFunc = users.verify;
+    } else {
+        verifyFunc = users.verifyWithUsername;
+    }
+
+    const [error, user] = await safe(verifyFunc(commonName, req.credentials || '', ''));
+    if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+    if (error) return next(new ldap.OperationsError(error.message));
+
+    // currently this is only optional if totpToken is provided and user has 2fa enabled
+    if (totpToken && user.twoFactorAuthenticationEnabled) {
+        const verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: totpToken, window: 2 });
+        if (!verified) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+    }
+
+    req.user = user;
+
+    next();
+}
+
+// FIXME this needs to be restarted if settings changes or dashboard cert got renewed
+async function start() {
+    if (gServer) return; // already running
+
+    const logger = {
+        trace: NOOP,
+        debug: NOOP,
+        info: debug,
+        warn: debug,
+        error: debug,
+        fatal: debug
+    };
+
+    const domainObject = await domains.get(settings.dashboardDomain());
+    const dashboardFqdn = dns.fqdn(constants.DASHBOARD_SUBDOMAIN, domainObject);
+    const certificatePath = await reverseproxy.getCertificatePath(dashboardFqdn, domainObject.domain);
+
+    gServer = ldap.createServer({
+        certificate: fs.readFileSync(certificatePath.certFilePath, 'utf8'),
+        key: fs.readFileSync(certificatePath.keyFilePath, 'utf8'),
+        log: logger
+    });
+
+    gServer.on('error', function (error) {
+        debug('server startup error ', error);
+    });
+
+    gServer.bind('ou=system,dc=cloudron', async function(req, res, next) {
+        debug('system bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
+
+        const tmp = await settings.getDirectoryServerConfig();
+
+        if (!req.dn.equals(constants.USER_DIRECTORY_LDAP_DN)) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+        if (req.credentials !== tmp.secret) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+
+        req.user = { user: 'directoryServerAdmin' };
+
+        res.end();
+
+        // if we use next in the callback, ldapjs requires this after res.end();
+        return next();
+    });
+
+    gServer.search('ou=users,dc=cloudron', authorize, userSearch);
+    gServer.search('ou=groups,dc=cloudron', authorize, groupSearch);
+    gServer.bind('ou=users,dc=cloudron', userAuth, async function (req, res) {
+        assert.strictEqual(typeof req.user, 'object');
+
+        await eventlog.upsertLoginEvent(req.user.ghost ? eventlog.ACTION_USER_LOGIN_GHOST : eventlog.ACTION_USER_LOGIN, { authType: 'directoryserver', id: req.connection.ldap.id }, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+
+        res.end();
+    });
+
+    gServer.search('', maybeRootDSE); // when '', it seems the callback is called for everything else
+
+    // just log that an attempt was made to unknown route, this helps a lot during app packaging
+    gServer.use(function(req, res, next) {
+        debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);
+        return next();
+    });
+
+    debug(`starting server on port ${constants.USER_DIRECTORY_LDAPS_PORT}`);
+    await util.promisify(gServer.listen.bind(gServer))(constants.USER_DIRECTORY_LDAPS_PORT, '::');
+}
+
+async function stop() {
+    if (!gServer) return;
+
+    debug('stopping server');
+
+    gServer.close();
+    gServer = null;
+}
--- a/Show More
+++ b/Show More