graphite: carbon crash fix

https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=923464
https://forum.cloudron.io/topic/4797/graphite-keeps-crashing-oom/34
(cherry picked from commit cd300bb6e2)

.eslintrc.json

@@ -5,7 +5,7 @@
     },
     "extends": "eslint:recommended",
     "parserOptions": {
-        "ecmaVersion": 2020
+        "ecmaVersion": 8
     },
     "rules": {
         "indent": [

.gitignore

@@ -1,6 +1,5 @@
 node_modules/
 coverage/
-.nyc_output/
 webadmin/dist/
 installer/src/certs/server.key
 

.jshintrc

@@ -1,5 +0,0 @@
-{
-    "node": true,
-    "unused": true,
-    "esversion": 11
-}

CHANGES

@@ -2243,7 +2243,7 @@
 * Fix issue where collectd is restarted too quickly before graphite
 
 [6.2.7]
-* redis: backup before upgrade
+* collectd: restart on graphite upgrade
 
 [6.2.8]
 * linode object storage: update aws sdk to make it work again
@@ -2253,258 +2253,3 @@
 * turn: turn off verbose logging
 * Fix crash when parsing df output (set LC_ALL for box service)
 
-[6.3.0]
-* mail: allow TLS from internal hosts
-* tokens: add lastUsedTime
-* update: set memory limit properly
-* addons: better error handling
-* filemanager: various enhancements
-* sftp: fix rebuild condition
-* app mailbox is now optional
-* Fix display of user management/dashboard visiblity for email apps
-* graphite: disable tagdb and reduce log noise
-* hsts: change max-age to 2 years
-* clone: copy over redis memory limit
-* namecheap: fix bug where records were not removed
-* add UI to disable 2FA of a user
-* mail: add active flag to mailboxes and lists
-* Implement OCSP stapling
-* security: send new browser login location notification email
-* backups: add fqdn to the backup filename
-* import all boxdata settings into the database
-* volumes: generate systemd mount configs based on type
-* postgresql: set max conn limit per db
-* ubuntu 16: add alert about EOL
-* clone: save and restore app config
-* app import: restore icon, tag, label, proxy configs etc
-* sieve: fix redirects to not do SRS
-* notifications are now system level instead of per-user
-* vultr DNS
-* vultr object storage
-* mail: do not forward spam to mailing lists
-
-[6.3.1]
-* Fix cert migration issues
-
-[6.3.2]
-* Avatar was migrated as base64 instead of binary
-* Fix issue where filemanager came up empty for CIFS mounts
-
-[6.3.3]
-* volumes: add filesystem volume type for shared folders
-* mail: enable sieve extension editheader
-* mail: update solr to 8.9.0
-
-[6.3.4]
-* Fix issue where old nginx configs where not removed before upgrade
-
-[6.3.5]
-* Fix permission issues with sshfs
-* filemanager: reset selection if directory has changed
-* branding: fix error highlight with empty cloudron name
-* better text instead of "Cloudron in the wild"
-* Make sso login hint translatable
-* Give unread notifications a small left border
-* Fix issue where clicking update indicator opened app in new tab
-* Ensure notifications are only fetched and shown for at least admins
-* setupaccount: Show input field errors below input field
-* Set focus automatically for new alias or redirect
-* eventlog: fix issue where old events are not periodically removed
-* ssfs: fix chown
-
-[6.3.6]
-* Fix broken reboot button
-* app updated notification shown despite failure
-* Update translation for sso login information
-* Hide groups/tags/state filter in app listing for normal users
-* filemanager: Ensure breadcrumbs and hash are correctly updated on folder navigation
-* cloudron-setup: check if nginx/docker is already installed
-* Use the addresses of all available interfaces for port 53 binding
-* refresh config on appstore login
-* password reset: check 2fa when enabled
-
-[7.0.0]
-* Ubuntu 16 is not supported anymore
-* Do not use Gravatar as the default but only an option
-* redis: suppress password warning
-* setup UI: fix dark mode
-* wellknown: response to .wellknown/matrix/client
-* purpose field is not required anymore during appstore signup
-* sftp: fix symlink deletion
-* Show correct/new app version info in updated finished notification
-* Make new login email translatable
-* Hide ticket form if cloudron.io mail is not verified
-* Refactor code to use async/await
-* postgresql: bump shm size and disable parallel queries
-* update nodejs to 14.17.6
-* external ldap: If we detect a local user with the same username as found on LDAP/AD we map it
-* add basic eventlog for apps in app view
-* Enable sshfs/cifs/nfs in app import UI
-* Require password for fallback email change
-* Make password reset logic translatable
-* support: only verified email address can open support tickets
-* Logout users without 2FA when mandatory 2fa is enabled
-* notifications: better oom message for redis
-* Add way to impersonate users for presetup
-* mail: open up port 465 for mail submission (TLS)
-* Implement operator role for apps
-* sftp: normal users do not have SFTP access anymore. Use operator role instead
-* eventlog: add service rebuild/restart/configure events
-* upcloud: add object storage integration
-* Each app can now have a custom crontab
-* services: add recovery mode
-* postgresql: fix restore issue with long table names
-* recvmail: make the addon work again
-* mail: update solr to 8.10.0
-* mail: POP3 support
-* update docker to 20.10.7
-* volumes: add remount button
-* mail: add spam eventlog filter type
-* mail: configure dnsbl
-* mail: add duplication detection for lists
-* mail: add SRS for Sieve Forwarding
-
-[7.0.1]
-* Fix matrix wellKnown client migration
-
-[7.0.2]
-* mail: POP3 flag was not returned correctly
-* external ldap: fix crash preventing users from logging in
-* volumes: ensure we don't crash if mount status is unexpected
-* backups: set default backup memory limit to 800
-* users: allow admins to specify password recovery email
-* retry startup tasks on database error
-
-[7.0.3]
-* support: fix remoe support not working for 'root' user
-* Fix cog icon on app grid item hover for darkmode
-* Disable password reset and impersonate button for self user instead of hiding them
-* pop3: fix crash with auth of non-existent mailbox
-* mail: fix direction field in eventlog of deferred mails
-* mail: fix eventlog search
-* mail: save message-id in eventlog
-* backups: fix issue which resulted in incomplete backups when an app has backups disabled
-* restore: do not redirect until mail data has been restored
-* proxyauth: set viewport meta tag in login view
-
-[7.0.4]
-* Add password reveal button to login pages
-* appstore: fix crash if account already registered
-* Do not nuke all the logrotate configs on update
-* Remove unused httpPaths from manifest
-* cloudron-support: add option to reset cloudron.io account
-* Fix flicker in login page
-* Fix LE account key re-use issue in DO 1-click image
-* mail: add non-tls ports for recvmail addon
-* backups: fix issue where mail backups where not cleaned up
-* notifications: fix automatic app update notifications
-
-[7.1.0]
-* Add mail manager role
-* mailbox: app can be set as owner when recvmail addon enabled
-* domains: add well known config UI (for jitsi configuration)
-* Prefix email addon variables with CLOUDRON_EMAIL instead of CLOUDRON_MAIL
-* remove support for manifest version 1
-* Add option to enable/disable mailbox sharing
-* base image 3.2.0
-* Update node to 16.13.1
-* mongodb: update to 4.4
-* Add `upstreamVersion` to manifest
-* Add `logPaths` to manifest
-* Add cifs seal support for backup and volume mounts
-* add a way for admins to set username when profiles are locked
-* Add support for secondary domains
-* postgresql: enable postgis
-* remove nginx config of stopped apps
-* mail: use port25check.cloudron.io to check outbound port 25 connectivity
-* Add import/export of mailboxes and users
-* LDAP server can now be exposed
-* Update monaco-editor to 0.32.1
-* Update xterm.js to 4.17.0
-* Update docker to 20.10.12
-* IPv6 support
-
-[7.1.1]
-* Fix issue where dkimKey of a mail domain is sometimes null
-* firewall: add retry for xtables lock
-* redis: fix issue where protected mode was enabled with no password
-
-[7.1.2]
-* Fix crash in cloudron-firewall when ports are whitelisted
-* eventlog: add event for certificate cleanup
-* eventlog: log event for mailbox alias update
-* backups: fix incorrect mountpoint check with managed mounts
-
-[7.1.3]
-* Fix security issue where an admin can impersonate an owner
-* block list: can upload up to 2MB
-* dns: fix issue where link local address was picked up for ipv6
-* setup: ufw may not be installed
-* mysql: fix default collation of databases
-
-[7.1.4]
-* wildcard dns: fix handling of ENODATA
-* cloudflare: fix error handling
-* openvpn: ipv6 support
-* dyndns: fix issue where eventlog was getting filled with empty entries
-* mandatory 2fa: Fix typo in 2FA check
-
-[7.2.0]
-* mail: hide log button for non-superadmins
-* firewall: do not add duplicate ldap redirect rules
-* ldap: respond to RootDSE
-* Check if CNAME record exists and remove it if overwrite is set
-* cifs: use credentials file for better password support
-* installer: rework script to fix DNS resolution issues
-* backup cleaner: do not clean if not mounted
-* restore: fix sftp private key perms
-* support: add a separate system user named cloudron-support
-* sshfs: fix bug where sshfs mounts were generated without unbound dependancy
-* cloudron-setup: add --setup-token
-* notifications: add installation event
-* backups: set label of backup and control it's retention
-* wasabi: add new regions (London, Frankfurt, Paris, Toronto)
-* docker: update to 20.10.14
-* Ensure LDAP usernames are always treated lowercase
-* Add a way to make LDAP users local
-* proxyAuth: set X-Remote-User (rfc3875)
-* GoDaddy: there is now a delete API
-* nginx: use ubuntu packages for ubuntu 20.04 and 22.04
-* Ubuntu 22.04 LTS support
-* Add Hetzner DNS
-* cron: add support for extensions (@reboot, @weekly etc)
-* Add profile backgroundImage api
-* exec: rework API to get exit code
-* Add update available filter
-
-[7.2.1]
-* Refactor backup code to use async/await
-* mongodb: fix bug where a small timeout prevented import of large backups
-* Add update available filter
-* exec: rework API to get exit code
-* Add profile backgroundImage api
-* cron: add support for extensions (@reboot, @weekly etc)
-
-[7.2.2]
-* Update cloudron-manifestformat for new scheduler patterns
-* collectd: FQDNLookup causes collectd install to fail
-
-[7.2.3]
-* appstore: allow re-registration on server side delete
-* transfer ownership route is not used anymore
-* graphite: fix issue where disk names with '.' do not render
-* dark mode fixes
-* sendmail: mail from display name
-* Use volumes for app data instead of raw path
-* initial xfs support
-
-[7.2.4]
-* volumes: Ensure long volume names do not overflow the table
-* Move all appstore filter to the left
-* app data: allow sameness of old and new dir
-
-[7.2.5]
-* Fix storage volume migration
-* Fix issue where only 25 group members were returned
-* Fix eventlog display
-

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2022 Cloudron UG
+Copyright (c) 2020 Cloudron UG
 
 With regard to the Cloudron Software:
 

README.md

@@ -1,5 +1,3 @@
-![Translation status](https://translate.cloudron.io/widgets/cloudron/-/svg-badge.svg)
-
 # Cloudron
 
 [Cloudron](https://cloudron.io) is the best way to run apps on your server.
@@ -72,13 +70,8 @@ Just to give some heads up, we are a bit restrictive in merging changes. We are
 would like to keep our maintenance burden low. It might be best to discuss features first in the [forum](https://forum.cloudron.io),
 to also figure out how many other people will use it to justify maintenance for a feature.
 
-# Localization
-
-![Translation status](https://translate.cloudron.io/widgets/cloudron/-/287x66-white.png)
-
 ## Support
 
 * [Documentation](https://docs.cloudron.io/)
 * [Forum](https://forum.cloudron.io/)
 
-

baseimage/initializeBaseUbuntuImage.sh

@@ -0,0 +1,176 @@
+#!/bin/bash
+
+set -euv -o pipefail
+
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+readonly arg_infraversionpath="${SOURCE_DIR}/../src"
+
+function die {
+    echo $1
+    exit 1
+}
+
+export DEBIAN_FRONTEND=noninteractive
+
+readonly ubuntu_codename=$(lsb_release -cs)
+readonly ubuntu_version=$(lsb_release -rs)
+
+# hold grub since updating it breaks on some VPS providers. also, dist-upgrade will trigger it
+apt-mark hold grub* >/dev/null
+apt-get -o Dpkg::Options::="--force-confdef" update -y
+apt-get -o Dpkg::Options::="--force-confdef" upgrade -y
+apt-mark unhold grub* >/dev/null
+
+echo "==> Installing required packages"
+
+debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
+debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
+
+# this enables automatic security upgrades (https://help.ubuntu.com/community/AutomaticSecurityUpdates)
+# resolvconf is needed for unbound to work property after disabling systemd-resolved in 18.04
+
+gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
+mysql_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "mysql-server-8.0" || echo "mysql-server-5.7")
+apt-get -y install --no-install-recommends \
+    acl \
+    apparmor \
+    build-essential \
+    cifs-utils \
+    cron \
+    curl \
+    debconf-utils \
+    dmsetup \
+    $gpg_package \
+    ipset \
+    iptables \
+    libpython2.7 \
+    linux-generic \
+    logrotate \
+    $mysql_package \
+    openssh-server \
+    pwgen \
+    resolvconf \
+    swaks \
+    tzdata \
+    unattended-upgrades \
+    unbound \
+    unzip \
+    xfsprogs
+
+echo "==> installing nginx for xenial for TLSv3 support"
+curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
+# apt install with install deps (as opposed to dpkg -i)
+apt install -y /tmp/nginx.deb
+rm /tmp/nginx.deb
+
+# on some providers like scaleway the sudo file is changed and we want to keep the old one
+apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends sudo
+
+# this ensures that unattended upgades are enabled, if it was disabled during ubuntu install time (see #346)
+# debconf-set-selection of unattended-upgrades/enable_auto_updates + dpkg-reconfigure does not work
+cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
+
+echo "==> Installing node.js"
+readonly node_version=14.15.4
+mkdir -p /usr/local/node-${node_version}
+curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-${node_version}
+ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
+ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
+apt-get install -y --no-install-recommends python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
+
+# https://docs.docker.com/engine/installation/linux/ubuntulinux/
+echo "==> Installing Docker"
+
+# create systemd drop-in file. if you channge options here, be sure to fixup installer.sh as well
+mkdir -p /etc/systemd/system/docker.service.d
+echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
+
+# there are 3 packages for docker - containerd, CLI and the daemon
+readonly docker_version=20.10.3
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
+# apt install with install deps (as opposed to dpkg -i)
+apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
+rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
+
+storage_driver=$(docker info | grep "Storage Driver" | sed 's/.*: //')
+if [[ "${storage_driver}" != "overlay2" ]]; then
+    echo "Docker is using "${storage_driver}" instead of overlay2"
+    exit 1
+fi
+
+# do not upgrade grub because it might prompt user and break this script
+echo "==> Enable memory accounting"
+apt-get -y --no-upgrade --no-install-recommends install grub2-common
+sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
+update-grub
+
+echo "==> Downloading docker images"
+if [ ! -f "${arg_infraversionpath}/infra_version.js" ]; then
+    echo "No infra_versions.js found"
+    exit 1
+fi
+
+images=$(node -e "var i = require('${arg_infraversionpath}/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
+
+echo -e "\tPulling docker images: ${images}"
+for image in ${images}; do
+    docker pull "${image}"
+    docker pull "${image%@sha256:*}" # this will tag the image for readability
+done
+
+echo "==> Install collectd"
+# without this, libnotify4 will install gnome-shell
+apt-get install -y libnotify4 --no-install-recommends
+if ! apt-get install -y --no-install-recommends libcurl3-gnutls collectd collectd-utils; then
+    # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
+    echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
+    sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
+fi
+# https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
+[[ "${ubuntu_version}" == "20.04" ]] && echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
+
+# some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
+echo "==> Configuring host"
+sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
+if systemctl is-active ntp; then
+    systemctl stop ntp
+    apt purge -y ntp
+fi
+timedatectl set-ntp 1
+# mysql follows the system timezone
+timedatectl set-timezone UTC
+
+echo "==> Adding sshd configuration warning"
+sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://docs.cloudron.io/security/#securing-ssh-access' -i /etc/ssh/sshd_config
+
+# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
+echo "==> Disabling motd news"
+if [ -f "/etc/default/motd-news" ]; then
+    sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
+fi
+
+# Disable bind for good measure (on online.net, kimsufi servers these are pre-installed)
+systemctl stop bind9 || true
+systemctl disable bind9 || true
+
+# on ovh images dnsmasq seems to run by default
+systemctl stop dnsmasq || true
+systemctl disable dnsmasq || true
+
+# on ssdnodes postfix seems to run by default
+systemctl stop postfix || true
+systemctl disable postfix || true
+
+# on ubuntu 18.04 and 20.04, this is the default. this requires resolvconf for DNS to work further after the disable
+systemctl stop systemd-resolved || true
+systemctl disable systemd-resolved || true
+
+# ubuntu's default config for unbound does not work if ipv6 is disabled. this config is overwritten in start.sh
+# we need unbound to work as this is required for installer.sh to do any DNS requests
+ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
+echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: ${ip6}" > /etc/unbound/unbound.conf.d/cloudron-network.conf
+systemctl restart unbound

box.js

@@ -2,76 +2,65 @@
 
 'use strict';
 
-const fs = require('fs'),
+let async = require('async'),
+    dockerProxy = require('./src/dockerproxy.js'),
+    fs = require('fs'),
     ldap = require('./src/ldap.js'),
     paths = require('./src/paths.js'),
     proxyAuth = require('./src/proxyauth.js'),
-    safe = require('safetydance'),
-    server = require('./src/server.js'),
-    settings = require('./src/settings.js'),
-    userdirectory = require('./src/userdirectory.js');
+    server = require('./src/server.js');
 
-let logFd;
+const NOOP_CALLBACK = function () { };
 
-async function setupLogging() {
-    if (process.env.BOX_ENV === 'test') return;
+function setupLogging(callback) {
+    if (process.env.BOX_ENV === 'test') return callback();
 
-    logFd = fs.openSync(paths.BOX_LOG_FILE, 'a');
-    // we used to write using a stream before but it caches internally and there is no way to flush it when things crash
-    process.stdout.write = process.stderr.write = function (...args) {
-        const callback = typeof args[args.length-1] === 'function' ? args.pop() : function () {}; // callback is required for fs.write
-        fs.write.apply(fs, [logFd, ...args, callback]);
-    };
+    var logfileStream = fs.createWriteStream(paths.BOX_LOG_FILE, { flags:'a' });
+    process.stdout.write = process.stderr.write = logfileStream.write.bind(logfileStream);
+
+    callback();
 }
 
-// this is also used as the 'uncaughtException' handler which can only have synchronous functions
-function exitSync(status) {
-    if (status.error) fs.write(logFd, status.error.stack + '\n', function () {});
-    fs.fsyncSync(logFd);
-    fs.closeSync(logFd);
-    process.exit(status.code);
-}
+async.series([
+    setupLogging,
+    server.start, // do this first since it also inits the database
+    proxyAuth.start,
+    ldap.start,
+    dockerProxy.start
+], function (error) {
+    if (error) {
+        console.log('Error starting server', error);
+        process.exit(1);
+    }
 
-async function startServers() {
-    await setupLogging();
-    await server.start(); // do this first since it also inits the database
-    await proxyAuth.start();
-    await ldap.start();
-
-    const conf = await settings.getUserDirectoryConfig();
-    if (conf.enabled) await userdirectory.start();
-}
-
-async function main() {
-    const [error] = await safe(startServers());
-    if (error) return exitSync({ error: new Error(`Error starting server: ${JSON.stringify(error)}`), code: 1 });
-
-    // require this here so that logging handler is already setup
+    // require those here so that logging handler is already setup
+    require('supererror');
     const debug = require('debug')('box:box');
 
-    process.on('SIGINT', async function () {
+    process.on('SIGINT', function () {
         debug('Received SIGINT. Shutting down.');
 
-        await proxyAuth.stop();
-        await server.stop();
-        await userdirectory.stop();
-        await ldap.stop();
+        proxyAuth.stop(NOOP_CALLBACK);
+        server.stop(NOOP_CALLBACK);
+        ldap.stop(NOOP_CALLBACK);
+        dockerProxy.stop(NOOP_CALLBACK);
         setTimeout(process.exit.bind(process), 3000);
     });
 
-    process.on('SIGTERM', async function () {
+    process.on('SIGTERM', function () {
         debug('Received SIGTERM. Shutting down.');
 
-        await proxyAuth.stop();
-        await server.stop();
-        await userdirectory.stop();
-        await ldap.stop();
+        proxyAuth.stop(NOOP_CALLBACK);
+        server.stop(NOOP_CALLBACK);
+        ldap.stop(NOOP_CALLBACK);
+        dockerProxy.stop(NOOP_CALLBACK);
         setTimeout(process.exit.bind(process), 3000);
     });
 
-    process.on('uncaughtException', (error) => exitSync({ error, code: 1 }));
+    process.on('uncaughtException', function (error) {
+        console.error((error && error.stack) ? error.stack : error);
+        setTimeout(process.exit.bind(process, 1), 3000);
+    });
 
     console.log(`Cloudron is up and running. Logs are at ${paths.BOX_LOG_FILE}`); // this goes to journalctl
-}
-
-main();
+});

crashnotifierservice.js

@@ -2,21 +2,27 @@
 
 'use strict';
 
-const database = require('./src/database.js');
+var database = require('./src/database.js');
 
-const crashNotifier = require('./src/crashnotifier.js');
+var crashNotifier = require('./src/crashnotifier.js');
 
 // This is triggered by systemd with the crashed unit name as argument
-async function main() {
+function main() {
     if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <unitName>');
 
-    const unitName = process.argv[2];
+    var unitName = process.argv[2];
     console.log('Started crash notifier for', unitName);
 
     // eventlog api needs the db
-    await database.initialize();
+    database.initialize(function (error) {
+        if (error) return console.error('Cannot connect to database. Unable to send crash log.', error);
 
-    await crashNotifier.sendFailureLogs(unitName);
+        crashNotifier.sendFailureLogs(unitName, function (error) {
+            if (error) console.error(error);
+
+            process.exit();
+        });
+    });
 }
 
 main();

migrations/20210315194623-tokens-add-lastUsedTime.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE tokens ADD COLUMN lastUsedTime TIMESTAMP NULL', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE tokens DROP COLUMN lastUsedTime', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20210317014101-apps-add-enableMailbox.js

@@ -1,16 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN enableMailbox BOOLEAN DEFAULT 1', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN enableMailbox', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20210415030645-mailboxes-add-active.js

@@ -1,17 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE mailboxes ADD COLUMN active BOOLEAN DEFAULT 1', function (error) {
-        if (error) return callback(error);
-
-        callback();
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE mailboxes DROP COLUMN active', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20210429194328-users-add-avatar.js

@@ -1,37 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    fs = require('fs'),
-    path = require('path');
-
-const AVATAR_DIR = '/home/yellowtent/boxdata/profileicons';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN avatar MEDIUMBLOB', function (error) {
-        if (error) return callback(error);
-
-        fs.readdir(AVATAR_DIR, function (error, filenames) {
-            if (error && error.code === 'ENOENT') return callback();
-            if (error) return callback(error);
-
-            async.eachSeries(filenames, function (filename, iteratorCallback) {
-                const avatar = fs.readFileSync(path.join(AVATAR_DIR, filename));
-                const userId = filename;
-
-                db.runSql('UPDATE users SET avatar=? WHERE id=?', [ avatar, userId ], iteratorCallback);
-            }, function (error) {
-                if (error) return callback(error);
-
-                fs.rmdir(AVATAR_DIR, { recursive: true }, callback);
-            });
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN avatar', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20210429224354-settings-add-valueBlob.js

@@ -1,20 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE settings ADD COLUMN valueBlob MEDIUMBLOB', function (error) {
-        if (error) return callback(error);
-
-        fs.readFile('/home/yellowtent/boxdata/avatar.png', function (error, avatar) {
-            if (error && error.code === 'ENOENT') return callback();
-            if (error) return callback(error);
-
-            db.runSql('INSERT INTO settings (name, valueBlob) VALUES (?, ?)', [ 'cloudron_avatar', avatar ], callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20210430120936-users-add-loginLocationsJson.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN loginLocationsJson TEXT', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN loginLocationsJson', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20210430200947-apps-add-icon.js

@@ -1,42 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    fs = require('fs'),
-    path = require('path');
-
-const APPICONS_DIR = '/home/yellowtent/boxdata/appicons';
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN icon MEDIUMBLOB'),
-        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN appStoreIcon MEDIUMBLOB'),
-        function migrateIcons(next) {
-            fs.readdir(APPICONS_DIR, function (error, filenames) {
-                if (error && error.code === 'ENOENT') return next();
-                if (error) return next(error);
-
-                async.eachSeries(filenames, function (filename, iteratorCallback) {
-                    const icon = fs.readFileSync(path.join(APPICONS_DIR, filename));
-                    const appId = filename.split('.')[0];
-
-                    if (filename.endsWith('.user.png')) {
-                        db.runSql('UPDATE apps SET icon=? WHERE id=?', [ icon, appId ], iteratorCallback);
-                    } else {
-                        db.runSql('UPDATE apps SET appStoreIcon=? WHERE id=?', [ icon, appId ], iteratorCallback);
-                    }
-                }, function (error) {
-                    if (error) return next(error);
-
-                    fs.rmdir(APPICONS_DIR, { recursive: true }, next);
-                });
-            });
-        }
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN icon'),
-        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN appStoreIcon'),
-    ], callback);
-};

migrations/20210430230901-apps-alter-ts-update.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps MODIFY ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps MODIFY ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20210501044940-blobs-add-table.js

@@ -1,20 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    const cmd = 'CREATE TABLE blobs(' +
-      'id VARCHAR(128) NOT NULL UNIQUE,' +
-      'value MEDIUMBLOB,' +
-      'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin';
-
-    db.runSql(cmd, function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('DROP TABLE blobs', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20210503182308-blobs-migrate-secrets.js

@@ -1,49 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    fs = require('fs'),
-    safe = require('safetydance');
-
-const BOX_DATA_DIR = '/home/yellowtent/boxdata';
-const PLATFORM_DATA_DIR = '/home/yellowtent/platformdata';
-
-exports.up = function (db, callback) {
-    let funcs = [];
-
-    const acmeKey = safe.fs.readFileSync(`${BOX_DATA_DIR}/acme/acme.key`);
-    if (acmeKey) {
-        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'acme_account_key', acmeKey ]));
-        funcs.push(fs.rmdir.bind(fs, `${BOX_DATA_DIR}/acme`, { recursive: true }));
-    }
-    const dhparams = safe.fs.readFileSync(`${BOX_DATA_DIR}/dhparams.pem`);
-    if (dhparams) {
-        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/dhparams.pem`, dhparams);
-        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'dhparams', dhparams ]));
-        // leave the dhparms here for the moment because startup code regenerates box nginx config and reloads nginx. at that point,
-        // nginx config of apps has not been re-generated yet and the reload fails. post 6.3, this file can be removed in start.sh
-        // funcs.push(fs.unlink.bind(fs, `${BOX_DATA_DIR}/dhparams.pem`));
-    }
-    const turnSecret = safe.fs.readFileSync(`${BOX_DATA_DIR}/addon-turn-secret`);
-    if (turnSecret) {
-        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'addon_turn_secret', turnSecret ]));
-        funcs.push(fs.unlink.bind(fs, `${BOX_DATA_DIR}/addon-turn-secret`));
-    }
-
-    // sftp keys get moved to platformdata in start.sh
-    const sftpPublicKey = safe.fs.readFileSync(`${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key.pub`);
-    const sftpPrivateKey = safe.fs.readFileSync(`${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key`);
-    if (sftpPublicKey) {
-        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/sftp/ssh/ssh_host_rsa_key.pub`, sftpPublicKey);
-        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/sftp/ssh/ssh_host_rsa_key`, sftpPrivateKey);
-        safe.fs.chmodSync(`${PLATFORM_DATA_DIR}/sftp/ssh/ssh_host_rsa_key`, 0o600);
-        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'sftp_public_key', sftpPublicKey ]));
-        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'sftp_private_key', sftpPrivateKey ]));
-        funcs.push(fs.rmdir.bind(fs, `${BOX_DATA_DIR}/sftp`, { recursive: true }));
-    }
-
-    async.series(funcs, callback);
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20210504221634-settings-migrate-firewall.js

@@ -1,31 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    fs = require('fs'),
-    safe = require('safetydance');
-
-const BOX_DATA_DIR = '/home/yellowtent/boxdata';
-const PLATFORM_DATA_DIR = '/home/yellowtent/platformdata';
-
-exports.up = function (db, callback) {
-    if (!fs.existsSync(`${BOX_DATA_DIR}/firewall`)) return callback();
-
-    const ports = safe.fs.readFileSync(`${BOX_DATA_DIR}/firewall/ports.json`);
-    if (ports) {
-        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/firewall/ports.json`, ports);
-    }
-
-    const blocklist = safe.fs.readFileSync(`${BOX_DATA_DIR}/firewall/blocklist.txt`);
-    async.series([
-        (next) => {
-            if (!blocklist) return next();
-            db.runSql('INSERT INTO settings (name, valueBlob) VALUES (?, ?)', [ 'firewall_blocklist', blocklist ], next);
-        },
-        fs.writeFile.bind(fs, `${PLATFORM_DATA_DIR}/firewall/blocklist.txt`, blocklist || ''),
-        fs.rmdir.bind(fs, `${BOX_DATA_DIR}/firewall`, { recursive: true })
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20210504230054-domains-add-fallbackCertificateJson.js

@@ -1,38 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    safe = require('safetydance');
-
-const CERTS_DIR = '/home/yellowtent/boxdata/certs',
-    PLATFORM_CERTS_DIR = '/home/yellowtent/platformdata/nginx/cert';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE domains ADD COLUMN fallbackCertificateJson MEDIUMTEXT', function (error) {
-        if (error) return callback(error);
-
-        db.all('SELECT * FROM domains', [ ], function (error, domains) {
-            if (error) return callback(error);
-
-            async.eachSeries(domains, function (domain, iteratorDone) {
-                // b94dbf5fa33a6d68d784571721ff44348c2d88aa seems to have moved certs from platformdata to boxdata
-                let cert = safe.fs.readFileSync(`${CERTS_DIR}/${domain.domain}.host.cert`, 'utf8');
-                let key = safe.fs.readFileSync(`${CERTS_DIR}/${domain.domain}.host.key`, 'utf8');
-
-                if (!cert) {
-                    cert = safe.fs.readFileSync(`${PLATFORM_CERTS_DIR}/${domain.domain}.host.cert`, 'utf8');
-                    key = safe.fs.readFileSync(`${PLATFORM_CERTS_DIR}/${domain.domain}.host.key`, 'utf8');
-                }
-
-                const fallbackCertificate = { cert, key };
-
-                db.runSql('UPDATE domains SET fallbackCertificateJson=? WHERE domain=?', [ JSON.stringify(fallbackCertificate), domain.domain ], iteratorDone);
-            }, callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.run(db, 'ALTER TABLE domains DROP COLUMN fallbackCertificateJson')
-    ], callback);
-};

migrations/20210505165936-subdomains-add-certificateJson.js

@@ -1,34 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    fs = require('fs'),
-    safe = require('safetydance');
-
-const CERTS_DIR = '/home/yellowtent/boxdata/certs';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE subdomains ADD COLUMN certificateJson MEDIUMTEXT', function (error) {
-        if (error) return callback(error);
-
-        db.all('SELECT * FROM subdomains', [ ], function (error, subdomains) {
-            if (error) return callback(error);
-
-            async.eachSeries(subdomains, function (subdomain, iteratorDone) {
-                const cert = safe.fs.readFileSync(`${CERTS_DIR}/${subdomain.subdomain}.${subdomain.domain}.user.cert`, 'utf8');
-                const key = safe.fs.readFileSync(`${CERTS_DIR}/${subdomain.subdomain}.${subdomain.domain}.user.key`, 'utf8');
-
-                if (!cert || !key) return iteratorDone();
-
-                const certificate = { cert, key };
-
-                db.runSql('UPDATE subdomains SET certificateJson=? WHERE domain=? AND subdomain=?', [ JSON.stringify(certificate), subdomain.domain, subdomain.subdomain ], iteratorDone);
-            }, callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.run(db, 'ALTER TABLE subdomains DROP COLUMN certificateJson')
-    ], callback);
-};

migrations/20210505223829-blobs-migrate-certs.js

@@ -1,52 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    child_process = require('child_process'),
-    fs = require('fs'),
-    path = require('path'),
-    safe = require('safetydance');
-
-const OLD_CERTS_DIR = '/home/yellowtent/boxdata/certs';
-const NEW_CERTS_DIR = '/home/yellowtent/platformdata/nginx/cert';
-
-exports.up = function(db, callback) {
-    fs.readdir(OLD_CERTS_DIR, function (error, filenames) {
-        if (error && error.code === 'ENOENT') return callback();
-        if (error) return callback(error);
-
-        filenames = filenames.filter(f => f.endsWith('.key') && !f.endsWith('.host.key') && !f.endsWith('.user.key')); // ignore fallback and user keys
-
-        async.eachSeries(filenames, function (filename, iteratorCallback) {
-            const privateKeyFile = filename;
-            const privateKey = fs.readFileSync(path.join(OLD_CERTS_DIR, filename));
-            const certificateFile = filename.replace(/\.key$/, '.cert');
-            const certificate = safe.fs.readFileSync(path.join(OLD_CERTS_DIR, certificateFile));
-            if (!certificate) {
-                console.log(`${certificateFile} is missing. skipping migration`);
-                return iteratorCallback();
-            }
-            const csrFile = filename.replace(/\.key$/, '.csr');
-            const csr = safe.fs.readFileSync(path.join(OLD_CERTS_DIR, csrFile));
-            if (!csr) {
-                console.log(`${csrFile} is missing. skipping migration`);
-                return iteratorCallback();
-            }
-
-            async.series([
-                db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', `cert-${privateKeyFile}`, privateKey),
-                db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', `cert-${certificateFile}`, certificate),
-                db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', `cert-${csrFile}`, csr),
-            ], iteratorCallback);
-        }, function (error) {
-            if (error) return callback(error);
-
-            child_process.execSync(`cp ${OLD_CERTS_DIR}/* ${NEW_CERTS_DIR}`); // this way we copy the non-migrated ones like .host, .user etc as well
-            fs.rmdir(OLD_CERTS_DIR, { recursive: true }, callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
-

migrations/20210512210953-volumes-add-mountType-and-mountOptionsJson.js

@@ -1,17 +0,0 @@
-'use strict';
-
-const async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE volumes ADD COLUMN mountType VARCHAR(16) DEFAULT "noop"'),
-        db.runSql.bind(db, 'ALTER TABLE volumes ADD COLUMN mountOptionsJson TEXT')
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE volumes DROP COLUMN mountType'),
-        db.runSql.bind(db, 'ALTER TABLE volumes DROP COLUMN mountOptionsJson')
-    ], callback);
-};

migrations/20210517135636-add-order-by-indexes.js

@@ -1,21 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE backups ADD INDEX creationTime_index (creationTime)'),
-        db.runSql.bind(db, 'ALTER TABLE eventlog ADD INDEX creationTime_index (creationTime)'),
-        db.runSql.bind(db, 'ALTER TABLE notifications ADD INDEX creationTime_index (creationTime)'),
-        db.runSql.bind(db, 'ALTER TABLE tasks ADD INDEX creationTime_index (creationTime)'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE backups DROP INDEX creationTime_index'),
-        db.runSql.bind(db, 'ALTER TABLE eventlog DROP INDEX creationTime_index'),
-        db.runSql.bind(db, 'ALTER TABLE notifications DROP INDEX creationTime_index'),
-        db.runSql.bind(db, 'ALTER TABLE tasks DROP INDEX creationTime_index'),
-    ], callback);
-};

migrations/20210517141957-users-rename-createdAt-to-creationTime.js

@@ -1,33 +0,0 @@
-'use strict';
-
-const async = require('async');
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
-        if (error) return callback(error);
-
-        db.runSql('ALTER TABLE users ADD INDEX creationTime_index (creationTime)', function (error) {
-            if (error) return callback(error);
-
-            db.all('SELECT id, createdAt FROM users', function (error, results) {
-                if (error) return callback(error);
-
-                async.eachSeries(results, function (r, iteratorDone) {
-                    const creationTime = new Date(r.createdAt);
-                    db.runSql('UPDATE users SET creationTime=? WHERE id=?', [ creationTime, r.id ], iteratorDone);
-                }, function (error) {
-                    if (error) return callback(error);
-
-                    db.runSql('ALTER TABLE users DROP COLUMN createdAt', callback);
-                });
-            });
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN creationTime', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20210517194116-backups-provider-mountpoint.js

@@ -1,27 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        const backupConfig = JSON.parse(results[0].value);
-        if (backupConfig.provider === 'sshfs' || backupConfig.provider === 'cifs' || backupConfig.provider === 'nfs' || backupConfig.externalDisk) {
-            backupConfig.chown = backupConfig.provider === 'nfs' || backupConfig.provider === 'sshfs' || backupConfig.externalDisk;
-            backupConfig.preserveAttributes = !!backupConfig.externalDisk;
-            backupConfig.provider = 'mountpoint';
-            if (backupConfig.externalDisk) {
-                backupConfig.mountPoint = backupConfig.backupFolder;
-                backupConfig.prefix = '';
-                delete backupConfig.backupFolder;
-                delete backupConfig.externalDisk;
-            }
-            db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [JSON.stringify(backupConfig)], callback);
-        } else {
-            callback();
-        }
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20210528205138-notifications-drop-userId.js

@@ -1,13 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE notifications DROP COLUMN userId', function (error) {
-        if (error) return callback(error);
-
-        db.runSql('DELETE FROM notifications', callback); // just clear notifications table
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE notifications ADD COLUMN userId VARCHAR(128) NOT NULL', callback);
-};

migrations/20210625061443-volumes-change-noop-to-mountpoint.js

@@ -1,26 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    safe = require('safetydance');
-
-exports.up = function(db, callback) {
-    db.all('SELECT * FROM volumes', function (error, volumes) {
-        if (error || volumes.length === 0) return callback(error);
-
-        async.eachSeries(volumes, function (volume, iteratorDone) {
-            if (volume.mountType !== 'noop') return iteratorDone();
-
-            let mountType;
-            if (safe.child_process.execSync(`mountpoint -q -- ${volume.hostPath}`)) {
-                mountType = 'mountpoint';
-            } else {
-                mountType = 'filesystem';
-            }
-            db.runSql('UPDATE volumes SET mountType=? WHERE id=?', [ mountType, volume.id ], iteratorDone);
-        }, callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20210707154724-users-migrate-avatar.js

@@ -1,13 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('UPDATE users SET avatar="gravatar" WHERE avatar IS NULL', function (error) {
-        if (error) return callback(error);
-
-        db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB NOT NULL', callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB', callback);
-};

migrations/20210729191138-domains-update-wellKnown-matrix-client.js

@@ -1,30 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    safe = require('safetydance');
-
-exports.up = function(db, callback) {
-    db.all('SELECT * from domains', [], function (error, results) {
-        if (error) return callback(error);
-
-        async.eachSeries(results, function (r, iteratorDone) {
-            if (!r.wellKnownJson) return iteratorDone();
-
-            const wellKnown = safe.JSON.parse(r.wellKnownJson);
-            if (!wellKnown || !wellKnown['matrix/server']) return iteratorDone();
-            const matrixHostname = JSON.parse(wellKnown['matrix/server'])['m.server'];
-
-            wellKnown['matrix/client'] = JSON.stringify({
-                'm.homeserver': {
-                    'base_url': 'https://' + matrixHostname
-                }
-            });
-
-            db.runSql('UPDATE domains SET wellKnownJson=? WHERE domain=?', [ JSON.stringify(wellKnown), r.domain ], iteratorDone);
-        }, callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20210809203855-appAddonConfigs-alter-value.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE appAddonConfigs MODIFY value TEXT NOT NULL', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE appAddonConfigs MODIFY value VARCHAR(512)', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20210820172800-users-alter-loginLocationsJson.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users MODIFY loginLocationsJson MEDIUMTEXT', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users MODIFY loginLocationsJson TEXT', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20210921022843-apps-add-operatorsJson.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN operatorsJson TEXT', callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN operatorsJson', callback);
-};

migrations/20210927211658-apps-add-crontab.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN crontab TEXT', callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN crontab', callback);
-};

migrations/20211001095256-users-add-inviteToken.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN inviteToken VARCHAR(128) DEFAULT ""', callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN inviteToken', callback);
-};

migrations/20211001160903-apps-add-inbox.js

@@ -1,19 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN enableInbox BOOLEAN DEFAULT 0'),
-        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxName VARCHAR(128)'),
-        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxDomain VARCHAR(128)'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN enableInbox'),
-        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxName'),
-        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxDomain'),
-    ], callback);
-};

migrations/20211006200150-domains-ensure-fallbackCertificate.js

@@ -1,35 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    reverseProxy = require('../src/reverseproxy.js'),
-    safe = require('safetydance');
-
-const NGINX_CERT_DIR = '/home/yellowtent/platformdata/nginx/cert';
-
-// ensure fallbackCertificate of domains are present in database and the cert dir. it seems a bad migration lost them.
-// https://forum.cloudron.io/topic/5683/data-argument-must-be-of-type-received-null-error-during-restore-process
-exports.up = function(db, callback) {
-    db.all('SELECT * FROM domains', [ ], function (error, domains) {
-        if (error) return callback(error);
-
-        // this code is br0ken since async 3.x since async functions won't get iteratorDone anymore
-        // no point fixing this migration though since it won't run again in old cloudrons. and in new cloudron domains will be empty
-        async.eachSeries(domains, async function (domain, iteratorDone) {
-            let fallbackCertificate = safe.JSON.parse(domain.fallbackCertificateJson);
-            if (!fallbackCertificate || !fallbackCertificate.cert || !fallbackCertificate.key) {
-                let error;
-                [error, fallbackCertificate] = await safe(reverseProxy.generateFallbackCertificate(domain.domain));
-                if (error) return iteratorDone(error);
-            }
-
-            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.cert`, fallbackCertificate.cert, 'utf8')) return iteratorDone(safe.error);
-            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.key`, fallbackCertificate.key, 'utf8')) return iteratorDone(safe.error);
-
-            db.runSql('UPDATE domains SET fallbackCertificateJson=? WHERE domain=?', [ JSON.stringify(fallbackCertificate), domain.domain ], iteratorDone);
-        }, callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20211008171126-mailboxes-add-enablePop3.js

@@ -1,16 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE mailboxes ADD COLUMN enablePop3 BOOLEAN DEFAULT 0', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE mailboxes DROP COLUMN enablePop3', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20211012020255-mail-add-dkimKeyJson.js

@@ -1,44 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    fs = require('fs'),
-    path = require('path'),
-    safe = require('safetydance');
-
-const MAIL_DATA_DIR = '/home/yellowtent/boxdata/mail';
-const DKIM_DIR = `${MAIL_DATA_DIR}/dkim`;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE mail ADD COLUMN dkimKeyJson MEDIUMTEXT', function (error) {
-        if (error) return callback(error);
-
-        fs.readdir(DKIM_DIR, function (error, filenames) {
-            if (error && error.code === 'ENOENT') return callback();
-            if (error) return callback(error);
-
-            async.eachSeries(filenames, function (filename, iteratorCallback) {
-                const domain = filename;
-                const publicKey = safe.fs.readFileSync(path.join(DKIM_DIR, domain, 'public'), 'utf8');
-                const privateKey = safe.fs.readFileSync(path.join(DKIM_DIR, domain, 'private'), 'utf8');
-                if (!publicKey || !privateKey) return iteratorCallback();
-
-                const dkimKey = {
-                    publicKey,
-                    privateKey
-                };
-
-                db.runSql('UPDATE mail SET dkimKeyJson=? WHERE domain=?', [ JSON.stringify(dkimKey), domain ], iteratorCallback);
-            }, function (error) {
-                if (error) return callback(error);
-
-                fs.rmdir(DKIM_DIR, { recursive: true }, callback);
-            });
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.run(db, 'ALTER TABLE mail DROP COLUMN dkimKeyJson')
-    ], callback);
-};

migrations/20211117070204-blobs-delete-dhparams.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('DELETE FROM blobs WHERE id=?', [ 'dhparams' ], callback);
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20211117201805-eventlog-rename-source-to-sourceJson.js

@@ -1,17 +0,0 @@
-'use strict';
-
-const async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE source sourceJson TEXT', []),
-        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE data dataJson TEXT', []),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE sourceJson source TEXT', []),
-        db.runSql.bind(db, 'ALTER TABLE eventlog CHANGE dataJson data TEXT', []),
-    ], callback);
-};

migrations/20220107015249-settings-sysinfo-ip-to-ipv4.js

@@ -1,17 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('SELECT value FROM settings WHERE name=?', [ 'sysinfo_config' ], function (error, result) {
-        if (error || result.length === 0) return callback(error);
-        const sysinfoConfig = JSON.parse(result[0].value);
-        if (sysinfoConfig.provider !== 'fixed' || !sysinfoConfig.ip) return callback();
-        sysinfoConfig.ipv4 = sysinfoConfig.ip;
-        delete sysinfoConfig.ip;
-
-        db.runSql('REPLACE INTO settings (name, value) VALUES(?, ?)', [ 'sysinfo_config', JSON.stringify(sysinfoConfig) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20220113223425-settings-rename-directory-config-to-profile-config.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'directory_config', 'profile_config' ], callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'profile_config', 'directory_config' ], callback);
-};

migrations/20220115070646-subdomains-add-environmentVariable.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE subdomains ADD COLUMN environmentVariable VARCHAR(128)', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE subdomains DROP COLUMN environmentVariable', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20220202011148-blobs-migrate-token-secret.js

@@ -1,19 +0,0 @@
-'use strict';
-
-const safe = require('safetydance');
-
-const PROXY_AUTH_TOKEN_SECRET_FILE = '/home/yellowtent/platformdata/proxy-auth-token-secret';
-
-exports.up = function (db, callback) {
-    const token = safe.fs.readFileSync(PROXY_AUTH_TOKEN_SECRET_FILE);
-    if (!token) return callback();
-    db.runSql('INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'proxy_auth_token_secret', token ], function (error) {
-        if (error) return callback(error);
-        safe.fs.unlinkSync(PROXY_AUTH_TOKEN_SECRET_FILE);
-        callback();
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20220207214956-rename-subdomains-to-locations.js

@@ -1,12 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('RENAME TABLE subdomains TO locations', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20220219220751-mail-ensure-dkimKey.js

@@ -1,27 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    mail = require('../src/mail.js'),
-    safe = require('safetydance'),
-    util = require('util');
-
-// it seems some mail domains do not have dkimKey in the database for some reason because of some previous bad migration
-exports.up = function(db, callback) {
-    db.all('SELECT * FROM mail', [ ], function (error, mailDomains) {
-        if (error) return callback(error);
-
-        async.eachSeries(mailDomains, function (mailDomain, iteratorDone) {
-            let dkimKey = safe.JSON.parse(mailDomain.dkimKeyJson);
-            if (dkimKey && dkimKey.publicKey && dkimKey.privateKey) return iteratorDone();
-            console.log(`${mailDomain.domain} has no dkim key in the database. generating a new one`);
-            util.callbackify(mail.generateDkimKey)(function (error, dkimKey) {
-                if (error) return iteratorDone(error);
-                db.runSql('UPDATE mail SET dkimKeyJson=? WHERE domain=?', [ JSON.stringify(dkimKey), mailDomain.domain ], iteratorDone);
-            });
-        }, callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20220331193223-settings-delete-licenseKey.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('DELETE FROM settings WHERE name=?', [ 'license_key' ], callback);
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20220401044845-settings-rename-cloudron-token-to-appstore-api-token.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'appstore_api_token', 'cloudron_token' ], callback);
-};
-
-exports.down = function(db, callback) {
-  callback();
-};

migrations/20220401045439-settings-get-appstore-web-token.js

@@ -1,37 +0,0 @@
-'use strict';
-
-const superagent = require('superagent');
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="api_server_origin"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-        const apiServerOrigin = results[0].value;
-
-        db.all('SELECT value FROM settings WHERE name="appstore_api_token"', function (error, results) {
-            if (error || results.length === 0) return callback(error);
-            const apiToken = results[0].value;
-
-            console.log(`Getting appstore web token from ${apiServerOrigin}`);
-
-            superagent.post(`${apiServerOrigin}/api/v1/user_token`)
-                .send({})
-                .query({ accessToken: apiToken })
-                .timeout(30 * 1000).end(function (error, response) {
-                    if (error && !error.response) {
-                        console.log('Network error getting web token', error);
-                        return callback();
-                    }
-                    if (response.statusCode !== 201 || !response.body.accessToken) {
-                        console.log(`Bad status getting web token: ${response.status} ${response.text}`);
-                        return callback();
-                    }
-
-                    db.runSql('INSERT settings (name, value) VALUES(?, ?)', [ 'appstore_web_token', response.body.accessToken ], callback);
-                });
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20220402233643-backups-add-label.js

@@ -1,16 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE backups ADD COLUMN label VARCHAR(128) DEFAULT ""', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE backups DROP COLUMN label', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20220404211215-backups-rename-id-to-remotePath.js

@@ -1,51 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    hat = require('../src/hat.js');
-
-exports.up = function(db, callback) {
-    db.all('SELECT * from backups', function (error, allBackups) {
-        if (error) return callback(error);
-
-        console.log(`Fixing up ${allBackups.length} backup entries`);
-        const idMap = {};
-        allBackups.forEach(b => {
-            b.remotePath = b.id;
-            b.id = `${b.type}_${b.identifier}_v${b.packageVersion}_${hat(256)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying
-            idMap[b.remotePath] = b.id;
-        });
-
-        db.runSql('ALTER TABLE backups ADD COLUMN remotePath VARCHAR(256)', function (error) {
-            if (error) return callback(error);
-
-            db.runSql('ALTER TABLE backups CHANGE COLUMN dependsOn dependsOnJson TEXT', function (error) {
-                if (error) return callback(error);
-
-                async.eachSeries(allBackups, function (backup, iteratorDone) {
-                    const dependsOnPaths = backup.dependsOn ? backup.dependsOn.split(',') : []; // previously, it was paths
-                    let dependsOnIds = [];
-                    dependsOnPaths.forEach(p => { if (idMap[p]) dependsOnIds.push(idMap[p]); });
-
-                    db.runSql('UPDATE backups SET id = ?, remotePath = ?, dependsOnJson = ? WHERE id = ?', [ backup.id, backup.remotePath, JSON.stringify(dependsOnIds), backup.remotePath ], iteratorDone);
-                }, function (error) {
-                    if (error) return callback(error);
-
-                    db.runSql('ALTER TABLE backups MODIFY COLUMN remotePath VARCHAR(256) NOT NULL UNIQUE', callback);
-                });
-            });
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE backups DROP COLUMN remotePath', function (error) {
-        if (error) console.error(error);
-
-        db.runSql('ALTER TABLE backups RENAME COLUMN dependsOnJson to dependsOn', function (error) {
-            if (error) return callback(error);
-
-            callback(error);
-        });
-    });
-};
-

migrations/20220426060528-make-apps-sso-consistent.js

@@ -1,22 +0,0 @@
-'use strict';
-
-const async = require('async');
-
-exports.up = function(db, callback) {
-    db.all('SELECT * FROM apps', function (error, apps) {
-        if (error) return callback(error);
-
-        async.eachSeries(apps, function (app, iteratorDone) {
-            const manifest = JSON.parse(app.manifestJson);
-            const hasSso = !!manifest.addons['proxyAuth'] || !!manifest.addons['ldap'];
-            if (hasSso || !app.sso) return iteratorDone();
-
-            console.log(`Unsetting sso flag of ${app.id}`);
-            db.runSql('UPDATE apps SET sso=? WHERE id=?', [ 0, app.id ], iteratorDone);
-        }, callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20220505162417-settings-add-console-origin.js

@@ -1,20 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT * FROM settings WHERE name = ?', [ 'api_server_origin' ], function (error, result) {
-        if (error || result.length === 0) return callback(error);
-
-        let consoleOrigin;
-        switch (result[0].value) {
-        case 'https://api.dev.cloudron.io': consoleOrigin = 'https://console.dev.cloudron.io'; break;
-        case 'https://api.staging.cloudron.io': consoleOrigin = 'https://console.staging.cloudron.io'; break;
-        default: consoleOrigin = 'https://console.cloudron.io'; break;
-        }
-
-        db.runSql('REPLACE INTO settings (name, value) VALUES (?, ?)', [ 'console_server_origin', consoleOrigin ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20220514170234-users-add-backgroundImage.js

@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN backgroundImage MEDIUMBLOB', callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN backgroundImage', callback);
-};

migrations/20220601004757-apps-add-mailboxDisplayName.js

@@ -1,12 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN mailboxDisplayName VARCHAR(128) DEFAULT "" NOT NULL', [], callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN mailboxDisplayName', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20220602050517-apps-add-storageVolumeId.js

@@ -1,58 +0,0 @@
-'use strict';
-
-const path = require('path'),
-    safe = require('safetydance'),
-    uuid = require('uuid');
-
-function getMountPoint(dataDir) {
-    const output = safe.child_process.execSync(`df --output=target "${dataDir}" | tail -1`, { encoding: 'utf8' });
-    if (!output) return dataDir;
-    const mountPoint = output.trim();
-    if (mountPoint === '/') return dataDir;
-    return mountPoint;
-}
-
-exports.up = async function(db) {
-    // use safe() here because this migration failed midway in 7.2.4
-    await safe(db.runSql('ALTER TABLE apps ADD storageVolumeId VARCHAR(128), ADD FOREIGN KEY(storageVolumeId) REFERENCES volumes(id)'));
-    await safe(db.runSql('ALTER TABLE apps ADD storageVolumePrefix VARCHAR(128)'));
-    await safe(db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_storageVolume UNIQUE (storageVolumeId, storageVolumePrefix)'));
-
-    const apps = await db.runSql('SELECT * FROM apps WHERE dataDir IS NOT NULL');
-
-    for (const app of apps) {
-        if (app.storageVolumeId) {
-            console.log(`data-dir (${app.id}): app was migrated in 7.2.4`);
-            continue;
-        }
-
-        const allVolumes = await db.runSql('SELECT * FROM volumes');
-
-        console.log(`data-dir (${app.id}): migrating data dir ${app.dataDir}`);
-
-        const mountPoint = getMountPoint(app.dataDir);
-        const prefix = path.relative(mountPoint, app.dataDir);
-
-        console.log(`data-dir (${app.id}): migrating to mountpoint ${mountPoint} and prefix ${prefix}`);
-
-        const volume = allVolumes.find(v => v.hostPath === mountPoint);
-        if (volume) {
-            console.log(`data-dir (${app.id}): using existing volume ${volume.id}`);
-            await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ volume.id, prefix, app.id ]);
-            continue;
-        }
-
-        const id = uuid.v4().replace(/-/g, ''); // to make systemd mount file names more readable
-        const name = `appdata-${id}`;
-        const type = app.dataDir === mountPoint ? 'filesystem' : 'mountpoint';
-
-        console.log(`data-dir (${app.id}): creating new volume ${id}`);
-        await db.runSql('INSERT INTO volumes (id, name, hostPath, mountType, mountOptionsJson) VALUES (?, ?, ?, ?, ?)', [ id, name, mountPoint, type, JSON.stringify({}) ]);
-        await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ id, prefix, app.id ]);
-    }
-
-    await db.runSql('ALTER TABLE apps DROP COLUMN dataDir');
-};
-
-exports.down = async function(/*db*/) {
-};

migrations/schema.sql

@@ -6,7 +6,7 @@
 #### Strict mode is enabled
 #### VARCHAR - stored as part of table row (use for strings)
 #### TEXT - stored offline from table row (use for strings)
-#### BLOB (64KB), MEDIUMBLOB (16MB), LONGBLOB (4GB) - stored offline from table row (use for binary data)
+#### BLOB - stored offline from table row (use for binary data)
 #### https://dev.mysql.com/doc/refman/5.0/en/storage-requirements.html
 #### Times are stored in the database in UTC. And precision is seconds
 
@@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS users(
     email VARCHAR(254) NOT NULL UNIQUE,
     password VARCHAR(1024) NOT NULL,
     salt VARCHAR(512) NOT NULL,
-    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    createdAt VARCHAR(512) NOT NULL,
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     displayName VARCHAR(512) DEFAULT "",
     fallbackEmail VARCHAR(512) DEFAULT "",
@@ -28,15 +28,10 @@ CREATE TABLE IF NOT EXISTS users(
     twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
     source VARCHAR(128) DEFAULT "",
     role VARCHAR(32),
-    inviteToken VARCHAR(128) DEFAULT "",
     resetToken VARCHAR(128) DEFAULT "",
     resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     active BOOLEAN DEFAULT 1,
-    avatar MEDIUMBLOB NOT NULL,
-    backgroundImage MEDIUMBLOB,
-    loginLocationsJson MEDIUMTEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
 
-    INDEX creationTime_index (creationTime),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS userGroups(
@@ -60,7 +55,6 @@ CREATE TABLE IF NOT EXISTS tokens(
     clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
     expires BIGINT NOT NULL, // FIXME: make this a timestamp
-    lastUsedTime TIMESTAMP NULL,
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS apps(
@@ -84,29 +78,18 @@ CREATE TABLE IF NOT EXISTS apps(
     reverseProxyConfigJson TEXT, // { robotsTxt, csp }
     enableBackup BOOLEAN DEFAULT 1, // misnomer: controls automatic daily backups
     enableAutomaticUpdate BOOLEAN DEFAULT 1,
-    enableMailbox BOOLEAN DEFAULT 1, // whether sendmail addon is enabled
     mailboxName VARCHAR(128), // mailbox of this app
-    mailboxDomain VARCHAR(128), // mailbox domain of this app
-    mailboxDisplayName VARCHAR(128), // mailbox display name
-    enableInbox BOOLEAN DEFAULT 0, // whether recvmail addon is enabled
-    inboxName VARCHAR(128), // mailbox of this app
-    inboxDomain VARCHAR(128), // mailbox domain of this app
+    mailboxDomain VARCHAR(128), // mailbox domain of this apps
     label VARCHAR(128),     // display name
     tagsJson VARCHAR(2048), // array of tags
-    storageVolumeId VARCHAR(128),
-    storageVolumePrefix VARCHAR(128),
+    dataDir VARCHAR(256) UNIQUE,
     taskId INTEGER, // current task
     errorJson TEXT,
     servicesConfigJson TEXT, // app services configuration
     containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair'
-    appStoreIcon MEDIUMBLOB,
-    icon MEDIUMBLOB,
-    crontab TEXT,
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
-    FOREIGN KEY(storageVolumeId) REFERENCES volumes(id),
-    UNIQUE (storageVolumeId, storageVolumePrefix),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
@@ -120,14 +103,13 @@ CREATE TABLE IF NOT EXISTS appPortBindings(
 CREATE TABLE IF NOT EXISTS settings(
     name VARCHAR(128) NOT NULL UNIQUE,
     value TEXT,
-    valueBlob MEDIUMBLOB,
     PRIMARY KEY(name));
 
 CREATE TABLE IF NOT EXISTS appAddonConfigs(
     appId VARCHAR(128) NOT NULL,
     addonId VARCHAR(32) NOT NULL,
     name VARCHAR(128) NOT NULL,
-    value TEXT NOT NULL,
+    value VARCHAR(512) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id));
 
 CREATE TABLE IF NOT EXISTS appEnvVars(
@@ -138,30 +120,26 @@ CREATE TABLE IF NOT EXISTS appEnvVars(
 
 CREATE TABLE IF NOT EXISTS backups(
     id VARCHAR(128) NOT NULL,
-    remotePath VARCHAR(256) NOT NULL UNIQUE,
-    label VARCHAR(128) DEFAULT "",
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     packageVersion VARCHAR(128) NOT NULL, /* app version or box version */
     encryptionVersion INTEGER, /* when null, unencrypted backup */
     type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
     identifier VARCHAR(128) NOT NULL, /* 'box' or the app id */
-    dependsOnJson TEXT, /* comma separate list of objects this backup depends on */
+    dependsOn TEXT, /* comma separate list of objects this backup depends on */
     state VARCHAR(16) NOT NULL,
     manifestJson TEXT, /* to validate if the app can be installed in this version of box */
     format VARCHAR(16) DEFAULT "tgz",
     preserveSecs INTEGER DEFAULT 0,
 
-    INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
 CREATE TABLE IF NOT EXISTS eventlog(
     id VARCHAR(128) NOT NULL,
     action VARCHAR(128) NOT NULL,
-    sourceJson TEXT, /* { userId, username, ip }. userId can be null for cron,sysadmin */
-    dataJson TEXT, /* free flowing json based on action */
+    source TEXT, /* { userId, username, ip }. userId can be null for cron,sysadmin */
+    data TEXT, /* free flowing json based on action */
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
 
-    INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
 CREATE TABLE IF NOT EXISTS domains(
@@ -172,8 +150,6 @@ CREATE TABLE IF NOT EXISTS domains(
     tlsConfigJson TEXT, /* JSON containing the tls provider config */
     wellKnownJson TEXT, /* JSON containing well known docs for this domain */
 
-    fallbackCertificateJson MEDIUMTEXT,
-
     PRIMARY KEY (domain))
 
     /* the default db collation is utf8mb4_unicode_ci but for the app table domain constraint we have to use the old one */
@@ -188,7 +164,6 @@ CREATE TABLE IF NOT EXISTS mail(
     relayJson TEXT,
     bannerJson TEXT,
 
-    dkimKeyJson MEDIUMTEXT,
     dkimSelector VARCHAR(128) NOT NULL DEFAULT "cloudron",
 
     FOREIGN KEY(domain) REFERENCES domains(domain),
@@ -214,21 +189,16 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     membersOnly BOOLEAN DEFAULT false,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     domain VARCHAR(128),
-    active BOOLEAN DEFAULT 1,
-    enablePop3 BOOLEAN DEFAULT 0,
 
     FOREIGN KEY(domain) REFERENCES mail(domain),
     FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
     UNIQUE (name, domain));
 
-CREATE TABLE IF NOT EXISTS locations(
+CREATE TABLE IF NOT EXISTS subdomains(
     appId VARCHAR(128) NOT NULL,
     domain VARCHAR(128) NOT NULL,
     subdomain VARCHAR(128) NOT NULL,
-    type VARCHAR(128) NOT NULL, /* primary, secondary, redirect, alias */
-    environmentVariable VARCHAR(128), /* only set for secondary */
-
-    certificateJson MEDIUMTEXT,
+    type VARCHAR(128) NOT NULL, /* primary or redirect */
 
     FOREIGN KEY(domain) REFERENCES domains(domain),
     FOREIGN KEY(appId) REFERENCES apps(id),
@@ -237,27 +207,23 @@ CREATE TABLE IF NOT EXISTS locations(
 CREATE TABLE IF NOT EXISTS tasks(
     id int NOT NULL AUTO_INCREMENT,
     type VARCHAR(32) NOT NULL,
-    argsJson TEXT,
     percent INTEGER DEFAULT 0,
     message TEXT,
     errorJson TEXT,
     resultJson TEXT,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-
-    INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
 CREATE TABLE IF NOT EXISTS notifications(
     id int NOT NULL AUTO_INCREMENT,
+    userId VARCHAR(128) NOT NULL,
     eventId VARCHAR(128), // reference to eventlog. can be null
     title VARCHAR(512) NOT NULL,
     message TEXT,
     acknowledged BOOLEAN DEFAULT false,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-
-    INDEX creationTime_index (creationTime),
-    FOREIGN KEY(eventId) REFERENCES eventlog(id),
+    UNIQUE KEY appPasswords_name_appId_identifier (name, userId, identifier),
     PRIMARY KEY (id)
 );
 
@@ -268,7 +234,6 @@ CREATE TABLE IF NOT EXISTS appPasswords(
     identifier VARCHAR(128) NOT NULL, // resourceId: app id or mail or webadmin
     hashedPassword VARCHAR(1024) NOT NULL,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    UNIQUE KEY appPasswords_name_appId_identifier (name, userId, identifier)
     FOREIGN KEY(userId) REFERENCES users(id),
 
     PRIMARY KEY (id)
@@ -279,8 +244,6 @@ CREATE TABLE IF NOT EXISTS volumes(
     name VARCHAR(256) NOT NULL UNIQUE,
     hostPath VARCHAR(1024) NOT NULL UNIQUE,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    mountType VARCHAR(16) DEFAULT "noop",
-    mountOptionsJson TEXT,
     PRIMARY KEY (id)
 );
 
@@ -292,9 +255,4 @@ CREATE TABLE IF NOT EXISTS appMounts(
     FOREIGN KEY(appId) REFERENCES apps(id),
     FOREIGN KEY(volumeId) REFERENCES volumes(id));
 
-CREATE TABLE IF NOT EXISTS blobs(
-    id VARCHAR(128) NOT NULL UNIQUE,
-    value MEDIUMBLOB,
-    PRIMARY KEY(id));
-
 CHARACTER SET utf8 COLLATE utf8_bin;

package.json

@@ -11,67 +11,77 @@
     "url": "https://git.cloudron.io/cloudron/box.git"
   },
   "dependencies": {
-    "@google-cloud/dns": "^2.2.4",
-    "@google-cloud/storage": "^5.19.2",
+    "@google-cloud/dns": "^2.1.0",
+    "@google-cloud/storage": "^5.8.3",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^3.2.3",
-    "aws-sdk": "^2.1115.0",
+    "async": "^3.2.0",
+    "aws-sdk": "^2.879.0",
     "basic-auth": "^2.0.1",
-    "body-parser": "^1.20.0",
-    "cloudron-manifestformat": "^5.16.0",
+    "body-parser": "^1.19.0",
+    "cloudron-manifestformat": "^5.10.1",
     "connect": "^3.7.0",
-    "connect-lastmile": "^2.1.1",
+    "connect-lastmile": "^2.0.0",
     "connect-timeout": "^1.9.0",
-    "cookie-parser": "^1.4.6",
-    "cookie-session": "^2.0.0",
+    "cookie-parser": "^1.4.5",
+    "cookie-session": "^1.4.0",
     "cron": "^1.8.2",
-    "db-migrate": "^0.11.13",
-    "db-migrate-mysql": "^2.2.0",
-    "debug": "^4.3.4",
-    "dockerode": "^3.3.1",
+    "db-migrate": "^0.11.12",
+    "db-migrate-mysql": "^2.1.2",
+    "debug": "^4.3.1",
+    "dockerode": "^3.2.1",
     "ejs": "^3.1.6",
-    "ejs-cli": "^2.2.3",
-    "express": "^4.17.3",
-    "ipaddr.js": "^2.0.1",
-    "js-yaml": "^4.1.0",
-    "json": "^11.0.0",
+    "ejs-cli": "^2.2.1",
+    "express": "^4.17.1",
+    "ipaddr.js": "^2.0.0",
+    "js-yaml": "^4.0.0",
+    "json": "^10.0.0",
     "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.3.2",
+    "ldapjs": "^2.2.4",
     "lodash": "^4.17.21",
-    "moment": "^2.29.2",
-    "moment-timezone": "^0.5.34",
+    "lodash.chunk": "^4.2.0",
+    "mime": "^2.5.2",
+    "moment": "^2.29.1",
+    "moment-timezone": "^0.5.33",
     "morgan": "^1.10.0",
-    "multiparty": "^4.2.3",
+    "multiparty": "^4.2.2",
+    "mustache-express": "^1.3.0",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.7.3",
+    "nodemailer": "^6.5.0",
     "nodemailer-smtp-transport": "^2.7.4",
+    "once": "^1.4.0",
+    "pretty-bytes": "^5.6.0",
     "progress-stream": "^2.0.0",
-    "qrcode": "^1.5.0",
+    "proxy-middleware": "^0.15.0",
+    "qrcode": "^1.4.4",
     "readdirp": "^3.6.0",
-    "safetydance": "^2.2.0",
-    "semver": "^7.3.7",
+    "request": "^2.88.2",
+    "rimraf": "^3.0.2",
+    "s3-block-read-stream": "^0.5.0",
+    "safetydance": "^1.1.1",
+    "semver": "^7.3.5",
+    "showdown": "^1.9.1",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^7.1.1",
+    "superagent": "^6.1.0",
+    "supererror": "^0.7.2",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
-    "ua-parser-js": "^1.0.2",
-    "underscore": "^1.13.2",
+    "underscore": "^1.12.1",
     "uuid": "^8.3.2",
-    "validator": "^13.7.0",
-    "ws": "^8.5.0",
+    "validator": "^13.5.2",
+    "ws": "^7.4.4",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.4.1",
-    "js2xmlparser": "^4.0.2",
-    "mocha": "^9.2.2",
+    "js2xmlparser": "^4.0.1",
+    "mocha": "^8.3.2",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^13.2.4",
-    "node-sass": "^7.0.1",
-    "nyc": "^15.1.0"
+    "nock": "^13.0.11",
+    "node-sass": "^5.0.0",
+    "recursive-readdir": "^2.2.2"
   },
   "scripts": {
     "test": "./runTests",

runTests

@@ -22,8 +22,8 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/box boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
-mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update
+mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com boxdata/sftp/ssh
+mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
 sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
 
 # translations
@@ -34,18 +34,15 @@ cp -r ${source_dir}/../dashboard/dist/translation/* box/dashboard/dist/translati
 echo "=> Generating a localhost selfsigned cert"
 openssl req -x509 -newkey rsa:2048 -keyout platformdata/nginx/cert/host.key -out platformdata/nginx/cert/host.cert -days 3650 -subj '/CN=localhost' -nodes -config <(cat /etc/ssl/openssl.cnf <(printf "\n[SAN]\nsubjectAltName=DNS:*.localhost"))
 
-# clear out any containers if FAST is unset
-if [[ -z ${FAST+x} ]]; then
-    echo "=> Delete all docker containers first"
-    docker ps -qa --filter "label=isCloudronManaged" | xargs --no-run-if-empty docker rm -f
-    docker rm -f mysql-server
-    echo "==> To skip this run with: FAST=1 ./runTests"
-else
-    echo "==> WARNING!! Skipping docker container cleanup, the database might not be pristine!"
-fi
+# generate legacy key format for sftp
+ssh-keygen -m PEM -t rsa -f boxdata/sftp/ssh/ssh_host_rsa_key -q -N ""
+
+# clear out any containers
+echo "=> Delete all docker containers first"
+docker ps -qa | xargs --no-run-if-empty docker rm -f
 
 # create docker network (while the infra code does this, most tests skip infra setup)
-docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 --gateway 172.18.0.1 cloudron --ipv6 --subnet=fd00:c107:d509::/64 || true
+docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
 
 # create the same mysql server version to test with
 OUT=`docker inspect mysql-server` || true
@@ -63,12 +60,6 @@ while ! mysqladmin ping -h"${MYSQL_IP}" --silent; do
     sleep 1
 done
 
-echo "=> Ensure local base image"
-docker pull cloudron/base:3.0.0@sha256:455c70428723e3a823198c57472785437eb6eab082e79b3ff04ea584faf46e92
-
-echo "=> Create iptables blocklist"
-sudo ipset create cloudron_blocklist hash:net || true
-
 echo "=> Starting cloudron-syslog"
 cloudron-syslog --logdir "${DATA_DIR}/platformdata/logs/" &
 
@@ -79,15 +70,10 @@ echo "=> Run database migrations"
 cd "${source_dir}"
 BOX_ENV=test DATABASE_URL=mysql://root:password@${MYSQL_IP}/box node_modules/.bin/db-migrate up
 
+echo "=> Run tests with mocha"
 TESTS=${DEFAULT_TESTS}
 if [[ $# -gt 0 ]]; then
     TESTS="$*"
 fi
 
-if [[ -z ${COVERAGE+x} ]]; then
-    echo "=> Run tests with mocha"
-    BOX_ENV=test ./node_modules/.bin/mocha --bail --no-timeouts --exit -R spec ${TESTS}
-else
-    echo "=> Run tests with mocha and coverage"
-    BOX_ENV=test ./node_modules/.bin/nyc --reporter=html  ./node_modules/.bin/mocha --no-timeouts --exit -R spec ${TESTS}
-fi
+BOX_ENV=test ./node_modules/mocha/bin/_mocha --bail --no-timeouts --exit -R spec ${TESTS}

scripts/cloudron-setup

@@ -26,8 +26,8 @@ readonly GREEN='\033[32m'
 readonly DONE='\033[m'
 
 # verify the system has minimum requirements met
-if [[ "${rootfs_type}" != "ext4" && "${rootfs_type}" != "xfs" ]]; then
-    echo "Error: Cloudron requires '/' to be ext4 or xfs" # see #364
+if [[ "${rootfs_type}" != "ext4" ]]; then
+    echo "Error: Cloudron requires '/' to be ext4" # see #364
     exit 1
 fi
 
@@ -41,35 +41,22 @@ if [[ "${disk_size_gb}" -lt "${MINIMUM_DISK_SIZE_GB}" ]]; then
     exit 1
 fi
 
-if [[ "$(uname -m)" != "x86_64" ]]; then
-    echo "Error: Cloudron only supports amd64/x86_64"
-    exit 1
-fi
-
-if cvirt=$(systemd-detect-virt --container); then
-    echo "Error: Cloudron does not support ${cvirt}, only runs on bare metal or with full hardware virtualization"
-    exit 1
-fi
-
-# do not use is-active in case box service is down and user attempts to re-install
-if systemctl cat box.service >/dev/null 2>&1; then
+if systemctl -q is-active box; then
     echo "Error: Cloudron is already installed. To reinstall, start afresh"
     exit 1
 fi
 
+initBaseImage="true"
 provider="generic"
 requestedVersion=""
 installServerOrigin="https://api.cloudron.io"
 apiServerOrigin="https://api.cloudron.io"
 webServerOrigin="https://cloudron.io"
-consoleServerOrigin="https://console.cloudron.io"
 sourceTarballUrl=""
 rebootServer="true"
-setupToken="" # this is a OTP for securing an installation (https://forum.cloudron.io/topic/6389/add-password-for-initial-configuration)
-appstoreSetupToken=""
-redo="false"
+setupToken=""
 
-args=$(getopt -o "" -l "help,provider:,version:,env:,skip-reboot,generate-setup-token,setup-token:,redo" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot,generate-setup-token" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
@@ -81,20 +68,17 @@ while true; do
         if [[ "$2" == "dev" ]]; then
             apiServerOrigin="https://api.dev.cloudron.io"
             webServerOrigin="https://dev.cloudron.io"
-            consoleServerOrigin="https://console.dev.cloudron.io"
             installServerOrigin="https://api.dev.cloudron.io"
         elif [[ "$2" == "staging" ]]; then
             apiServerOrigin="https://api.staging.cloudron.io"
             webServerOrigin="https://staging.cloudron.io"
-            consoleServerOrigin="https://console.staging.cloudron.io"
             installServerOrigin="https://api.staging.cloudron.io"
         elif [[ "$2" == "unstable" ]]; then
             installServerOrigin="https://api.dev.cloudron.io"
         fi
         shift 2;;
+    --skip-baseimage-init) initBaseImage="false"; shift;;
     --skip-reboot) rebootServer="false"; shift;;
-    --redo) redo="true"; shift;;
-    --setup-token) appstoreSetupToken="$2"; shift 2;;
     --generate-setup-token) setupToken="$(openssl rand -hex 10)"; shift;;
     --) break;;
     *) echo "Unknown option $1"; exit 1;;
@@ -109,18 +93,11 @@ fi
 
 # Only --help works with mismatched ubuntu
 ubuntu_version=$(lsb_release -rs)
-if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubuntu_version}" != "20.04" && "${ubuntu_version}" != "22.04" ]]; then
-    echo "Cloudron requires Ubuntu 18.04, 20.04, 22.04" > /dev/stderr
+if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubuntu_version}" != "20.04" ]]; then
+    echo "Cloudron requires Ubuntu 16.04, 18.04 or 20.04" > /dev/stderr
     exit 1
 fi
 
-if which nginx >/dev/null || which docker >/dev/null || which node > /dev/null; then
-    if [[ "${redo}" == "false" ]]; then
-        echo "Error: Some packages like nginx/docker/nodejs are already installed. Cloudron requires specific versions of these packages and will install them as part of it's installation. Please start with a fresh Ubuntu install and run this script again." > /dev/stderr
-        exit 1
-    fi
-fi
-
 # Install MOTD file for stack script style installations. this is removed by the trap exit handler. Heredoc quotes prevents parameter expansion
 cat > /etc/update-motd.d/91-cloudron-install-in-progress <<'EOF'
 #!/bin/bash
@@ -155,15 +132,17 @@ echo ""
 echo " Join us at https://forum.cloudron.io for any questions."
 echo ""
 
-echo "=> Updating apt and installing script dependencies"
-if ! apt-get update &>> "${LOG_FILE}"; then
-    echo "Could not update package repositories. See ${LOG_FILE}"
-    exit 1
-fi
+if [[ "${initBaseImage}" == "true" ]]; then
+    echo "=> Updating apt and installing script dependencies"
+    if ! apt-get update &>> "${LOG_FILE}"; then
+        echo "Could not update package repositories. See ${LOG_FILE}"
+        exit 1
+    fi
 
-if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install --no-install-recommends curl python3 ubuntu-standard software-properties-common -y &>> "${LOG_FILE}"; then
-    echo "Could not install setup dependencies (curl). See ${LOG_FILE}"
-    exit 1
+    if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install --no-install-recommends curl python3 ubuntu-standard software-properties-common -y &>> "${LOG_FILE}"; then
+        echo "Could not install setup dependencies (curl). See ${LOG_FILE}"
+        exit 1
+    fi
 fi
 
 echo "=> Checking version"
@@ -183,7 +162,7 @@ if ! sourceTarballUrl=$(echo "${releaseJson}" | python3 -c 'import json,sys;obj=
     exit 1
 fi
 
-echo "=> Downloading Cloudron version ${version} ..."
+echo "=> Downloading version ${version} ..."
 box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)
 
 if ! $curl -sL "${sourceTarballUrl}" | tar -zxf - -C "${box_src_tmp_dir}"; then
@@ -191,16 +170,18 @@ if ! $curl -sL "${sourceTarballUrl}" | tar -zxf - -C "${box_src_tmp_dir}"; then
     exit 1
 fi
 
-echo -n "=> Installing base dependencies and downloading docker images (this takes some time) ..."
-init_ubuntu_script=$(test -f "${box_src_tmp_dir}/scripts/init-ubuntu.sh" && echo "${box_src_tmp_dir}/scripts/init-ubuntu.sh" || echo "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh")
-if ! /bin/bash "${init_ubuntu_script}" &>> "${LOG_FILE}"; then
-    echo "Init script failed. See ${LOG_FILE} for details"
-    exit 1
+if [[ "${initBaseImage}" == "true" ]]; then
+    echo -n "=> Installing base dependencies and downloading docker images (this takes some time) ..."
+    # initializeBaseUbuntuImage.sh args (provider, infraversion path) are only to support installation of pre 5.3 Cloudrons
+    if ! /bin/bash "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "generic" "../src" &>> "${LOG_FILE}"; then
+        echo "Init script failed. See ${LOG_FILE} for details"
+        exit 1
+    fi
+    echo ""
 fi
-echo ""
 
 # The provider flag is still used for marketplace images
-echo "=> Installing Cloudron version ${version} (this takes some time) ..."
+echo "=> Installing version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
 echo "${provider}" > /etc/cloudron/PROVIDER
 [[ ! -z "${setupToken}" ]] && echo "${setupToken}" > /etc/cloudron/SETUP_TOKEN
@@ -212,20 +193,6 @@ fi
 
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('api_server_origin', '${apiServerOrigin}');" 2>/dev/null
 mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('web_server_origin', '${webServerOrigin}');" 2>/dev/null
-mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('console_server_origin', '${consoleServerOrigin}');" 2>/dev/null
-
-if [[ -n "${appstoreSetupToken}" ]]; then
-    if ! setupResponse=$(curl -sX POST -H "Content-type: application/json" --data "{\"setupToken\": \"${appstoreSetupToken}\"}" "${apiServerOrigin}/api/v1/cloudron_setup_done"); then
-        echo "Could not complete setup. See ${LOG_FILE} for details"
-        exit 1
-    fi
-
-    cloudronId=$(echo "${setupResponse}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["cloudronId"])')
-    mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('cloudron_id', '${cloudronId}');" 2>/dev/null
-
-    appstoreApiToken=$(echo "${setupResponse}" | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["cloudronToken"])')
-    mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('appstore_api_token', '${appstoreApiToken}');" 2>/dev/null
-fi
 
 echo -n "=> Waiting for cloudron to be ready (this takes some time) ..."
 while true; do
@@ -236,7 +203,7 @@ while true; do
     sleep 10
 done
 
-if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://ipv4.api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
     ip='<IP>'
 fi
 if [[ -z "${setupToken}" ]]; then

scripts/cloudron-support

@@ -8,15 +8,14 @@ set -eu -o pipefail
 PASTEBIN="https://paste.cloudron.io"
 OUT="/tmp/cloudron-support.log"
 LINE="\n========================================================\n"
-CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGWS+930b8QdzbchGljt3KSljH9wRhYvht8srrtQHdzg support@cloudron.io"
+CLOUDRON_SUPPORT_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQVilclYAIu+ioDp/sgzzFz6YU0hPcRYY7ze/LiF/lC7uQqK062O54BFXTvQ3ehtFZCx3bNckjlT2e6gB8Qq07OM66De4/S/g+HJW4TReY2ppSPMVNag0TNGxDzVH8pPHOysAm33LqT2b6L/wEXwC6zWFXhOhHjcMqXvi8Ejaj20H1HVVcf/j8qs5Thkp9nAaFTgQTPu8pgwD8wDeYX1hc9d0PYGesTADvo6HF4hLEoEnefLw7PaStEbzk2fD3j7/g5r5HcgQQXBe74xYZ/1gWOX2pFNuRYOBSEIrNfJEjFJsqk3NR1+ZoMGK7j+AZBR4k0xbrmncQLcQzl6MMDzkp support@cloudron.io"
 HELP_MESSAGE="
-This script collects diagnostic information to help debug server related issues.
+This script collects diagnostic information to help debug server related issues
 
  Options:
-   --owner-login             Login as owner
-   --enable-ssh              Enable SSH access for the Cloudron support team
-   --reset-appstore-account  Reset associated cloudron.io account
-   --help                    Show this message
+   --owner-login   Login as owner
+   --enable-ssh    Enable SSH access for the Cloudron support team
+   --help          Show this message
 "
 
 # We require root
@@ -27,7 +26,7 @@ fi
 
 enableSSH="false"
 
-args=$(getopt -o "" -l "help,enable-ssh,admin-login,owner-login,reset-appstore-account" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,enable-ssh,admin-login,owner-login" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
@@ -38,20 +37,13 @@ while true; do
         # fall through
         ;&
     --owner-login)
-        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY creationTime LIMIT 1" 2>/dev/null)
+        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY createdAt LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
-        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/null)
-        mysql -NB -uroot -ppassword -e "INSERT INTO box.settings (name, value) VALUES ('ghosts_config', '{\"${admin_username}\":\"${admin_password}\"}') ON DUPLICATE KEY UPDATE name='ghosts_config', value='{\"${admin_username}\":\"${admin_password}\"}'" 2>/dev/null
-        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once."
-        exit 0
-        ;;
-    --reset-appstore-account)
-        echo -e "This will reset the Cloudron.io account associated with this Cloudron. Once reset, you can re-login with a different account in the Cloudron Dashboard. See https://docs.cloudron.io/appstore/#change-account for more information.\n"
-        read -e -p "Reset the Cloudron.io account? [y/N] " choice
-        [[ "$choice" != [Yy]* ]] && exit 1
-        mysql -uroot -ppassword -e "DELETE FROM box.settings WHERE name='cloudron_token';" 2>/dev/null
-        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/null)
-        echo "Account reset. Please re-login at https://${dashboard_domain}/#/appstore"
+        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/nul)
+        ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
+        printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > "${ghost_file}"
+        chown yellowtent:yellowtent "${ghost_file}" && chmod o-r,g-r "${ghost_file}"
+        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
         exit 0
         ;;
     --) break;;
@@ -77,31 +69,6 @@ if [[ "`df --output="avail" /tmp | sed -n 2p`" -lt "5120" ]]; then
     exit 1
 fi
 
-if [[ "${enableSSH}" == "true" ]]; then
-    ssh_port=$(cat /etc/ssh/sshd_config | grep "Port " | sed -e "s/.*Port //")
-
-    ssh_user="cloudron-support"
-    keys_file="/home/cloudron-support/.ssh/authorized_keys"
-
-    echo -e $LINE"SSH"$LINE >> $OUT
-    echo "Username: ${ssh_user}" >> $OUT
-    echo "Port:     ${ssh_port}" >> $OUT
-    echo "Key file: ${keys_file}" >> $OUT
-
-    echo -n "Enabling ssh access for the Cloudron support team..."
-    mkdir -p $(dirname "${keys_file}")       # .ssh does not exist sometimes
-    touch "${keys_file}"                # required for concat to work
-    if ! grep -q "${CLOUDRON_SUPPORT_PUBLIC_KEY}" "${keys_file}"; then
-        echo -e "\n${CLOUDRON_SUPPORT_PUBLIC_KEY}" >> "${keys_file}"
-        chmod 600 "${keys_file}"
-        chown "${ssh_user}" "${keys_file}"
-    fi
-
-    echo "Done"
-
-    exit 0
-fi
-
 echo -n "Generating Cloudron Support stats..."
 
 # clear file
@@ -144,8 +111,40 @@ iptables -L &>> $OUT
 
 echo "Done"
 
+if [[ "${enableSSH}" == "true" ]]; then
+    ssh_port=$(cat /etc/ssh/sshd_config | grep "Port " | sed -e "s/.*Port //")
+    permit_root_login=$(grep -q ^PermitRootLogin.*yes /etc/ssh/sshd_config && echo "yes" || echo "no")
+
+    # support.js uses similar logic
+    if [[ -d /home/ubuntu ]]; then
+        ssh_user="ubuntu"
+        keys_file="/home/ubuntu/.ssh/authorized_keys"
+    else
+        ssh_user="root"
+        keys_file="/root/.ssh/authorized_keys"
+    fi
+
+    echo -e $LINE"SSH"$LINE >> $OUT
+    echo "Username: ${ssh_user}" >> $OUT
+    echo "Port:     ${ssh_port}" >> $OUT
+    echo "PermitRootLogin: ${permit_root_login}" >> $OUT
+    echo "Key file: ${keys_file}" >> $OUT
+
+    echo -n "Enabling ssh access for the Cloudron support team..."
+    mkdir -p $(dirname "${keys_file}")       # .ssh does not exist sometimes
+    touch "${keys_file}"                # required for concat to work
+    if ! grep -q "${CLOUDRON_SUPPORT_PUBLIC_KEY}" "${keys_file}"; then
+        echo -e "\n${CLOUDRON_SUPPORT_PUBLIC_KEY}" >> "${keys_file}"
+        chmod 600 "${keys_file}"
+        chown "${ssh_user}" "${keys_file}"
+    fi
+
+    echo "Done"
+fi
+
 echo -n "Uploading information..."
-paste_key=$(curl -X POST ${PASTEBIN}/documents --silent --data-binary "@$OUT" | python3 -c "import sys, json; print(json.load(sys.stdin)['key'])")
+# for some reason not using $(cat $OUT) will not contain newlines!?
+paste_key=$(curl -X POST ${PASTEBIN}/documents --silent -d "$(cat $OUT)" | python3 -c "import sys, json; print(json.load(sys.stdin)['key'])")
 echo "Done"
 
 echo ""

scripts/createReleaseTarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v16.13.1" ]]; then
-    echo "This script requires node 16.13.1"
+if [[ "$(node --version)" != "v14.15.4" ]]; then
+    echo "This script requires node 14.15.4"
     exit 1
 fi
 

scripts/init-ubuntu.sh

@@ -1,198 +0,0 @@
-#!/bin/bash
-
-# This script is run on the base ubuntu. Put things here which are managed by ubuntu
-
-set -euv -o pipefail
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-readonly arg_infraversionpath="${SOURCE_DIR}/../src"
-
-function die {
-    echo $1
-    exit 1
-}
-
-export DEBIAN_FRONTEND=noninteractive
-
-readonly ubuntu_codename=$(lsb_release -cs)
-readonly ubuntu_version=$(lsb_release -rs)
-
-# hold grub since updating it breaks on some VPS providers. also, dist-upgrade will trigger it
-apt-mark hold grub* >/dev/null
-apt-get -o Dpkg::Options::="--force-confdef" update -y
-apt-get -o Dpkg::Options::="--force-confdef" upgrade -y
-apt-mark unhold grub* >/dev/null
-
-echo "==> Installing required packages"
-
-debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
-debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
-
-# this enables automatic security upgrades (https://help.ubuntu.com/community/AutomaticSecurityUpdates)
-# resolvconf is needed for unbound to work property after disabling systemd-resolved in 18.04
-case "${ubuntu_version}" in
-16.04)
-    gpg_package="gnupg"
-    mysql_package="mysql-server-5.7"
-    ntpd_package=""
-    python_package="python2.7"
-    nginx_package="" # we use custom package for TLS v1.3 support
-    ;;
-18.04)
-    gpg_package="gpg"
-    mysql_package="mysql-server-5.7"
-    ntpd_package=""
-    python_package="python2.7"
-    nginx_package="" # we use custom package for TLS v1.3 support
-    ;;
-20.04)
-    gpg_package="gpg"
-    mysql_package="mysql-server-8.0"
-    ntpd_package="systemd-timesyncd"
-    python_package="python3.8"
-    nginx_package="nginx-full"
-    ;;
-22.04)
-    gpg_package="gpg"
-    mysql_package="mysql-server-8.0"
-    ntpd_package="systemd-timesyncd"
-    python_package="python3.10"
-    nginx_package="nginx-full"
-    ;;
-esac
-
-apt-get -y install --no-install-recommends \
-    acl \
-    apparmor \
-    build-essential \
-    cifs-utils \
-    cron \
-    curl \
-    debconf-utils \
-    dmsetup \
-    $gpg_package \
-    ipset \
-    iptables \
-    lib${python_package} \
-    linux-generic \
-    logrotate \
-    $mysql_package \
-    nfs-common \
-    $nginx_package \
-    $ntpd_package \
-    openssh-server \
-    pwgen \
-    resolvconf \
-    sshfs \
-    swaks \
-    tzdata \
-    unattended-upgrades \
-    unbound \
-    unzip \
-    xfsprogs
-
-# on some providers like scaleway the sudo file is changed and we want to keep the old one
-apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends sudo
-
-# this ensures that unattended upgades are enabled, if it was disabled during ubuntu install time (see #346)
-# debconf-set-selection of unattended-upgrades/enable_auto_updates + dpkg-reconfigure does not work
-cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
-
-apt-get install -y --no-install-recommends $python_package   # Install python which is required for npm rebuild
-
-# do not upgrade grub because it might prompt user and break this script
-echo "==> Enable memory accounting"
-apt-get -y --no-upgrade --no-install-recommends install grub2-common
-sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
-update-grub
-
-echo "==> Install collectd"
-# without this, libnotify4 will install gnome-shell
-apt-get install -y libnotify4 libcurl3-gnutls --no-install-recommends
-# https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
-if [[ "${ubuntu_version}" == "22.04" ]]; then
-    readonly launchpad="https://launchpad.net/ubuntu/+source/collectd/5.12.0-9/+build/23189375/+files"
-    cd /tmp && wget -q "${launchpad}/collectd_5.12.0-9_amd64.deb" "${launchpad}/collectd-utils_5.12.0-9_amd64.deb" "${launchpad}/collectd-core_5.12.0-9_amd64.deb" "${launchpad}/libcollectdclient1_5.12.0-9_amd64.deb"
-    cd /tmp && apt install -y --no-install-recommends ./libcollectdclient1_5.12.0-9_amd64.deb ./collectd-core_5.12.0-9_amd64.deb ./collectd_5.12.0-9_amd64.deb ./collectd-utils_5.12.0-9_amd64.deb && rm -f /tmp/collectd_*.deb
-    echo -e "\nLD_PRELOAD=/usr/lib/python3.10/config-3.10-x86_64-linux-gnu/libpython3.10.so" >> /etc/default/collectd
-else
-    if ! apt-get install -y --no-install-recommends collectd collectd-utils; then
-        # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
-        echo "Failed to install collectd, continuing anyway. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
-        sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
-    fi
-
-    if [[ "${ubuntu_version}" == "20.04" ]]; then
-        echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
-    fi
-fi
-
-# some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
-echo "==> Configuring host"
-sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
-if systemctl is-active ntp; then
-    systemctl stop ntp
-    apt purge -y ntp
-fi
-timedatectl set-ntp 1
-# mysql follows the system timezone
-timedatectl set-timezone UTC
-
-echo "==> Adding sshd configuration warning"
-sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://docs.cloudron.io/security/#securing-ssh-access' -i /etc/ssh/sshd_config
-
-# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
-echo "==> Disabling motd news"
-if [[ -f "/etc/default/motd-news" ]]; then
-    sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
-fi
-
-# If privacy extensions are not disabled on server, this breaks IPv6 detection
-# https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756
-if [[ ! -f /etc/sysctl.d/99-cloudimg-ipv6.conf ]]; then
-    echo "==> Disable temporary address (IPv6)"
-    echo -e "# See https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756\nnet.ipv6.conf.all.use_tempaddr = 0\nnet.ipv6.conf.default.use_tempaddr = 0\n\n" > /etc/sysctl.d/99-cloudimg-ipv6.conf
-fi
-
-# Disable exim4 (1blu.de)
-systemctl stop exim4 || true
-systemctl disable exim4 || true
-
-# Disable bind for good measure (on online.net, kimsufi servers these are pre-installed)
-systemctl stop bind9 || true
-systemctl disable bind9 || true
-
-# on ovh images dnsmasq seems to run by default
-systemctl stop dnsmasq || true
-systemctl disable dnsmasq || true
-
-# on ssdnodes postfix seems to run by default
-systemctl stop postfix || true
-systemctl disable postfix || true
-
-# on ubuntu 18.04 and 20.04, this is the default. this requires resolvconf for DNS to work further after the disable
-systemctl stop systemd-resolved || true
-systemctl disable systemd-resolved || true
-
-# on vultr, ufw is enabled by default. we have our own firewall
-ufw disable || true
-
-# we need unbound to work as this is required for installer.sh to do any DNS requests
-echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: no" > /etc/unbound/unbound.conf.d/cloudron-network.conf
-systemctl restart unbound
-
-# Ubuntu 22 has private home directories by default (https://discourse.ubuntu.com/t/private-home-directories-for-ubuntu-21-04-onwards/)
-sed -e 's/^HOME_MODE\([[:space:]]\+\).*$/HOME_MODE\10755/' -i /etc/login.defs
-
-# create the yellowtent user. system user has different numeric range, no age and won't show in login/gdm UI
-# the nologin will also disable su/login
-if ! id yellowtent 2>/dev/null; then
-    useradd --system --comment "Cloudron Box" --create-home --shell /usr/sbin/nologin yellowtent
-fi
-
-# add support user (no password, sudo)
-if ! id cloudron-support 2>/dev/null; then
-    useradd --system --comment "Cloudron Support (support@cloudron.io)" --create-home --no-user-group --shell /bin/bash cloudron-support
-fi
-

scripts/installer.sh

@@ -15,48 +15,6 @@ function log() {
   echo -e "$(date +'%Y-%m-%dT%H:%M:%S')" "==> installer: $1"
 }
 
-apt_ready="no"
-function prepare_apt_once() {
-    [[ "${apt_ready}" == "yes" ]] && return
-
-    log "Making sure apt is in a good state"
-
-    log "Waiting for all dpkg tasks to finish..."
-    while fuser /var/lib/dpkg/lock; do
-        sleep 1
-    done
-
-    # it's unclear what needs to be run first or whether both these command should be run. so keep trying both
-    for count in {1..3}; do
-        # alternative to apt-install -y --fix-missing ?
-        if ! dpkg --force-confold --configure -a; then
-            log "dpkg reconfigure failed (try $count)"
-            dpkg_configure="no"
-        else
-            dpkg_configure="yes"
-        fi
-
-        if ! apt update -y; then
-            log "apt update failed (try $count)"
-            apt_update="no"
-        else
-            apt_update="yes"
-        fi
-
-        [[ "${dpkg_configure}" == "yes" && "${apt_update}" == "yes" ]] && break
-
-        sleep 1
-    done
-
-    apt_ready="yes"
-
-    if [[ "${dpkg_configure}" == "yes" && "${apt_update}" == "yes" ]]; then
-        log "apt is ready"
-    else
-        log "apt is not ready but proceeding anyway"
-    fi
-}
-
 readonly user=yellowtent
 readonly box_src_dir=/home/${user}/box
 
@@ -71,60 +29,59 @@ readonly is_update=$(systemctl is-active -q box && echo "yes" || echo "no")
 
 log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION)"
 
-# https://docs.docker.com/engine/installation/linux/ubuntulinux/
-readonly docker_version=20.10.14
-if ! which docker 2>/dev/null || [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
-    log "installing/updating docker"
-
-    # create systemd drop-in file already to make sure images are with correct driver
-    mkdir -p /etc/systemd/system/docker.service.d
-    echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
+log "updating docker"
 
+readonly docker_version=20.10.3
+if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
     # there are 3 packages for docker - containerd, CLI and the daemon
-    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.5.11-1_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
     $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
-    log "installing docker"
-    prepare_apt_once
-    apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
+    log "Waiting for all dpkg tasks to finish..."
+    while fuser /var/lib/dpkg/lock; do
+        sleep 1
+    done
+
+    while ! dpkg --force-confold --configure -a; do
+        log "Failed to fix packages. Retry"
+        sleep 1
+    done
+
+    # the latest docker might need newer packages
+    while ! apt update -y; do
+        log "Failed to update packages. Retry"
+        sleep 1
+    done
+
+    while ! apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb; do
+        log "Failed to install docker. Retry"
+        sleep 1
+    done
+
     rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
 fi
 
-# we want atleast nginx 1.14 for TLS v1.3 support. Ubuntu 20/22 already has nginx 1.18
-# Ubuntu 18 OpenSSL does not have TLS v1.3 support, so we use the upstream nginx packages
 readonly nginx_version=$(nginx -v 2>&1)
-if [[ "${ubuntu_version}" == "20.04" ]]; then
-    if [[ "${nginx_version}" == *"Ubuntu"* ]]; then
-        log "switching nginx to ubuntu package"
-        prepare_apt_once
-        apt remove -y nginx
-        apt install -y nginx-full
-    fi
-elif [[ "${ubuntu_version}" == "18.04" ]]; then
-    if [[ "${nginx_version}" != *"1.18."* ]]; then
-        log "installing/updating nginx 1.18"
-        $curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
-
-        prepare_apt_once
-
-        # apt install with install deps (as opposed to dpkg -i)
-        apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
-        rm /tmp/nginx.deb
-    fi
+if [[ "${nginx_version}" != *"1.18."* ]]; then
+    log "installing nginx 1.18"
+    $curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
+    # apt install with install deps (as opposed to dpkg -i)
+    apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
+    rm /tmp/nginx.deb
 fi
 
-readonly node_version=16.14.2
-if ! which node 2>/dev/null || [[ "$(node --version)" != "v${node_version}" ]]; then
-    log "installing/updating node ${node_version}"
+log "updating node"
+readonly node_version=14.15.4
+if [[ "$(node --version)" != "v${node_version}" ]]; then
     mkdir -p /usr/local/node-${node_version}
     $curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-${node_version}
     ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
     ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-16.13.1
+    rm -rf /usr/local/node-10.18.1
 fi
 
-# note that rebuild requires the above node
+# this is here (and not in updater.js) because rebuild requires the above node
 for try in `seq 1 10`; do
     # for reasons unknown, the dtrace package will fail. but rebuilding second time will work
 
@@ -142,7 +99,7 @@ if [[ ${try} -eq 10 ]]; then
 fi
 
 log "downloading new addon images"
-images=$(node -e "let i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
+images=$(node -e "var i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
 log "\tPulling docker images: ${images}"
 for image in ${images}; do
@@ -159,7 +116,7 @@ done
 log "update cloudron-syslog"
 CLOUDRON_SYSLOG_DIR=/usr/local/cloudron-syslog
 CLOUDRON_SYSLOG="${CLOUDRON_SYSLOG_DIR}/bin/cloudron-syslog"
-CLOUDRON_SYSLOG_VERSION="1.1.0"
+CLOUDRON_SYSLOG_VERSION="1.0.3"
 while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLOUDRON_SYSLOG_VERSION} ]]; do
     rm -rf "${CLOUDRON_SYSLOG_DIR}"
     mkdir -p "${CLOUDRON_SYSLOG_DIR}"
@@ -168,15 +125,10 @@ while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLO
     sleep 5
 done
 
-log "creating cloudron-support user"
-if ! id cloudron-support 2>/dev/null; then
-    useradd --system --comment "Cloudron Support (support@cloudron.io)" --create-home --no-user-group --shell /bin/bash cloudron-support
+if ! id "${user}" 2>/dev/null; then
+    useradd "${user}" -m
 fi
 
-log "locking the ${user} account"
-usermod --shell /usr/sbin/nologin "${user}"
-passwd --lock "${user}"
-
 if [[ "${is_update}" == "yes" ]]; then
     log "stop box service for update"
     ${box_src_dir}/setup/stop.sh

setup/start.sh

@@ -14,10 +14,9 @@ log "Cloudron Start"
 readonly USER="yellowtent"
 readonly HOME_DIR="/home/${USER}"
 readonly BOX_SRC_DIR="${HOME_DIR}/box"
-readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata"
-readonly APPS_DATA_DIR="${HOME_DIR}/appsdata"
-readonly BOX_DATA_DIR="${HOME_DIR}/boxdata/box"
-readonly MAIL_DATA_DIR="${HOME_DIR}/boxdata/mail"
+readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata" # platform data
+readonly APPS_DATA_DIR="${HOME_DIR}/appsdata" # app data
+readonly BOX_DATA_DIR="${HOME_DIR}/boxdata" # box data
 
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly json="$(realpath ${script_dir}/../node_modules/.bin/json)"
@@ -37,17 +36,11 @@ systemctl enable apparmor
 systemctl restart apparmor
 
 usermod ${USER} -a -G docker
-
-if ! grep -q ip6tables /etc/systemd/system/docker.service.d/cloudron.conf; then
-    log "Adding ip6tables flag to docker" # https://github.com/moby/moby/pull/41622
-    echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2 --experimental --ip6tables" > /etc/systemd/system/docker.service.d/cloudron.conf
-    systemctl daemon-reload
-    systemctl restart docker
-fi
+# unbound (which starts after box code) relies on this interface to exist. dockerproxy also relies on this.
+docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
 
 mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
-mkdir -p "${MAIL_DATA_DIR}"
 
 # keep these in sync with paths.js
 log "Ensuring directories"
@@ -57,8 +50,7 @@ mkdir -p "${PLATFORM_DATA_DIR}/mysql"
 mkdir -p "${PLATFORM_DATA_DIR}/postgresql"
 mkdir -p "${PLATFORM_DATA_DIR}/mongodb"
 mkdir -p "${PLATFORM_DATA_DIR}/redis"
-mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner" \
-         "${PLATFORM_DATA_DIR}/addons/mail/dkim"
+mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner"
 mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
 mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
 mkdir -p "${PLATFORM_DATA_DIR}/acme"
@@ -69,10 +61,15 @@ mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
          "${PLATFORM_DATA_DIR}/logs/crash" \
          "${PLATFORM_DATA_DIR}/logs/collectd"
 mkdir -p "${PLATFORM_DATA_DIR}/update"
-mkdir -p "${PLATFORM_DATA_DIR}/sftp/ssh" # sftp keys
-mkdir -p "${PLATFORM_DATA_DIR}/firewall"
-mkdir -p "${PLATFORM_DATA_DIR}/sshfs"
-mkdir -p "${PLATFORM_DATA_DIR}/cifs"
+
+mkdir -p "${BOX_DATA_DIR}/appicons"
+mkdir -p "${BOX_DATA_DIR}/firewall"
+mkdir -p "${BOX_DATA_DIR}/profileicons"
+mkdir -p "${BOX_DATA_DIR}/certs"
+mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
+mkdir -p "${BOX_DATA_DIR}/mail/dkim"
+mkdir -p "${BOX_DATA_DIR}/well-known" # .well-known documents
+mkdir -p "${BOX_DATA_DIR}/sftp/ssh" # sftp keys
 
 # ensure backups folder exists and is writeable
 mkdir -p /var/backups
@@ -88,19 +85,23 @@ sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
 sed -e "s/^WatchdogSec=.*$/WatchdogSec=3min/" \
     -i /lib/systemd/system/systemd-journald.service
 
-usermod -a -G systemd-journal ${USER} # Give user access to system logs
-if [[ ! -d /var/log/journal ]]; then # in some images, this directory is not created making system log to /run/systemd instead
-    mkdir -p /var/log/journal
-    chown root:systemd-journal /var/log/journal
-    chmod g+s /var/log/journal  # sticky bit for group propagation
-fi
+# Give user access to system logs
+usermod -a -G systemd-journal ${USER}
+mkdir -p /var/log/journal  # in some images, this directory is not created making system log to /run/systemd instead
+chown root:systemd-journal /var/log/journal
 systemctl daemon-reload
 systemctl restart systemd-journald
+setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
 
 # Give user access to nginx logs (uses adm group)
 usermod -a -G adm ${USER}
 
 log "Setting up unbound"
+# DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org)
+# We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
+# We listen on 0.0.0.0 because there is no way control ordering of docker (which creates the 172.18.0.0/16) and unbound
+# If IP6 is not enabled, dns queries seem to fail on some hosts. -s returns false if file missing or 0 size
+ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
 cp -f "${script_dir}/start/unbound.conf" /etc/unbound/unbound.conf.d/cloudron-network.conf
 # update the root anchor after a out-of-disk-space situation (see #269)
 unbound-anchor -a /var/lib/unbound/root.key
@@ -108,7 +109,6 @@ unbound-anchor -a /var/lib/unbound/root.key
 log "Adding systemd services"
 cp -r "${script_dir}/start/systemd/." /etc/systemd/system/
 [[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/MemoryMax/MemoryLimit/g' -i /etc/systemd/system/box.service
-[[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/Type=notify/Type=simple/g' -i /etc/systemd/system/unbound.service
 systemctl daemon-reload
 systemctl enable --now cloudron-syslog
 systemctl enable unbound
@@ -116,7 +116,7 @@ systemctl enable box
 systemctl enable cloudron-firewall
 systemctl enable --now cloudron-disable-thp
 
-# update firewall rules. this must be done after docker created it's rules
+# update firewall rules
 systemctl restart cloudron-firewall
 
 # For logrotate
@@ -129,28 +129,26 @@ systemctl restart unbound
 systemctl restart cloudron-syslog
 
 log "Configuring sudoers"
-rm -f /etc/sudoers.d/${USER} /etc/sudoers.d/cloudron
-cp "${script_dir}/start/sudoers" /etc/sudoers.d/cloudron
+rm -f /etc/sudoers.d/${USER}
+cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER}
 
 log "Configuring collectd"
 rm -rf /etc/collectd /var/log/collectd.log
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
-systemctl restart collectd
-
-log "Configuring sysctl"
-# If privacy extensions are not disabled on server, this breaks IPv6 detection
-# https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756
-if [[ ! -f /etc/sysctl.d/99-cloudimg-ipv6.conf ]]; then
-    echo "==> Disable temporary address (IPv6)"
-    echo -e "# See https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756\nnet.ipv6.conf.all.use_tempaddr = 0\nnet.ipv6.conf.default.use_tempaddr = 0\n\n" > /etc/sysctl.d/99-cloudimg-ipv6.conf
-    sysctl -p
+if [[ "${ubuntu_version}" == "20.04" ]]; then
+    # https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
+    if ! grep -q LD_PRELOAD /etc/default/collectd; then
+        echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
+    fi
 fi
+systemctl restart collectd
 
 log "Configuring logrotate"
 if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then
     echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf
 fi
+rm -f "${PLATFORM_DATA_DIR}/logrotate.d/"*
 cp "${script_dir}/start/logrotate/"* "${PLATFORM_DATA_DIR}/logrotate.d/"
 
 # logrotate files have to be owned by root, this is here to fixup existing installations where we were resetting the owner to yellowtent
@@ -174,7 +172,7 @@ fi
 
 # worker_rlimit_nofile in nginx config can be max this number
 mkdir -p /etc/systemd/system/nginx.service.d
-if ! grep -q "^LimitNOFILE=" /etc/systemd/system/nginx.service.d/cloudron.conf 2>/dev/null; then
+if ! grep -q "^LimitNOFILE=" /etc/systemd/system/nginx.service.d/cloudron.conf; then
     echo -e "[Service]\nLimitNOFILE=16384\n" > /etc/systemd/system/nginx.service.d/cloudron.conf
 fi
 
@@ -209,8 +207,7 @@ done
 
 readonly mysql_root_password="password"
 mysqladmin -u root -ppassword password password # reset default root password
-readonly mysqlVersion=$(mysql -NB -u root -p${mysql_root_password} -e 'SELECT VERSION()' 2>/dev/null)
-if [[ "${mysqlVersion}" == "8.0."* ]]; then
+if [[ "${ubuntu_version}" == "20.04" ]]; then
     # mysql 8 added a new caching_sha2_password scheme which mysqljs does not support
     mysql -u root -p${mysql_root_password} -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '${mysql_root_password}';"
 fi
@@ -227,18 +224,37 @@ fi
 
 rm -f /etc/cloudron/cloudron.conf
 
+if [[ ! -f "${BOX_DATA_DIR}/dhparams.pem" ]]; then
+    log "Generating dhparams (takes forever)"
+    openssl dhparam -out "${BOX_DATA_DIR}/dhparams.pem" 2048
+    cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem"
+else
+    cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem"
+fi
+
+if [[ ! -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" ]]; then
+    # the key format in Ubuntu 20 changed, so we create keys in legacy format. for older ubuntu, just re-use the host keys
+    # see https://github.com/proftpd/proftpd/issues/793
+    if [[ "${ubuntu_version}" == "20.04" ]]; then
+        ssh-keygen -m PEM -t rsa -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" -q -N ""
+    else
+        cp /etc/ssh/ssh_host_rsa_key* ${BOX_DATA_DIR}/sftp/ssh
+    fi
+fi
+
 log "Changing ownership"
-# note, change ownership after db migrate. this allow db migrate to move files around as root and then we can fix it up here
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
 chown -R "${USER}" /etc/cloudron
-chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall" "${PLATFORM_DATA_DIR}/sshfs" "${PLATFORM_DATA_DIR}/cifs"
+chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update"
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 chown "${USER}:${USER}" "${APPS_DATA_DIR}"
 
-chown "${USER}:${USER}" -R "${BOX_DATA_DIR}"
-# do not chown the boxdata/mail directory entirely; dovecot gets upset
-chown "${USER}:${USER}" "${MAIL_DATA_DIR}"
+# do not chown the boxdata/mail directory; dovecot gets upset
+chown "${USER}:${USER}" "${BOX_DATA_DIR}"
+find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${BOX_DATA_DIR}/mail" -exec chown -R "${USER}:${USER}" {} \;
+chown "${USER}:${USER}" "${BOX_DATA_DIR}/mail"
+chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys
 
 log "Starting Cloudron"
 systemctl start box

setup/start/cloudron-firewall.sh

@@ -3,154 +3,100 @@
 set -eu -o pipefail
 
 echo "==> Setting up firewall"
-
-has_ipv6=$(cat /proc/net/if_inet6 >/dev/null 2>&1 && echo "yes" || echo "no")
-
-# wait for 120 seconds for xtables lock, checking every 1 second
-readonly iptables="iptables --wait 120 --wait-interval 1"
-readonly ip6tables="ip6tables --wait 120 --wait-interval 1"
-
-function ipxtables() {
-    $iptables "$@"
-    [[ "${has_ipv6}" == "yes" ]] && $ip6tables "$@"
-}
-
-ipxtables -t filter -N CLOUDRON || true
-ipxtables -t filter -F CLOUDRON # empty any existing rules
+iptables -t filter -N CLOUDRON || true
+iptables -t filter -F CLOUDRON # empty any existing rules
 
 # first setup any user IP block lists
 ipset create cloudron_blocklist hash:net || true
-ipset create cloudron_blocklist6 hash:net family inet6 || true
-
 /home/yellowtent/box/src/scripts/setblocklist.sh
 
-$iptables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist src -j DROP
+iptables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist src -j DROP
 # the DOCKER-USER chain is not cleared on docker restart
-if ! $iptables -t filter -C DOCKER-USER -m set --match-set cloudron_blocklist src -j DROP; then
-    $iptables -t filter -I DOCKER-USER 1 -m set --match-set cloudron_blocklist src -j DROP
+if ! iptables -t filter -C DOCKER-USER -m set --match-set cloudron_blocklist src -j DROP; then
+    iptables -t filter -I DOCKER-USER 1 -m set --match-set cloudron_blocklist src -j DROP
 fi
 
-$ip6tables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist6 src -j DROP
-# there is no DOCKER-USER chain in ip6tables, bug?
-$ip6tables -D FORWARD -m set --match-set cloudron_blocklist6 src -j DROP || true
-$ip6tables -I FORWARD 1 -m set --match-set cloudron_blocklist6 src -j DROP
-
 # allow related and establisted connections
-ipxtables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
-ipxtables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,443 -j ACCEPT # 202 is the alternate ssh port
+iptables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
+iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,25,80,202,443 -j ACCEPT # 202 is the alternate ssh port
 
 # whitelist any user ports. we used to use --dports but it has a 15 port limit (XT_MULTI_PORTS)
-ports_json="/home/yellowtent/platformdata/firewall/ports.json"
-if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(' '))" 2>/dev/null); then
-    for p in $allowed_tcp_ports; do
-        ipxtables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
+ports_json="/home/yellowtent/boxdata/firewall/ports.json"
+if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(','))" 2>/dev/null); then
+    IFS=',' arr=(${allowed_tcp_ports})
+    for p in "${arr[@]}"; do
+        iptables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
     done
 fi
 
-if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(' '))" 2>/dev/null); then
-    for p in $allowed_udp_ports; do
-        ipxtables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
+if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(','))" 2>/dev/null); then
+    IFS=',' arr=(${allowed_udp_ports})
+    for p in "${arr[@]}"; do
+        iptables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
     done
 fi
 
-# LDAP user directory allow list
-ipset create cloudron_ldap_allowlist hash:net || true
-ipset flush cloudron_ldap_allowlist
-
-ipset create cloudron_ldap_allowlist6 hash:net family inet6 || true
-ipset flush cloudron_ldap_allowlist6
-
-ldap_allowlist_json="/home/yellowtent/platformdata/firewall/ldap_allowlist.txt"
-# delete any existing redirect rule
-$iptables -t nat -D PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004 2>/dev/null || true
-$ip6tables -t nat -D PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004 >/dev/null || true
-if [[ -f "${ldap_allowlist_json}" ]]; then
-    # without the -n block, any last line without a new line won't be read it!
-    while read -r line || [[ -n "$line" ]]; do
-        [[ -z "${line}" ]] && continue # ignore empty lines
-        [[ "$line" =~ ^#.*$ ]] && continue # ignore lines starting with #
-        if [[ "$line" == *":"* ]]; then
-            ipset add -! cloudron_ldap_allowlist6 "${line}" # the -! ignore duplicates
-        else
-            ipset add -! cloudron_ldap_allowlist "${line}" # the -! ignore duplicates
-        fi
-    done < "${ldap_allowlist_json}"
-
-    # ldap server we expose 3004 and also redirect from standard ldaps port 636
-    $iptables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
-    $iptables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist src -p tcp --dport 3004 -j ACCEPT
-
-    $ip6tables -t nat -I PREROUTING -p tcp --dport 636 -j REDIRECT --to-ports 3004
-    $ip6tables -t filter -A CLOUDRON -m set --match-set cloudron_ldap_allowlist6 src -p tcp --dport 3004 -j ACCEPT
-fi
-
 # turn and stun service
-ipxtables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
-ipxtables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
-ipxtables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
+iptables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
+iptables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
+iptables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
 
-# ICMPv6 is very fundamental to IPv6 connectivity unlike ICMPv4
-$iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
-$iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
-$ip6tables -t filter -A CLOUDRON -p ipv6-icmp -j ACCEPT
-
-ipxtables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT
-$iptables -t filter -A CLOUDRON -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
-ipxtables -t filter -A CLOUDRON -i lo -j ACCEPT # required for localhost connections (mysql)
+iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
+iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
+iptables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT
+iptables -t filter -A CLOUDRON -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+iptables -t filter -A CLOUDRON -i lo -j ACCEPT # required for localhost connections (mysql)
 
 # log dropped incoming. keep this at the end of all the rules
-ipxtables -t filter -A CLOUDRON -m limit --limit 2/min -j LOG --log-prefix "Packet dropped: " --log-level 7
-ipxtables -t filter -A CLOUDRON -j DROP
+iptables -t filter -A CLOUDRON -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
+iptables -t filter -A CLOUDRON -j DROP
 
-# prepend our chain to the filter table
-$iptables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $iptables -t filter -I INPUT -j CLOUDRON
-$ip6tables -t filter -C INPUT -j CLOUDRON 2>/dev/null || $ip6tables -t filter -I INPUT -j CLOUDRON
+if ! iptables -t filter -C INPUT -j CLOUDRON 2>/dev/null; then
+    iptables -t filter -I INPUT -j CLOUDRON
+fi
 
 # Setup rate limit chain (the recent info is at /proc/net/xt_recent)
-ipxtables -t filter -N CLOUDRON_RATELIMIT || true
-ipxtables -t filter -F CLOUDRON_RATELIMIT # empty any existing rules
+iptables -t filter -N CLOUDRON_RATELIMIT || true
+iptables -t filter -F CLOUDRON_RATELIMIT # empty any existing rules
 
 # log dropped incoming. keep this at the end of all the rules
-ipxtables -t filter -N CLOUDRON_RATELIMIT_LOG || true
-ipxtables -t filter -F CLOUDRON_RATELIMIT_LOG # empty any existing rules
-ipxtables -t filter -A CLOUDRON_RATELIMIT_LOG -m limit --limit 2/min -j LOG --log-prefix "IPTables RateLimit: " --log-level 7
-ipxtables -t filter -A CLOUDRON_RATELIMIT_LOG -j DROP
+iptables -t filter -N CLOUDRON_RATELIMIT_LOG || true
+iptables -t filter -F CLOUDRON_RATELIMIT_LOG # empty any existing rules
+iptables -t filter -A CLOUDRON_RATELIMIT_LOG -m limit --limit 2/min -j LOG --log-prefix "IPTables RateLimit: " --log-level 7
+iptables -t filter -A CLOUDRON_RATELIMIT_LOG -j DROP
 
 # http https
 for port in 80 443; do
-    ipxtables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
+    iptables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
-# ssh
+# ssh smtp ssh msa imap sieve
 for port in 22 202; do
-    ipxtables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
-    ipxtables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
-done
-
-# ldaps
-for port in 636 3004; do
-    ipxtables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
+    iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
+    iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
 done
 
 # docker translates (dnat) 25, 587, 993, 4190 in the PREROUTING step
 for port in 2525 4190 9993; do
-    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
+    iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
 done
 
-# msa, ldap, imap, sieve, pop3
-for port in 2525 3002 4190 9993 9995; do
-    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
+# msa, ldap, imap, sieve
+for port in 2525 3002 4190 9993; do
+    iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
 done
 
 # cloudron docker network: mysql postgresql redis mongodb
 for port in 3306 5432 6379 27017; do
-    $iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
+    iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
-# Add the rate limit chain to input chain
-$iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
-$ip6tables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null || $ip6tables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
+if ! iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null; then
+    iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
+fi
 
 # Workaround issue where Docker insists on adding itself first in FORWARD table
-ipxtables -D FORWARD -j CLOUDRON_RATELIMIT || true
-ipxtables -I FORWARD 1 -j CLOUDRON_RATELIMIT
+iptables -D FORWARD -j CLOUDRON_RATELIMIT || true
+iptables -I FORWARD 1 -j CLOUDRON_RATELIMIT
+
+echo "==> Setting up firewall done"

setup/start/cloudron-motd

@@ -4,17 +4,13 @@
 
 printf "**********************************************************************\n\n"
 
-cache_file="/var/cache/cloudron-motd-cache"
-
-if [[ -z "$(ls -A /home/yellowtent/platformdata/addons/mail/dkim)" ]]; then
-    if [[ ! -f "${cache_file}" ]]; then
-        curl --fail --connect-timeout 2 --max-time 2 -q https://ipv4.api.cloudron.io/api/v1/helper/public_ip --output "${cache_file}" || true
-    fi
-    if [[ -f "${cache_file}" ]]; then
-        ip=$(sed -n -e 's/.*"ip": "\(.*\)"/\1/p' /var/cache/cloudron-motd-cache)
-    else
+if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
+    if [[ -f /tmp/.cloudron-motd-cache ]]; then
+        ip=$(cat /tmp/.cloudron-motd-cache)
+    elif ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
         ip='<IP>'
     fi
+    echo "${ip}" > /tmp/.cloudron-motd-cache
 
     if [[ ! -f /etc/cloudron/SETUP_TOKEN ]]; then
         url="https://${ip}"

setup/start/collectd/collectd.conf

@@ -13,7 +13,7 @@
 ##############################################################################
 
 Hostname "localhost"
-FQDNLookup false
+#FQDNLookup true
 #BaseDir "/var/lib/collectd"
 #PluginDir "/usr/lib/collectd"
 #TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db"
@@ -232,7 +232,7 @@ LoadPlugin swap
 
 <Plugin write_graphite>
    <Node "graphing">
-       Host "127.0.0.1"
+       Host "localhost"
        Port "2003"
        Protocol "tcp"
        LogSendErrors true

setup/start/collectd/df.py

@@ -14,7 +14,7 @@ def read():
     for d in disks:
         device = d[0]
         if 'devicemapper' in d[1] or not device.startswith('/dev/'): continue
-        instance = device[len('/dev/'):].replace('/', '_').replace('.', '_') # see #348
+        instance = device[len('/dev/'):].replace('/', '_') # see #348
 
         try:
             st = os.statvfs(d[1]) # handle disk removal
@@ -34,5 +34,4 @@ def read():
         val.dispatch(values=[used], type_instance='used')
 
 collectd.register_init(init)
-# see Interval setting in collectd.conf for polling interval
 collectd.register_read(read)

setup/start/mysql.cnf

@@ -18,12 +18,6 @@ default_time_zone='+00:00'
 # disable bin logs. they are only useful in replication mode
 skip-log-bin
 
-# this is used when creating an index using ALTER command
-innodb_sort_buffer_size=2097152
-
-# this is a per session sort (ORDER BY) variable for non-indexed fields
-sort_buffer_size = 4M
-
 [mysqldump]
 quick
 quote-names

setup/start/nginx/nginx.conf

@@ -19,10 +19,15 @@ http {
     include       mime.types;
     default_type  application/octet-stream;
 
+    # the collectd config depends on this log format
+    log_format combined2 '$remote_addr - [$time_local] '
+        '"$request" $status $body_bytes_sent $request_time '
+        '"$http_referer" "$host" "$http_user_agent"';
+
     # required for long host names
     server_names_hash_bucket_size 128;
 
-    access_log /var/log/nginx/access.log combined;
+    access_log /var/log/nginx/access.log combined2;
 
     sendfile        on;
 

setup/start/sudoers

@@ -1,9 +1,6 @@
 # sudo logging breaks journalctl output with very long urls (systemd bug)
 Defaults !syslog
 
-Defaults!/home/yellowtent/box/src/scripts/checkvolume.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/checkvolume.sh
-
 Defaults!/home/yellowtent/box/src/scripts/clearvolume.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/clearvolume.sh
 
@@ -56,17 +53,3 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/stoptask.sh
 Defaults!/home/yellowtent/box/src/scripts/setblocklist.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setblocklist.sh
 
-Defaults!/home/yellowtent/box/src/scripts/setldapallowlist.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setldapallowlist.sh
-
-Defaults!/home/yellowtent/box/src/scripts/addmount.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/addmount.sh
-
-Defaults!/home/yellowtent/box/src/scripts/rmmount.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmount.sh
-
-Defaults!/home/yellowtent/box/src/scripts/remountmount.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/remountmount.sh
-
-cloudron-support ALL=(ALL) NOPASSWD: ALL
-

setup/start/systemd/unbound.service

@@ -2,17 +2,13 @@
 
 [Unit]
 Description=Unbound DNS Resolver
-After=network-online.target
-Before=nss-lookup.target
-Wants=network-online.target nss-lookup.target
+After=network.target docker.service
 
 [Service]
 PIDFile=/run/unbound.pid
 ExecStart=/usr/sbin/unbound -d
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
-# On ubuntu 16, this doesn't work for some reason
-Type=notify
 
 [Install]
 WantedBy=multi-user.target

setup/start/unbound.conf

@@ -1,11 +1,7 @@
-# Unbound is used primarily for RBL queries (host 2.0.0.127.zen.spamhaus.org)
-# We cannot use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
-
 server:
         port: 53
         interface: 127.0.0.1
         interface: 172.18.0.1
-        ip-freebind: yes
         do-ip6: no
         access-control: 127.0.0.1 allow
         access-control: 172.18.0.1/16 allow
@@ -14,3 +10,4 @@ server:
         # enable below for logging to journalctl -u unbound
         # verbosity: 5
         # log-queries: yes
+

src/accesscontrol.js

@@ -1,26 +1,29 @@
 'use strict';
 
 exports = module.exports = {
-    verifyToken
+    verifyToken: verifyToken
 };
 
-const assert = require('assert'),
+var assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    safe = require('safetydance'),
-    tokens = require('./tokens.js'),
+    tokendb = require('./tokendb.js'),
     users = require('./users.js');
 
-async function verifyToken(accessToken) {
+function verifyToken(accessToken, callback) {
     assert.strictEqual(typeof accessToken, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await tokens.getByAccessToken(accessToken);
-    if (!token) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'No such token');
+    tokendb.getByAccessToken(accessToken, function (error, token) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+        if (error) return callback(error);
 
-    const user = await users.get(token.identifier);
-    if (!user) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not found');
-    if (!user.active) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not active');
+        users.get(token.identifier, function (error, user) {
+            if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (error) return callback(error);
 
-    await safe(tokens.update(token.id, { lastUsedTime: new Date() })); // ignore any error
+            if (!user.active) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
 
-    return user;
+            callback(null, user);
+        });
+    });
 }

src/acme2.js

@@ -1,553 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    getCertificate,
-
-    // testing
-    _name: 'acme',
-    _getChallengeSubdomain: getChallengeSubdomain
-};
-
-const assert = require('assert'),
-    blobs = require('./blobs.js'),
-    BoxError = require('./boxerror.js'),
-    crypto = require('crypto'),
-    debug = require('debug')('box:cert/acme2'),
-    dns = require('./dns.js'),
-    fs = require('fs'),
-    os = require('os'),
-    path = require('path'),
-    promiseRetry = require('./promise-retry.js'),
-    superagent = require('superagent'),
-    safe = require('safetydance'),
-    _ = require('underscore');
-
-const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
-    CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory';
-
-// http://jose.readthedocs.org/en/latest/
-// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
-// https://community.letsencrypt.org/t/list-of-client-implementations/2103
-
-function Acme2(options) {
-    assert.strictEqual(typeof options, 'object');
-
-    this.accountKeyPem = null; // Buffer .
-    this.email = options.email;
-    this.keyId = null;
-    this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
-    this.directory = {};
-    this.performHttpAuthorization = !!options.performHttpAuthorization;
-    this.wildcard = !!options.wildcard;
-}
-
-// urlsafe base64 encoding (jose)
-function urlBase64Encode(string) {
-    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
-}
-
-function b64(str) {
-    const buf = Buffer.isBuffer(str) ? str : Buffer.from(str);
-    return urlBase64Encode(buf.toString('base64'));
-}
-
-function getModulus(pem) {
-    assert(Buffer.isBuffer(pem));
-
-    const stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
-    if (!stdout) return null;
-    const match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
-    if (!match) return null;
-    return Buffer.from(match[1], 'hex');
-}
-
-Acme2.prototype.sendSignedRequest = async function (url, payload) {
-    assert.strictEqual(typeof url, 'string');
-    assert.strictEqual(typeof payload, 'string');
-
-    assert(Buffer.isBuffer(this.accountKeyPem));
-
-    const that = this;
-    let header = {
-        url: url,
-        alg: 'RS256'
-    };
-
-    // keyId is null when registering account
-    if (this.keyId) {
-        header.kid = this.keyId;
-    } else {
-        header.jwk = {
-            e: b64(Buffer.from([0x01, 0x00, 0x01])), // exponent - 65537
-            kty: 'RSA',
-            n: b64(getModulus(this.accountKeyPem))
-        };
-    }
-
-    const payload64 = b64(payload);
-
-    let [error, response] = await safe(superagent.get(this.directory.newNonce).timeout(30000).ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`);
-    if (response.status !== 204) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching nonce : ${response.status}`);
-
-    const nonce = response.headers['Replay-Nonce'.toLowerCase()];
-    if (!nonce) throw new BoxError(BoxError.ACME_ERROR, 'No nonce in response');
-
-    debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
-
-    const protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
-
-    const signer = crypto.createSign('RSA-SHA256');
-    signer.update(protected64 + '.' + payload64, 'utf8');
-    const signature64 = urlBase64Encode(signer.sign(that.accountKeyPem, 'base64'));
-
-    const data = {
-        protected: protected64,
-        payload: payload64,
-        signature: signature64
-    };
-
-    [error, response] = await safe(superagent.post(url).send(data).set('Content-Type', 'application/jose+json').set('User-Agent', 'acme-cloudron').timeout(30000).ok(() => true));
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`);
-
-    return response;
-};
-
-// https://tools.ietf.org/html/rfc8555#section-6.3
-Acme2.prototype.postAsGet = async function (url) {
-    return await this.sendSignedRequest(url, '');
-};
-
-Acme2.prototype.updateContact = async function (registrationUri) {
-    assert.strictEqual(typeof registrationUri, 'string');
-
-    debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
-
-    // https://github.com/ietf-wg-acme/acme/issues/30
-    const payload = {
-        contact: [ 'mailto:' + this.email ]
-    };
-
-    const result = await this.sendSignedRequest(registrationUri, JSON.stringify(payload));
-    if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
-
-    debug(`updateContact: contact of user updated to ${this.email}`);
-};
-
-async function generateAccountKey() {
-    const acmeAccountKey = safe.child_process.execSync('openssl genrsa 4096');
-    if (!acmeAccountKey) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate acme account key: ${safe.error.message}`);
-    return acmeAccountKey;
-}
-
-Acme2.prototype.ensureAccount = async function () {
-    const payload = {
-        termsOfServiceAgreed: true
-    };
-
-    debug('ensureAccount: registering user');
-
-    this.accountKeyPem = await blobs.get(blobs.ACME_ACCOUNT_KEY);
-    if (!this.accountKeyPem) {
-        debug('ensureAccount: generating new account keys');
-        this.accountKeyPem = await generateAccountKey();
-        await blobs.set(blobs.ACME_ACCOUNT_KEY, this.accountKeyPem);
-    }
-
-    let result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
-    if (result.status === 403 && result.body.type === 'urn:ietf:params:acme:error:unauthorized') {
-        debug(`ensureAccount: key was revoked. ${result.status} ${JSON.stringify(result.body)}. generating new account key`);
-        this.accountKeyPem = await generateAccountKey();
-        await blobs.set(blobs.ACME_ACCOUNT_KEY, this.accountKeyPem);
-        result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
-    }
-
-    // 200 if already exists. 201 for new accounts
-    if (result.status !== 200 && result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${JSON.stringify(result.body)}`);
-
-    debug(`ensureAccount: user registered keyid: ${result.headers.location}`);
-
-    this.keyId = result.headers.location;
-
-    await this.updateContact(result.headers.location);
-};
-
-Acme2.prototype.newOrder = async function (domain) {
-    assert.strictEqual(typeof domain, 'string');
-
-    const payload = {
-        identifiers: [{
-            type: 'dns',
-            value: domain
-        }]
-    };
-
-    debug(`newOrder: ${domain}`);
-
-    const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload));
-    if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`);
-    if (result.status !== 201) throw new BoxError(BoxError.ACME_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`);
-
-    debug('newOrder: created order %s %j', domain, result.body);
-
-    const order = result.body, orderUrl = result.headers.location;
-
-    if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.ACME_ERROR, 'invalid authorizations in order');
-    if (typeof order.finalize !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid finalize in order');
-    if (typeof orderUrl !== 'string') throw new BoxError(BoxError.ACME_ERROR, 'invalid order location in order header');
-
-    return { order, orderUrl };
-};
-
-Acme2.prototype.waitForOrder = async function (orderUrl) {
-    assert.strictEqual(typeof orderUrl, 'string');
-
-    debug(`waitForOrder: ${orderUrl}`);
-
-    return await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
-        debug('waitForOrder: getting status');
-
-        const result = await this.postAsGet(orderUrl);
-        if (result.status !== 200) {
-            debug(`waitForOrder: invalid response code getting uri ${result.status}`);
-            throw new BoxError(BoxError.ACME_ERROR, `Bad response when waiting for order. code: ${result.status}`);
-        }
-
-        debug('waitForOrder: status is "%s %j', result.body.status, result.body);
-
-        if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.ACME_ERROR, `Request is in ${result.body.status} state`);
-        else if (result.body.status === 'valid' && result.body.certificate) return result.body.certificate;
-        else throw new BoxError(BoxError.ACME_ERROR, `Unexpected status or invalid response when waiting for order: ${JSON.stringify(result.body)}`);
-    });
-};
-
-Acme2.prototype.getKeyAuthorization = function (token) {
-    assert(Buffer.isBuffer(this.accountKeyPem));
-
-    let jwk = {
-        e: b64(Buffer.from([0x01, 0x00, 0x01])), // Exponent - 65537
-        kty: 'RSA',
-        n: b64(getModulus(this.accountKeyPem))
-    };
-
-    let shasum = crypto.createHash('sha256');
-    shasum.update(JSON.stringify(jwk));
-    let thumbprint = urlBase64Encode(shasum.digest('base64'));
-    return token + '.' + thumbprint;
-};
-
-Acme2.prototype.notifyChallengeReady = async function (challenge) {
-    assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token }
-
-    debug('notifyChallengeReady: %s was met', challenge.url);
-
-    const keyAuthorization = this.getKeyAuthorization(challenge.token);
-
-    const payload = {
-        resource: 'challenge',
-        keyAuthorization: keyAuthorization
-    };
-
-    const result = await this.sendSignedRequest(challenge.url, JSON.stringify(payload));
-    if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
-};
-
-Acme2.prototype.waitForChallenge = async function (challenge) {
-    assert.strictEqual(typeof challenge, 'object');
-
-    debug('waitingForChallenge: %j', challenge);
-
-    await promiseRetry({ times: 15, interval: 20000, debug }, async () => {
-        debug('waitingForChallenge: getting status');
-
-        const result = await this.postAsGet(challenge.url);
-        if (result.status !== 200) {
-            debug(`waitForChallenge: invalid response code getting uri ${result.status}`);
-            throw new BoxError(BoxError.ACME_ERROR, `Bad response code when waiting for challenge : ${result.status}`);
-        }
-
-        debug(`waitForChallenge: status is "${result.body.status}" "${JSON.stringify(result.body)}"`);
-
-        if (result.body.status === 'pending') throw new BoxError(BoxError.ACME_ERROR, 'Challenge is in pending state');
-        else if (result.body.status === 'valid') return;
-        else throw new BoxError(BoxError.ACME_ERROR, `Unexpected status when waiting for challenge: ${result.body.status}`);
-    });
-};
-
-// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
-Acme2.prototype.signCertificate = async function (domain, finalizationUrl, csrDer) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof finalizationUrl, 'string');
-    assert(Buffer.isBuffer(csrDer));
-
-    const payload = {
-        csr: b64(csrDer)
-    };
-
-    debug('signCertificate: sending sign request');
-
-    const result = await this.sendSignedRequest(finalizationUrl, JSON.stringify(payload));
-    // 429 means we reached the cert limit for this domain
-    if (result.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to sign certificate. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
-};
-
-Acme2.prototype.createKeyAndCsr = async function (hostname, keyFilePath, csrFilePath) {
-    assert.strictEqual(typeof hostname, 'string');
-
-    if (safe.fs.existsSync(keyFilePath)) {
-        debug('createKeyAndCsr: reuse the key for renewal at %s', keyFilePath);
-    } else {
-        let key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
-        if (!key) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
-        if (!safe.fs.writeFileSync(keyFilePath, key)) throw new BoxError(BoxError.FS_ERROR, safe.error);
-
-        debug('createKeyAndCsr: key file saved at %s', keyFilePath);
-    }
-
-    const [error, tmpdir] = await safe(fs.promises.mkdtemp(path.join(os.tmpdir(), 'acme-')));
-    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating temporary directory for openssl config: ${error.message}`);
-
-    // OCSP must-staple is currently disabled because nginx does not provide staple on the first request (https://forum.cloudron.io/topic/4917/ocsp-stapling-for-tls-ssl/)
-    // ' -addext "tlsfeature = status_request"'; // this adds OCSP must-staple
-    // we used to use -addext to the CLI to add these but that arg doesn't work on Ubuntu 16.04
-    // empty distinguished_name section is required for Ubuntu 16 openssl
-    const conf = '[req]\nreq_extensions = v3_req\ndistinguished_name = req_distinguished_name\n'
-        + '[req_distinguished_name]\n\n'
-        + '[v3_req]\nbasicConstraints = CA:FALSE\nkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\nsubjectAltName = @alt_names\n'
-        + `[alt_names]\nDNS.1 = ${hostname}\n`;
-
-    const opensslConfigFile = path.join(tmpdir, 'openssl.conf');
-    if (!safe.fs.writeFileSync(opensslConfigFile, conf)) throw new BoxError(BoxError.FS_ERROR, `Failed to write openssl config: ${safe.error.message}`);
-
-    // while we pass the CN anyways, subjectAltName takes precedence
-    const csrDer = safe.child_process.execSync(`openssl req -new -key ${keyFilePath} -outform DER -subj /CN=${hostname} -config ${opensslConfigFile}`);
-    if (!csrDer) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
-    if (!safe.fs.writeFileSync(csrFilePath, csrDer)) throw new BoxError(BoxError.FS_ERROR, safe.error); // bookkeeping. inspect with openssl req -text -noout -in hostname.csr -inform der
-
-    await safe(fs.promises.rm(tmpdir, { recursive: true, force: true }));
-
-    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFilePath);
-
-    return csrDer;
-};
-
-Acme2.prototype.downloadCertificate = async function (hostname, certUrl, certFilePath) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof certUrl, 'string');
-
-    await promiseRetry({ times: 5, interval: 20000, debug }, async () => {
-        debug(`downloadCertificate: downloading certificate of ${hostname}`);
-
-        const result = await this.postAsGet(certUrl);
-        if (result.statusCode === 202) throw new BoxError(BoxError.ACME_ERROR, 'Retry downloading certificate');
-        if (result.statusCode !== 200) throw new BoxError(BoxError.ACME_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
-
-        const fullChainPem = result.body; // buffer
-
-        if (!safe.fs.writeFileSync(certFilePath, fullChainPem)) throw new BoxError(BoxError.FS_ERROR, safe.error);
-
-        debug(`downloadCertificate: cert file for ${hostname} saved at ${certFilePath}`);
-    });
-};
-
-Acme2.prototype.prepareHttpChallenge = async function (hostname, domain, authorization, acmeChallengesDir) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof authorization, 'object');
-    assert.strictEqual(typeof acmeChallengesDir, 'string');
-
-    debug('prepareHttpChallenge: challenges: %j', authorization);
-    let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
-    if (httpChallenges.length === 0) throw new BoxError(BoxError.ACME_ERROR, 'no http challenges');
-    let challenge = httpChallenges[0];
-
-    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
-
-    let keyAuthorization = this.getKeyAuthorization(challenge.token);
-
-    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(acmeChallengesDir, challenge.token));
-
-    if (!safe.fs.writeFileSync(path.join(acmeChallengesDir, challenge.token), keyAuthorization)) throw new BoxError(BoxError.FS_ERROR, `Error writing challenge: ${safe.error.message}`);
-
-    return challenge;
-};
-
-Acme2.prototype.cleanupHttpChallenge = async function (hostname, domain, challenge, acmeChallengesDir) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof acmeChallengesDir, 'string');
-
-    debug('cleanupHttpChallenge: unlinking %s', path.join(acmeChallengesDir, challenge.token));
-
-    if (!safe.fs.unlinkSync(path.join(acmeChallengesDir, challenge.token))) throw new BoxError(BoxError.FS_ERROR, `Error unlinking challenge: ${safe.error.message}`);
-};
-
-function getChallengeSubdomain(hostname, domain) {
-    let challengeSubdomain;
-
-    if (hostname === domain) {
-        challengeSubdomain = '_acme-challenge';
-    } else if (hostname.includes('*')) { // wildcard
-        let subdomain = hostname.slice(0, -domain.length - 1);
-        challengeSubdomain = subdomain ? subdomain.replace('*', '_acme-challenge') : '_acme-challenge';
-    } else {
-        challengeSubdomain = '_acme-challenge.' + hostname.slice(0, -domain.length - 1);
-    }
-
-    debug(`getChallengeSubdomain: challenge subdomain for hostname ${hostname} at domain ${domain} is ${challengeSubdomain}`);
-
-    return challengeSubdomain;
-}
-
-Acme2.prototype.prepareDnsChallenge = async function (hostname, domain, authorization) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof authorization, 'object');
-
-    debug('prepareDnsChallenge: challenges: %j', authorization);
-    const dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
-    if (dnsChallenges.length === 0) throw new BoxError(BoxError.ACME_ERROR, 'no dns challenges');
-    const challenge = dnsChallenges[0];
-
-    const keyAuthorization = this.getKeyAuthorization(challenge.token);
-    const shasum = crypto.createHash('sha256');
-    shasum.update(keyAuthorization);
-
-    const txtValue = urlBase64Encode(shasum.digest('base64'));
-    const challengeSubdomain = getChallengeSubdomain(hostname, domain);
-
-    debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
-
-    await dns.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
-
-    await dns.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 });
-
-    return challenge;
-};
-
-Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challenge) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof challenge, 'object');
-
-    const keyAuthorization = this.getKeyAuthorization(challenge.token);
-    let shasum = crypto.createHash('sha256');
-    shasum.update(keyAuthorization);
-
-    const txtValue = urlBase64Encode(shasum.digest('base64'));
-    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
-
-    debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
-
-    await dns.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
-};
-
-Acme2.prototype.prepareChallenge = async function (hostname, domain, authorizationUrl, acmeChallengesDir) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof authorizationUrl, 'string');
-    assert.strictEqual(typeof acmeChallengesDir, 'string');
-
-    debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
-
-    const response = await this.postAsGet(authorizationUrl);
-    if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code getting authorization : ${response.status}`);
-
-    const authorization = response.body;
-
-    if (this.performHttpAuthorization) {
-        return await this.prepareHttpChallenge(hostname, domain, authorization, acmeChallengesDir);
-    } else {
-        return await this.prepareDnsChallenge(hostname, domain, authorization);
-    }
-};
-
-Acme2.prototype.cleanupChallenge = async function (hostname, domain, challenge, acmeChallengesDir) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof acmeChallengesDir, 'string');
-
-    debug(`cleanupChallenge: http: ${this.performHttpAuthorization}`);
-
-    if (this.performHttpAuthorization) {
-        await this.cleanupHttpChallenge(hostname, domain, challenge, acmeChallengesDir);
-    } else {
-        await this.cleanupDnsChallenge(hostname, domain, challenge);
-    }
-};
-
-Acme2.prototype.acmeFlow = async function (hostname, domain, paths) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof paths, 'object');
-
-    const { certFilePath, keyFilePath, csrFilePath, acmeChallengesDir } = paths;
-
-    await this.ensureAccount();
-    const { order, orderUrl } = await this.newOrder(hostname);
-
-    for (let i = 0; i < order.authorizations.length; i++) {
-        const authorizationUrl = order.authorizations[i];
-        debug(`acmeFlow: authorizing ${authorizationUrl}`);
-
-        const challenge = await this.prepareChallenge(hostname, domain, authorizationUrl, acmeChallengesDir);
-        await this.notifyChallengeReady(challenge);
-        await this.waitForChallenge(challenge);
-        const csrDer = await this.createKeyAndCsr(hostname, keyFilePath, csrFilePath);
-        await this.signCertificate(hostname, order.finalize, csrDer);
-        const certUrl = await this.waitForOrder(orderUrl);
-        await this.downloadCertificate(hostname, certUrl, certFilePath);
-
-        try {
-            await this.cleanupChallenge(hostname, domain, challenge, acmeChallengesDir);
-        } catch (cleanupError) {
-            debug('acmeFlow: ignoring error when cleaning up challenge:', cleanupError);
-        }
-    }
-};
-
-Acme2.prototype.loadDirectory = async function () {
-    await promiseRetry({ times: 3, interval: 20000, debug }, async () => {
-        const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true);
-
-        if (response.status !== 200) throw new BoxError(BoxError.ACME_ERROR, `Invalid response code when fetching directory : ${response.status}`);
-
-        if (typeof response.body.newNonce !== 'string' ||
-            typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') throw new BoxError(BoxError.ACME_ERROR, `Invalid response body : ${response.body}`);
-
-        this.directory = response.body;
-    });
-};
-
-Acme2.prototype.getCertificate = async function (vhost, domain, paths) {
-    assert.strictEqual(typeof vhost, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof paths, 'object');
-
-    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
-
-    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
-        vhost = dns.makeWildcard(vhost);
-        debug(`getCertificate: will get wildcard cert for ${vhost}`);
-    }
-
-    await this.loadDirectory();
-    await this.acmeFlow(vhost, domain, paths);
-};
-
-async function getCertificate(vhost, domain, paths, options) {
-    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof paths, 'object');
-    assert.strictEqual(typeof options, 'object');
-
-    await promiseRetry({ times: 3, interval: 0, debug }, async function () {
-        debug(`getCertificate: for vhost ${vhost} and domain ${domain}`);
-
-        const acme = new Acme2(options || { });
-        return await acme.getCertificate(vhost, domain, paths);
-    });
-}

src/addonconfigs.js

@@ -1,81 +0,0 @@
-'use strict';
-
-exports = module.exports =  {
-    get,
-    set,
-    unset,
-
-    getByAppId,
-    getByName,
-    unsetByAppId,
-    getAppIdByValue,
-};
-
-const assert = require('assert'),
-    database = require('./database.js');
-
-async function set(appId, addonId, env) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert(Array.isArray(env));
-
-    await unset(appId, addonId);
-    if (env.length === 0) return;
-
-    const query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
-    const args = [ ], queryArgs = [ ];
-    for (let i = 0; i < env.length; i++) {
-        args.push(appId, addonId, env[i].name, env[i].value);
-        queryArgs.push('(?, ?, ?, ?)');
-    }
-
-    await database.query(query + queryArgs.join(','), args);
-}
-
-async function unset(appId, addonId) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-
-    await database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
-}
-
-async function unsetByAppId(appId) {
-    assert.strictEqual(typeof appId, 'string');
-
-    await database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ]);
-}
-
-async function get(appId, addonId) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-
-    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
-    return results;
-}
-
-async function getByAppId(appId) {
-    assert.strictEqual(typeof appId, 'string');
-
-    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ]);
-    return results;
-}
-
-async function getAppIdByValue(addonId, namePattern, value) {
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-    assert.strictEqual(typeof value, 'string');
-
-    const results = await database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ]);
-    if (results.length === 0) return null;
-    return results[0].appId;
-}
-
-async function getByName(appId, addonId, namePattern) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-
-    const results = await database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ]);
-    if (results.length === 0) return null;
-    return results[0].value;
-}

src/appdb.js

@@ -0,0 +1,573 @@
+'use strict';
+
+exports = module.exports = {
+    get,
+    add,
+    exists,
+    del,
+    update,
+    getAll,
+    getPortBindings,
+    delPortBinding,
+
+    setAddonConfig,
+    getAddonConfig,
+    getAddonConfigByAppId,
+    getAddonConfigByName,
+    unsetAddonConfig,
+    unsetAddonConfigByAppId,
+    getAppIdByAddonConfigValue,
+    getByIpAddress,
+
+    setHealth,
+    setTask,
+    getAppStoreIds,
+
+    // subdomain table types
+    SUBDOMAIN_TYPE_PRIMARY: 'primary',
+    SUBDOMAIN_TYPE_REDIRECT: 'redirect',
+    SUBDOMAIN_TYPE_ALIAS: 'alias',
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    safe = require('safetydance'),
+    util = require('util');
+
+const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
+    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
+    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson',
+    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp',
+    'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
+    'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(',');
+
+const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
+    result.manifest = safe.JSON.parse(result.manifestJson);
+    delete result.manifestJson;
+
+    assert(result.tagsJson === null || typeof result.tagsJson === 'string');
+    result.tags = safe.JSON.parse(result.tagsJson) || [];
+    delete result.tagsJson;
+
+    assert(result.reverseProxyConfigJson === null || typeof result.reverseProxyConfigJson === 'string');
+    result.reverseProxyConfig = safe.JSON.parse(result.reverseProxyConfigJson) || {};
+    delete result.reverseProxyConfigJson;
+
+    assert(result.hostPorts === null || typeof result.hostPorts === 'string');
+    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');
+
+    result.portBindings = { };
+    let hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
+    let environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
+    let portTypes = result.portTypes === null ? [ ] : result.portTypes.split(',');
+
+    delete result.hostPorts;
+    delete result.environmentVariables;
+    delete result.portTypes;
+
+    for (let i = 0; i < environmentVariables.length; i++) {
+        result.portBindings[environmentVariables[i]] = { hostPort: parseInt(hostPorts[i], 10), type: portTypes[i] };
+    }
+
+    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
+    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
+    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
+    delete result.accessRestrictionJson;
+
+    result.sso = !!result.sso; // make it bool
+    result.enableBackup = !!result.enableBackup; // make it bool
+    result.enableAutomaticUpdate = !!result.enableAutomaticUpdate; // make it bool
+    result.proxyAuth = !!result.proxyAuth;
+
+    assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
+    result.debugMode = safe.JSON.parse(result.debugModeJson);
+    delete result.debugModeJson;
+
+    assert(result.servicesConfigJson === null || typeof result.servicesConfigJson === 'string');
+    result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
+    delete result.servicesConfigJson;
+
+    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
+    delete result.subdomains;
+    delete result.domains;
+    delete result.subdomainTypes;
+
+    result.alternateDomains = [];
+    result.aliasDomains = [];
+    for (let i = 0; i < subdomainTypes.length; i++) {
+        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
+            result.location = subdomains[i];
+            result.domain = domains[i];
+        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
+            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
+        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
+            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
+        }
+    }
+
+    let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
+    delete result.envNames;
+    delete result.envValues;
+    result.env = {};
+    for (let i = 0; i < envNames.length; i++) { // NOTE: envNames is [ null ] when env of an app is empty
+        if (envNames[i]) result.env[envNames[i]] = envValues[i];
+    }
+
+    let volumeIds = JSON.parse(result.volumeIds);
+    delete result.volumeIds;
+    let volumeReadOnlys = JSON.parse(result.volumeReadOnlys);
+    delete result.volumeReadOnlys;
+
+    result.mounts = volumeIds[0] === null ? [] : volumeIds.map((v, idx) => { return { volumeId: v, readOnly: !!volumeReadOnlys[idx] }; }); // NOTE: volumeIds is [null] when volumes of an app is empty
+
+    result.error = safe.JSON.parse(result.errorJson);
+    delete result.errorJson;
+
+    result.taskId = result.taskId ? String(result.taskId) : null;
+}
+
+// each query simply join apps table with another table by id. we then join the full result together
+const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
+const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
+const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
+const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
+const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
+    + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
+    + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
+    + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
+    + ` LEFT JOIN (${MOUNTS_QUERY}) AS q4 on q4.id = apps.id`;
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`${APPS_QUERY} WHERE apps.id = ?`, [ id ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function getByIpAddress(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`${APPS_QUERY} WHERE apps.containerIp = ?`, [ ip ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function add(id, appStoreId, manifest, location, domain, portBindings, data, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof appStoreId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof manifest.version, 'string');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    portBindings = portBindings || { };
+
+    var manifestJson = JSON.stringify(manifest);
+
+    const accessRestriction = data.accessRestriction || null;
+    const accessRestrictionJson = JSON.stringify(accessRestriction);
+    const memoryLimit = data.memoryLimit || 0;
+    const cpuShares = data.cpuShares || 512;
+    const installationState = data.installationState;
+    const runState = data.runState;
+    const sso = 'sso' in data ? data.sso : null;
+    const debugModeJson = data.debugMode ? JSON.stringify(data.debugMode) : null;
+    const env = data.env || {};
+    const label = data.label || null;
+    const tagsJson = data.tags ? JSON.stringify(data.tags) : null;
+    const mailboxName = data.mailboxName || null;
+    const mailboxDomain = data.mailboxDomain || null;
+    const reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null;
+
+    var queries = [];
+
+    queries.push({
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
+            + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson) '
+            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
+            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson ]
+    });
+
+    queries.push({
+        query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+        args: [ id, domain, location, exports.SUBDOMAIN_TYPE_PRIMARY ]
+    });
+
+    Object.keys(portBindings).forEach(function (env) {
+        queries.push({
+            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, type, appId) VALUES (?, ?, ?, ?)',
+            args: [ env, portBindings[env].hostPort, portBindings[env].type, id ]
+        });
+    });
+
+    Object.keys(env).forEach(function (name) {
+        queries.push({
+            query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
+            args: [ id, name, env[name] ]
+        });
+    });
+
+    if (data.alternateDomains) {
+        data.alternateDomains.forEach(function (d) {
+            queries.push({
+                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]
+            });
+        });
+    }
+
+    if (data.aliasDomains) {
+        data.aliasDomains.forEach(function (d) {
+            queries.push({
+                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
+            });
+        });
+    }
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'no such domain'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function exists(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        return callback(null, result.length !== 0);
+    });
+}
+
+function getPortBindings(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        var portBindings = { };
+        for (var i = 0; i < results.length; i++) {
+            portBindings[results[i].environmentVariable] = { hostPort: results[i].hostPort, type: results[i].type };
+        }
+
+        callback(null, portBindings);
+    });
+}
+
+function delPortBinding(hostPort, type, callback) {
+    assert.strictEqual(typeof hostPort, 'number');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appPortBindings WHERE hostPort=? AND type=?', [ hostPort, type ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [
+        { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
+        { query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
+    ];
+
+    database.transaction(queries, function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results[5].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        database.query.bind(null, 'DELETE FROM subdomains'),
+        database.query.bind(null, 'DELETE FROM appPortBindings'),
+        database.query.bind(null, 'DELETE FROM appAddonConfigs'),
+        database.query.bind(null, 'DELETE FROM appEnvVars'),
+        database.query.bind(null, 'DELETE FROM apps')
+    ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        return callback(null);
+    });
+}
+
+function update(id, app, callback) {
+    updateWithConstraints(id, app, '', callback);
+}
+
+function updateWithConstraints(id, app, constraints, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof constraints, 'string');
+    assert.strictEqual(typeof callback, 'function');
+    assert(!('portBindings' in app) || typeof app.portBindings === 'object');
+    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
+    assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
+    assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
+    assert(!('tags' in app) || Array.isArray(app.tags));
+    assert(!('env' in app) || typeof app.env === 'object');
+
+    var queries = [ ];
+
+    if ('portBindings' in app) {
+        var portBindings = app.portBindings || { };
+        // replace entries by app id
+        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
+        Object.keys(portBindings).forEach(function (env) {
+            var values = [ portBindings[env].hostPort, portBindings[env].type, env, id ];
+            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, type, environmentVariable, appId) VALUES(?, ?, ?, ?)', args: values });
+        });
+    }
+
+    if ('env' in app) {
+        queries.push({ query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] });
+
+        Object.keys(app.env).forEach(function (name) {
+            queries.push({
+                query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
+                args: [ id, name, app.env[name] ]
+            });
+        });
+    }
+
+    if ('location' in app && 'domain' in app) { // must be updated together as they are unique together
+        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ]}); // all locations of an app must be updated together
+        queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, app.domain, app.location, exports.SUBDOMAIN_TYPE_PRIMARY ]});
+
+        if ('alternateDomains' in app) {
+            app.alternateDomains.forEach(function (d) {
+                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
+            });
+        }
+
+        if ('aliasDomains' in app) {
+            app.aliasDomains.forEach(function (d) {
+                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
+            });
+        }
+    }
+
+    if ('mounts' in app) {
+        queries.push({ query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ]});
+        app.mounts.forEach(function (m) {
+            queries.push({ query: 'INSERT INTO appMounts (appId, volumeId, readOnly) VALUES (?, ?, ?)', args: [ id, m.volumeId, m.readOnly ]});
+        });
+    }
+
+    var fields = [ ], values = [ ];
+    for (var p in app) {
+        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig') {
+            fields.push(`${p}Json = ?`);
+            values.push(JSON.stringify(app[p]));
+        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
+            fields.push(p + ' = ?');
+            values.push(app[p]);
+        }
+    }
+
+    if (values.length !== 0) {
+        values.push(id);
+        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
+    }
+
+    database.transaction(queries, function (error, results) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results[results.length - 1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        return callback(null);
+    });
+}
+
+function setHealth(appId, health, healthTime, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof health, 'string');
+    assert(util.isDate(healthTime));
+    assert.strictEqual(typeof callback, 'function');
+
+    var values = { health, healthTime };
+
+    updateWithConstraints(appId, values, '', callback);
+}
+
+function setTask(appId, values, options, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof values, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!options.requireNullTaskId)  return updateWithConstraints(appId, values, '', callback);
+
+    if (options.requiredState === null) {
+        updateWithConstraints(appId, values, 'AND taskId IS NULL', callback);
+    } else {
+        updateWithConstraints(appId, values, `AND taskId IS NULL AND installationState = "${options.requiredState}"`, callback);
+    }
+}
+
+function getAppStoreIds(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT id, appStoreId FROM apps', function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAddonConfig(appId, addonId, env, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert(util.isArray(env));
+    assert.strictEqual(typeof callback, 'function');
+
+    unsetAddonConfig(appId, addonId, function (error) {
+        if (error) return callback(error);
+
+        if (env.length === 0) return callback(null);
+
+        var query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
+        var args = [ ], queryArgs = [ ];
+        for (var i = 0; i < env.length; i++) {
+            args.push(appId, addonId, env[i].name, env[i].value);
+            queryArgs.push('(?, ?, ?, ?)');
+        }
+
+        database.query(query + queryArgs.join(','), args, function (error) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            return callback(null);
+        });
+    });
+}
+
+function unsetAddonConfig(appId, addonId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function unsetAddonConfigByAppId(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getAddonConfig(appId, addonId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getAddonConfigByAppId(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getAppIdByAddonConfigValue(addonId, namePattern, value, callback) {
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        callback(null, results[0].appId);
+    });
+}
+
+function getAddonConfigByName(appId, addonId, namePattern, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        callback(null, results[0].value);
+    });
+}

src/apphealthmonitor.js

@@ -1,10 +1,11 @@
 'use strict';
 
-const apps = require('./apps.js'),
+var appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
     assert = require('assert'),
-    AuditSource = require('./auditsource.js'),
+    async = require('async'),
+    auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
-    constants = require('./constants.js'),
     debug = require('debug')('box:apphealthmonitor'),
     docker = require('./docker.js'),
     eventlog = require('./eventlog.js'),
@@ -15,15 +16,17 @@ exports = module.exports = {
     run
 };
 
+const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
 const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes
 
-const OOM_EVENT_LIMIT = 60 * 60 * 1000; // will only raise 1 oom event every hour
+const OOM_EVENT_LIMIT = 60 * 60 * 1000; // 60 minutes
 let gStartTime = null; // time when apphealthmonitor was started
 let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago
 
-async function setHealth(app, health) {
+function setHealth(app, health, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof health, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     // app starts out with null health
     // if it became healthy, we update immediately. this is required for ui to say "running" etc
@@ -38,144 +41,160 @@ async function setHealth(app, health) {
             debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, auditSource.HEALTH_MONITOR, { app: app });
         }
     } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
         if (lastHealth === apps.HEALTH_HEALTHY) {
             debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) await eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, auditSource.HEALTH_MONITOR, { app: app });
         }
     } else {
         debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);
-        return;
+        return callback(null);
     }
 
-    const [error] = await safe(apps.setHealth(app.id, health, healthTime));
-    if (error && error.reason === BoxError.NOT_FOUND) return; // app uninstalled?
-    if (error) throw error;
+    appdb.setHealth(app.id, health, healthTime, function (error) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(null); // app uninstalled?
+        if (error) return callback(error);
 
-    app.health = health;
-    app.healthTime = healthTime;
+        app.health = health;
+        app.healthTime = healthTime;
+
+        callback(null);
+    });
 }
 
 
 // callback is called with error for fatal errors and not if health check failed
-async function checkAppHealth(app, options) {
+function checkAppHealth(app, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) return;
+    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) {
+        return callback(null);
+    }
 
     const manifest = app.manifest;
 
-    const [error, data] = await safe(docker.inspect(app.containerId));
-    if (error || !data || !data.State) return await setHealth(app, apps.HEALTH_ERROR);
-    if (data.State.Running !== true) return await setHealth(app, apps.HEALTH_DEAD);
+    docker.inspect(app.containerId, function (error, data) {
+        if (error || !data || !data.State) return setHealth(app, apps.HEALTH_ERROR, callback);
+        if (data.State.Running !== true) return setHealth(app, apps.HEALTH_DEAD, callback);
 
-    // non-appstore apps may not have healthCheckPath
-    if (!manifest.healthCheckPath) return await setHealth(app, apps.HEALTH_HEALTHY);
+        // non-appstore apps may not have healthCheckPath
+        if (!manifest.healthCheckPath) return setHealth(app, apps.HEALTH_HEALTHY, callback);
 
-    const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
-    const [healthCheckError, response] = await safe(superagent
-        .get(healthCheckUrl)
-        .set('Host', app.fqdn) // required for some apache configs with rewrite rules
-        .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
-        .redirects(0)
-        .ok(() => true)
-        .timeout(options.timeout * 1000));
-
-    if (healthCheckError) {
-        await setHealth(app, apps.HEALTH_UNHEALTHY);
-    } else if (response.status > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
-        await setHealth(app, apps.HEALTH_UNHEALTHY);
-    } else {
-        await setHealth(app, apps.HEALTH_HEALTHY);
-    }
+        const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
+        superagent
+            .get(healthCheckUrl)
+            .set('Host', app.fqdn) // required for some apache configs with rewrite rules
+            .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
+            .redirects(0)
+            .timeout(HEALTHCHECK_INTERVAL)
+            .end(function (error, res) {
+                if (error && !error.response) {
+                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
+                } else if (res.statusCode >= 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
+                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
+                } else {
+                    setHealth(app, apps.HEALTH_HEALTHY, callback);
+                }
+            });
+    });
 }
 
-async function getContainerInfo(containerId) {
-    const result = await docker.inspect(containerId);
+function getContainerInfo(containerId, callback) {
+    docker.inspect(containerId, function (error, result) {
+        if (error) return callback(error);
 
-    const appId = safe.query(result, 'Config.Labels.appId', null);
-    if (appId) return { app: await apps.get(appId) }; // don't get by container id as this can be an exec container
+        const appId = safe.query(result, 'Config.Labels.appId', null);
 
-    if (result.Name.startsWith('/redis-')) {
-        return { app: await apps.get(result.Name.slice('/redis-'.length)), addonName: 'redis' };
-    } else {
-        return { addonName: result.Name.slice(1) }; // addon . Name has a '/' in the beginning for some reason
-    }
+        if (!appId) return callback(null, null /* app */, { name: result.Name }); // addon
+
+        apps.get(appId, callback); // don't get by container id as this can be an exec container
+    });
 }
 
 /*
     OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:3.0.0 /bin/bash
+        docker run -ti -m 100M cloudron/base:2.0.0 /bin/bash
+        apt-get update && apt-get install stress
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
-async function processDockerEvents(options) {
-    assert.strictEqual(typeof options, 'object');
+function processDockerEvents(intervalSecs, callback) {
+    assert.strictEqual(typeof intervalSecs, 'number');
+    assert.strictEqual(typeof callback, 'function');
 
-    const since = ((new Date().getTime() / 1000) - options.intervalSecs).toFixed(0);
+    const since = ((new Date().getTime() / 1000) - intervalSecs).toFixed(0);
     const until = ((new Date().getTime() / 1000) - 1).toFixed(0);
 
-    const stream = await docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) });
-    stream.setEncoding('utf8');
-    stream.on('data', async function (data) { // this is actually ldjson, we only process the first line for now
-        const event = safe.JSON.parse(data);
-        if (!event) return;
-        const containerId = String(event.id);
+    docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
+        if (error) return callback(error);
 
-        const [error, info] = await safe(getContainerInfo(containerId));
-        const program = error ? containerId : (info.addonName || info.app.fqdn);
-        const now = Date.now();
+        stream.setEncoding('utf8');
+        stream.on('data', function (data) {
+            const event = JSON.parse(data);
+            const containerId = String(event.id);
 
-        // do not send mails for dev apps
-        const notifyUser = !(info.app && info.app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
+            getContainerInfo(containerId, function (error, app, addon) {
+                const program = error ? containerId : (app ? app.fqdn : addon.name);
+                const now = Date.now();
+                const notifyUser = !(app && app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
 
-        debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
+                debug('OOM %s notifyUser: %s. lastOomTime: %s (now: %s)', program, notifyUser, gLastOomMailTime, now);
 
-        if (notifyUser) {
-            await eventlog.add(eventlog.ACTION_APP_OOM, AuditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null });
+                // do not send mails for dev apps
+                if (notifyUser) {
+                    // app can be null for addon containers
+                    eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event: event, containerId: containerId, addon: addon || null, app: app || null });
 
-            gLastOomMailTime = now;
-        }
+                    gLastOomMailTime = now;
+                }
+            });
+        });
+
+        stream.on('error', function (error) {
+            debug('Error reading docker events', error);
+            callback();
+        });
+
+        stream.on('end', callback);
+
+        // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
+        setTimeout(stream.destroy.bind(stream), 3000); // https://github.com/apocas/dockerode/issues/179
     });
-
-    stream.on('error', function (error) {
-        debug('Error reading docker events', error);
-    });
-
-    stream.on('end', function () {
-        // debug('Event stream ended');
-    });
-
-    // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
-    setTimeout(stream.destroy.bind(stream), options.timeout); // https://github.com/apocas/dockerode/issues/179
 }
 
-async function processApp(options) {
-    assert.strictEqual(typeof options, 'object');
+function processApp(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    const allApps = await apps.list();
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(error);
 
-    const healthChecks = allApps.map((app) => checkAppHealth(app, options)); // start healthcheck in parallel
+        async.each(allApps, checkAppHealth, function (error) {
+            const alive = allApps
+                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
 
-    await Promise.allSettled(healthChecks); // wait for all promises to finish
+            debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.` + (error ? ` ${error.reason}` : ''));
 
-    const alive = allApps
-        .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
-
-    debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.`);
+            callback(null);
+        });
+    });
 }
 
-async function run(intervalSecs) {
+function run(intervalSecs, callback) {
     assert.strictEqual(typeof intervalSecs, 'number');
-
-    if (constants.TEST) return;
+    assert.strictEqual(typeof callback, 'function');
 
     if (!gStartTime) gStartTime = new Date();
 
-    await processApp({ timeout: (intervalSecs - 3) * 1000 });
-    await processDockerEvents({ intervalSecs, timeout: 3000 });
+    async.series([
+        processApp, // this is first because docker.getEvents seems to get 'stuck' sometimes
+        processDockerEvents.bind(null, intervalSecs)
+    ], function (error) {
+        if (error) debug(`run: could not check app health. ${error.message}`);
+
+        callback();
+    });
 }

src/apppasswords.js

@@ -1,88 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get,
-    add,
-    list,
-    del,
-
-    removePrivateFields
-};
-
-const assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    crypto = require('crypto'),
-    database = require('./database.js'),
-    hat = require('./hat.js'),
-    safe = require('safetydance'),
-    uuid = require('uuid'),
-    _ = require('underscore');
-
-const APP_PASSWORD_FIELDS = [ 'id', 'name', 'userId', 'identifier', 'hashedPassword', 'creationTime' ].join(',');
-
-function validateAppPasswordName(name) {
-    assert.strictEqual(typeof name, 'string');
-
-    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 char');
-    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'name too long');
-
-    return null;
-}
-
-function removePrivateFields(appPassword) {
-    return _.pick(appPassword, 'id', 'name', 'userId', 'identifier', 'creationTime');
-}
-
-async function get(id) {
-    assert.strictEqual(typeof id, 'string');
-
-    const result = await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE id = ?', [ id ]);
-    if (result.length === 0) return null;
-    return result[0];
-}
-
-async function add(userId, identifier, name) {
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof name, 'string');
-
-    let error = validateAppPasswordName(name);
-    if (error) throw error;
-
-    if (identifier.length < 1) throw new BoxError(BoxError.BAD_FIELD, 'identifier must be atleast 1 char');
-
-    const password = hat(16 * 4);
-    const hashedPassword = crypto.createHash('sha256').update(password).digest('base64');
-
-    const appPassword = {
-        id: 'uid-' + uuid.v4(),
-        name,
-        userId,
-        identifier,
-        password,
-        hashedPassword
-    };
-
-    const query = 'INSERT INTO appPasswords (id, userId, identifier, name, hashedPassword) VALUES (?, ?, ?, ?, ?)';
-    const args = [ appPassword.id, appPassword.userId, appPassword.identifier, appPassword.name, appPassword.hashedPassword ];
-
-    [error] = await safe(database.query(query, args));
-    if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('appPasswords_name_userId_identifier') !== -1) throw new BoxError(BoxError.ALREADY_EXISTS, 'name/app combination already exists');
-    if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.indexOf('userId')) throw new BoxError(BoxError.NOT_FOUND, 'user not found');
-    if (error) throw error;
-
-    return { id: appPassword.id, password: appPassword.password };
-}
-
-async function list(userId) {
-    assert.strictEqual(typeof userId, 'string');
-
-    return await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE userId = ?', [ userId ]);
-}
-
-async function del(id) {
-    assert.strictEqual(typeof id, 'string');
-
-    const result = await database.query('DELETE FROM appPasswords WHERE id = ?', [ id ]);
-    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'password not found');
-}

src/appstore.js

@@ -7,27 +7,27 @@ exports = module.exports = {
     getApp,
     getAppVersion,
 
+    trackBeginSetup,
+    trackFinishedSetup,
+
     registerWithLoginCredentials,
-    updateCloudron,
 
     purchaseApp,
     unpurchaseApp,
 
-    getWebToken,
+    getUserToken,
     getSubscription,
     isFreePlan,
 
     getAppUpdate,
     getBoxUpdate,
 
-    createTicket,
-
-    // exported for tests
-    _unregister: unregister
+    createTicket
 };
 
-const apps = require('./apps.js'),
+var apps = require('./apps.js'),
     assert = require('assert'),
+    async = require('async'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:appstore'),
@@ -37,7 +37,6 @@ const apps = require('./apps.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     settings = require('./settings.js'),
-    sysinfo = require('./sysinfo.js'),
     superagent = require('superagent'),
     support = require('./support.js'),
     util = require('util');
@@ -53,7 +52,7 @@ let gFeatures = {
     privateDockerRegistry: false,
     branding: false,
     support: false,
-    profileConfig: false,
+    directoryConfig: false,
     mailboxMaxCount: 5,
     emailPremium: false
 };
@@ -77,66 +76,99 @@ function isAppAllowed(appstoreId, listingConfig) {
     return true;
 }
 
-async function login(email, password, totpToken) {
+function getCloudronToken(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getCloudronToken(function (error, token) {
+        if (error) return callback(error);
+        if (!token) return callback(new BoxError(BoxError.LICENSE_ERROR, 'Missing token'));
+
+        callback(null, token);
+    });
+}
+
+function login(email, password, totpToken, callback) {
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof totpToken, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/login`)
-        .send({ email, password, totpToken })
-        .timeout(30 * 1000)
-        .ok(() => true));
+    var data = {
+        email: email,
+        password: password,
+        totpToken: totpToken
+    };
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. status code: ${response.status}`);
-    if (!response.body.accessToken) throw new BoxError(BoxError.EXTERNAL_ERROR, `Login error. invalid response: ${response.text}`);
+    const url = settings.apiServerOrigin() + '/api/v1/login';
+    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${result.statusCode}`));
 
-    return response.body; // { userId, accessToken }
+        callback(null, result.body); // { userId, accessToken }
+    });
 }
 
-async function registerUser(email, password) {
+function registerUser(email, password, callback) {
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/register_user`)
-        .send({ email, password, utmSource: 'cloudron-dashboard' })
-        .timeout(30 * 1000)
-        .ok(() => true));
+    var data = {
+        email: email,
+        password: password,
+    };
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, 'Registration error: account already exists');
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Registration error. invalid response: ${response.status}`);
+    const url = settings.apiServerOrigin() + '/api/v1/register_user';
+    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${result.statusCode}`));
+
+        callback(null);
+    });
 }
 
-async function getWebToken() {
-    if (settings.isDemo()) throw new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode');
+function getUserToken(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getAppstoreWebToken();
-    if (!token) throw new BoxError(BoxError.NOT_FOUND); // user will have to re-login with password somehow
+    if (settings.isDemo()) return callback(new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode'));
 
-    return token;
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
+
+        const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
+
+        superagent.post(url).send({}).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${result.status}`));
+
+            callback(null, result.body.accessToken);
+        });
+    });
 }
 
-async function getSubscription() {
-    const token = await settings.getAppstoreApiToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+function getSubscription(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/subscription`)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 502) throw new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`);
+        const url = settings.apiServerOrigin() + '/api/v1/subscription';
+        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR));
+            if (result.statusCode === 502) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`));
+            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`));
 
-    // update the features cache
-    gFeatures = response.body.features;
-    safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
+            // update the features cache
+            gFeatures = result.body.features;
+            safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
 
-    return response.body;
+            callback(null, result.body);
+        });
+    });
 }
 
 function isFreePlan(subscription) {
@@ -144,209 +176,226 @@ function isFreePlan(subscription) {
 }
 
 // See app.js install it will create a db record first but remove it again if appstore purchase fails
-async function purchaseApp(data) {
+function purchaseApp(data, callback) {
     assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId, appId }
     assert(data.appstoreId || data.manifestId);
     assert.strictEqual(typeof data.appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getAppstoreApiToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/cloudronapps`)
-        .send(data)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
+        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps`;
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND); // appstoreId does not exist
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
-    // 200 if already purchased, 201 is newly purchased
-    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', response.status, response.body));
+        superagent.post(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); // appstoreId does not exist
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 402) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            // 200 if already purchased, 201 is newly purchased
+            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', result.status, result.body)));
+
+            callback(null);
+        });
+    });
 }
 
-async function unpurchaseApp(appId, data) {
+function unpurchaseApp(appId, data, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId }
     assert(data.appstoreId || data.manifestId);
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getAppstoreApiToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
+        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
 
-    let [error, response] = await safe(superagent.get(url)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
+        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 404) return callback(null);   // was never purchased
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 404) return;   // was never purchased
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed to get app. status:${response.status}`);
+            superagent.del(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
+                if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
 
-    [error, response] = await safe(superagent.del(url)
-        .send(data)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
-
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, `App unpurchase failed. status:${response.status}`);
+                callback(null);
+            });
+        });
+    });
 }
 
-async function getBoxUpdate(options) {
+function getBoxUpdate(options, callback) {
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getAppstoreApiToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const query = {
-        accessToken: token,
-        boxVersion: constants.VERSION,
-        automatic: options.automatic
-    };
+        const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
 
-    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/boxupdate`)
-        .query(query)
-        .timeout(30 * 1000)
-        .ok(() => true));
+        const query = {
+            accessToken: token,
+            boxVersion: constants.VERSION,
+            automatic: options.automatic
+        };
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 204) return; // no update
-    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode === 204) return callback(null, null); // no update
+            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-    const updateInfo = response.body;
+            var updateInfo = result.body;
 
-    if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
-        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', response.status, response.text));
-    }
+            if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
+                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', result.statusCode, result.text)));
+            }
 
-    // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
-    if (!updateInfo.version || typeof updateInfo.version !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
-    if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
-    if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', response.status, response.text));
-    if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', response.status, response.text));
-    if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', response.status, response.text));
-    if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', response.status, response.text));
+            // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
+            if (!updateInfo.version || typeof updateInfo.version !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', result.statusCode, result.text)));
+            if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', result.statusCode, result.text)));
 
-    return updateInfo;
+            callback(null, updateInfo);
+        });
+    });
 }
 
-async function getAppUpdate(app, options) {
+function getAppUpdate(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getAppstoreApiToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const query = {
-        accessToken: token,
-        boxVersion: constants.VERSION,
-        appId: app.appStoreId,
-        appVersion: app.manifest.version,
-        automatic: options.automatic
-    };
+        const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
+        const query = {
+            accessToken: token,
+            boxVersion: constants.VERSION,
+            appId: app.appStoreId,
+            appVersion: app.manifest.version,
+            automatic: options.automatic
+        };
 
-    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/appupdate`)
-        .query(query)
-        .timeout(30 * 1000)
-        .ok(() => true));
+        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
+            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+            if (result.statusCode === 204) return callback(null); // no update
+            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 204) return; // no update
-    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+            const updateInfo = result.body;
 
-    const updateInfo = response.body;
+            // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
+            const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
 
-    // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
-    const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
+            // do some sanity checks
+            if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
+                debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
+                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', result.statusCode, result.text)));
+            }
 
-    // do some sanity checks
-    if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
-        debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
-        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', response.status, response.text));
-    }
+            updateInfo.unstable = !!updateInfo.unstable;
 
-    updateInfo.unstable = !!updateInfo.unstable;
-
-    // { id, creationDate, manifest, unstable }
-    return updateInfo;
+            // { id, creationDate, manifest, unstable }
+            callback(null, updateInfo);
+        });
+    });
 }
 
-async function registerCloudron(data) {
+function registerCloudron(data, callback) {
     assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const { domain, accessToken, version, existingApps } = data;
+    const url = `${settings.apiServerOrigin()}/api/v1/register_cloudron`;
 
-    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/register_cloudron`)
-        .send({ domain, accessToken, version, existingApps })
-        .timeout(30 * 1000)
-        .ok(() => true));
+    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${result.statusCode} ${error.message}`));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${response.statusCode} ${error.message}`);
+        // cloudronId, token, licenseKey
+        if (!result.body.cloudronId) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id'));
+        if (!result.body.cloudronToken) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token'));
+        if (!result.body.licenseKey) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no license'));
 
-    // cloudronId, token
-    if (!response.body.cloudronId) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id');
-    if (!response.body.cloudronToken) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token');
+        async.series([
+            settings.setCloudronId.bind(null, result.body.cloudronId),
+            settings.setCloudronToken.bind(null, result.body.cloudronToken),
+            settings.setLicenseKey.bind(null, result.body.licenseKey),
+        ], function (error) {
+            if (error) return callback(error);
 
-    await settings.setCloudronId(response.body.cloudronId);
-    await settings.setAppstoreApiToken(response.body.cloudronToken);
-    await settings.setAppstoreWebToken(accessToken);
+            debug(`registerCloudron: Cloudron registered with id ${result.body.cloudronId}`);
 
-    debug(`registerCloudron: Cloudron registered with id ${response.body.cloudronId}`);
+            callback();
+        });
+    });
 }
 
-async function updateCloudron(data) {
-    assert.strictEqual(typeof data, 'object');
+// This works without a Cloudron token as this Cloudron was not yet registered
+let gBeginSetupAlreadyTracked = false;
+function trackBeginSetup() {
+    // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
+    if (gBeginSetupAlreadyTracked) return;
+    gBeginSetupAlreadyTracked = true;
 
-    const { domain } = data;
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
 
-    const token = await settings.getAppstoreApiToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
-
-    const query = {
-        accessToken: token
-    };
-
-    const [error, response] = await safe(superagent.post(`${settings.apiServerOrigin()}/api/v1/update_cloudron`)
-        .query(query)
-        .send({ domain })
-        .timeout(30 * 1000)
-        .ok(() => true));
-
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
-
-    debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
+    superagent.post(url).send({}).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return debug(`trackBeginSetup: ${error.message}`);
+        if (result.statusCode !== 200) return debug(`trackBeginSetup: ${result.statusCode} ${error.message}`);
+    });
 }
 
-async function registerWithLoginCredentials(options) {
+// This works without a Cloudron token as this Cloudron was not yet registered
+function trackFinishedSetup(domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;
+
+    superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return debug(`trackFinishedSetup: ${error.message}`);
+        if (result.statusCode !== 200) return debug(`trackFinishedSetup: ${result.statusCode} ${error.message}`);
+    });
+}
+
+function registerWithLoginCredentials(options, callback) {
     assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (options.signup) await registerUser(options.email, options.password);
+    function maybeSignup(done) {
+        if (!options.signup) return done();
 
-    const result = await login(options.email, options.password, options.totpToken || '');
-    await registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION });
-
-    for (const app of await apps.list()) {
-        await purchaseApp({ appId: app.id, appstoreId: app.appStoreId, manifestId: app.manifest.id || 'customapp' });
+        registerUser(options.email, options.password, done);
     }
+
+    getCloudronToken(function (error, token) {
+        if (token) return callback(new BoxError(BoxError.CONFLICT, 'Cloudron is already registered'));
+
+        maybeSignup(function (error) {
+            if (error) return callback(error);
+
+            login(options.email, options.password, options.totpToken || '', function (error, result) {
+                if (error) return callback(error);
+
+                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
+            });
+        });
+    });
 }
 
-async function unregister() {
-    await settings.setCloudronId('');
-    await settings.setAppstoreApiToken('');
-    await settings.setAppstoreWebToken('');
-}
-
-async function createTicket(info, auditSource) {
+function createTicket(info, auditSource, callback) {
     assert.strictEqual(typeof info, 'object');
     assert.strictEqual(typeof info.email, 'string');
     assert.strictEqual(typeof info.displayName, 'string');
@@ -354,96 +403,128 @@ async function createTicket(info, auditSource) {
     assert.strictEqual(typeof info.subject, 'string');
     assert.strictEqual(typeof info.description, 'string');
     assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    const token = await settings.getAppstoreApiToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
-
-    if (info.enableSshSupport) {
-        await safe(support.enableRemoteSupport(true, auditSource));
-        info.ipv4 = await sysinfo.getServerIPv4();
+    function collectAppInfoIfNeeded(callback) {
+        if (!info.appId) return callback();
+        apps.get(info.appId, callback);
     }
 
-    info.app = info.appId ? await apps.get(info.appId) : null;
-    info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
+    function enableSshIfNeeded(callback) {
+        if (!info.enableSshSupport) return callback();
 
-    const request = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true);
+        support.enableRemoteSupport(true, auditSource, function (error) {
+            // ensure we can at least get the ticket through
+            if (error) debug('Unable to enable SSH support.', error);
 
-    // either send as JSON through body or as multipart, depending on attachments
-    if (info.app) {
-        request.field('infoJSON', JSON.stringify(info));
-
-        const logPaths = await apps.getLogPaths(info.app);
-        for (const logPath of logPaths) {
-            const logs = safe.child_process.execSync(`tail --lines=1000 ${logPath}`);
-            if (logs) request.attach(path.basename(logPath), logs, path.basename(logPath));
-        }
-    } else {
-        request.send(info);
+            callback();
+        });
     }
 
-    const [error, response] = await safe(request);
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    await eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+        enableSshIfNeeded(function (error) {
+            if (error) return callback(error);
 
-    return { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` };
+            collectAppInfoIfNeeded(function (error, app) {
+                if (error) return callback(error);
+                if (app) info.app = app;
+
+                info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
+
+                var req = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
+                    .query({ accessToken: token })
+                    .timeout(30 * 1000);
+
+                // either send as JSON through body or as multipart, depending on attachments
+                if (info.app) {
+                    req.field('infoJSON', JSON.stringify(info));
+
+                    apps.getLocalLogfilePaths(info.app).forEach(function (filePath) {
+                        var logs = safe.child_process.execSync(`tail --lines=1000 ${filePath}`);
+                        if (logs) req.attach(path.basename(filePath), logs, path.basename(filePath));
+                    });
+                } else {
+                    req.send(info);
+                }
+
+                req.end(function (error, result) {
+                    if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                    if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                    if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                    if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+
+                    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
+
+                    callback(null, { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
+                });
+            });
+        });
+    });
 }
 
-async function getApps() {
-    const token = await settings.getAppstoreApiToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+function getApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    const unstable = await settings.getUnstableAppsConfig();
+    getCloudronToken(function (error, token) {
+        if (error) return callback(error);
 
-    const [error, response] = await safe(superagent.get(`${settings.apiServerOrigin()}/api/v1/apps`)
-        .query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable })
-        .timeout(30 * 1000)
-        .ok(() => true));
+        settings.getUnstableAppsConfig(function (error, unstable) {
+            if (error) return callback(error);
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 403 || response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', response.status, response.body));
-    if (!response.body.apps) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+            const url = `${settings.apiServerOrigin()}/api/v1/apps`;
+            superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable }).timeout(30 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
+                if (!result.body.apps) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
-    const listingConfig = await settings.getAppstoreListingConfig();
-    const filteredApps = response.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
-    return filteredApps;
+                settings.getAppstoreListingConfig(function (error, listingConfig) {
+                    if (error) return callback(error);
+
+                    const filteredApps = result.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
+
+                    callback(null, filteredApps);
+                });
+            });
+        });
+    });
 }
 
-async function getAppVersion(appId, version) {
+function getAppVersion(appId, version, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof version, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    const listingConfig = await settings.getAppstoreListingConfig();
+    settings.getAppstoreListingConfig(function (error, listingConfig) {
+        if (error) return callback(error);
 
-    if (!isAppAllowed(appId, listingConfig)) throw new BoxError(BoxError.FEATURE_DISABLED);
+        if (!isAppAllowed(appId, listingConfig)) return callback(new BoxError(BoxError.FEATURE_DISABLED));
 
-    const token = await settings.getAppstoreApiToken();
-    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+        getCloudronToken(function (error, token) {
+            if (error) return callback(error);
 
-    let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
-    if (version !== 'latest') url += `/versions/${version}`;
+            let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
+            if (version !== 'latest') url += `/versions/${version}`;
 
-    const [error, response] = await safe(superagent.get(url)
-        .query({ accessToken: token })
-        .timeout(30 * 1000)
-        .ok(() => true));
+            superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
 
-    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
-    if (response.status === 403 || response.statusCode === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
-    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND);
-    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', response.status, response.body));
-
-    return response.body;
+                callback(null, result.body);
+            });
+        });
+    });
 }
 
-async function getApp(appId) {
+function getApp(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    return await getAppVersion(appId, 'latest');
+    getAppVersion(appId, 'latest', callback);
 }

src/apptaskmanager.js

@@ -4,7 +4,7 @@ exports = module.exports = {
     scheduleTask
 };
 
-const assert = require('assert'),
+let assert = require('assert'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:apptaskmanager'),
     fs = require('fs'),
@@ -13,6 +13,7 @@ const assert = require('assert'),
     path = require('path'),
     paths = require('./paths.js'),
     scheduler = require('./scheduler.js'),
+    services = require('./services.js'),
     tasks = require('./tasks.js');
 
 let gActiveTasks = { }; // indexed by app id
@@ -20,11 +21,12 @@ let gPendingTasks = [ ];
 let gInitialized = false;
 
 const TASK_CONCURRENCY = 3;
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 function waitText(lockOperation) {
     if (lockOperation === locker.OP_BOX_UPDATE) return 'Waiting for Cloudron to finish updating. See the Settings view';
     if (lockOperation === locker.OP_PLATFORM_START) return 'Waiting for Cloudron to initialize';
-    if (lockOperation === locker.OP_FULL_BACKUP) return 'Waiting for Cloudron to finish backup. See the Backups view';
+    if (lockOperation === locker.OP_FULL_BACKUP) return 'Wait for Cloudron to finish backup. See the Backups view';
 
     return ''; // cannot happen
 }
@@ -35,31 +37,31 @@ function initializeSync() {
 }
 
 // callback is called when task is finished
-function scheduleTask(appId, taskId, options, onFinished) {
+function scheduleTask(appId, taskId, options, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof taskId, 'string');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof onFinished, 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     if (!gInitialized) initializeSync();
 
     if (appId in gActiveTasks) {
-        return onFinished(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
+        return callback(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
     }
 
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' });
-        gPendingTasks.push({ appId, taskId, options, onFinished });
+        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
+        gPendingTasks.push({ appId, taskId, options, callback });
         return;
     }
 
-    const lockError = locker.recursiveLock(locker.OP_APPTASK);
+    var lockError = locker.recursiveLock(locker.OP_APPTASK);
 
     if (lockError) {
         debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) });
-        gPendingTasks.push({ appId, taskId, options, onFinished });
+        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
+        gPendingTasks.push({ appId, taskId, options, callback });
         return;
     }
 
@@ -72,7 +74,7 @@ function scheduleTask(appId, taskId, options, onFinished) {
     scheduler.suspendJobs(appId);
 
     tasks.startTask(taskId, Object.assign(options, { logFile }), function (error, result) {
-        onFinished(error, result);
+        callback(error, result);
 
         delete gActiveTasks[appId];
         locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
@@ -82,10 +84,15 @@ function scheduleTask(appId, taskId, options, onFinished) {
 }
 
 function startNextTask() {
-    if (gPendingTasks.length === 0) return;
+    if (gPendingTasks.length === 0) {
+        // rebuild sftp when task queue is empty. this minimizes risk of sftp rebuild overlapping with other app tasks
+        services.rebuildService('sftp', error => { if (error) debug('Unable to rebuild sftp:', error); });
+        return;
+    }
 
     assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
     const t = gPendingTasks.shift();
-    scheduleTask(t.appId, t.taskId, t.options, t.onFinished);
+    scheduleTask(t.appId, t.taskId, t.options, t.callback);
 }
+

src/auditsource.js

@@ -1,24 +1,16 @@
 'use strict';
 
-class AuditSource {
-    constructor(username, userId, ip) {
-        this.username = username;
-        this.userId = userId || null;
-        this.ip = ip || null;
-    }
+exports = module.exports = {
+    CRON:  { userId: null, username: 'cron' },
+    HEALTH_MONITOR: { userId: null, username: 'healthmonitor' },
+    APP_TASK: { userId: null, username: 'apptask' },
+    EXTERNAL_LDAP_TASK: { userId: null, username: 'externalldap' },
+    EXTERNAL_LDAP_AUTO_CREATE: { userId: null, username: 'externalldap' },
 
-    static fromRequest(req) {
-        const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-        return new AuditSource(req.user?.username, req.user?.id, ip);
-    }
+    fromRequest: fromRequest
+};
+
+function fromRequest(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
 }
-
-// these can be static variables but see https://stackoverflow.com/questions/60046847/eslint-does-not-allow-static-class-properties#comment122122927_60464446
-AuditSource.CRON =  new AuditSource('cron');
-AuditSource.HEALTH_MONITOR = new AuditSource('healthmonitor');
-AuditSource.EXTERNAL_LDAP_TASK = new AuditSource('externalldap');
-AuditSource.EXTERNAL_LDAP_AUTO_CREATE = new AuditSource('externalldap');
-AuditSource.APPTASK = new AuditSource('apptask');
-AuditSource.PLATFORM = new AuditSource('platform');
-
-exports = module.exports = AuditSource;

src/autoconfig.xml.ejs

@@ -19,13 +19,7 @@
       <username>%EMAILADDRESS%</username>
       <addThisServer>true</addThisServer>
     </outgoingServer>
-    <incomingServer type="pop3">
-      <hostname><%= mailFqdn %></hostname>
-      <port>995</port>
-      <socketType>SSL</socketType>
-      <username>%EMAILADDRESS%</username>
-      <authentication>password-cleartext</authentication>
-    </incomingServer>
+
     <documentation url="http://cloudron.io/email/#autodiscover">
        <descr lang="en">Cloudron Email</descr>
     </documentation>

src/backupcleaner.js

@@ -1,303 +0,0 @@
-'use strict';
-
-const BoxError = require('./boxerror.js');
-
-exports = module.exports = {
-    run,
-
-    _applyBackupRetentionPolicy: applyBackupRetentionPolicy
-};
-
-const apps = require('./apps.js'),
-    assert = require('assert'),
-    backupFormat = require('./backupformat.js'),
-    backups = require('./backups.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:backupcleaner'),
-    moment = require('moment'),
-    mounts = require('./mounts.js'),
-    path = require('path'),
-    paths = require('./paths.js'),
-    safe = require('safetydance'),
-    settings = require('./settings.js'),
-    storage = require('./storage.js'),
-    _ = require('underscore');
-
-function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
-    assert(Array.isArray(allBackups));
-    assert.strictEqual(typeof policy, 'object');
-    assert(Array.isArray(referencedBackupIds));
-
-    const now = new Date();
-
-    for (const backup of allBackups) {
-        if (backup.state === backups.BACKUP_STATE_ERROR) {
-            backup.discardReason = 'error';
-        } else if (backup.state === backups.BACKUP_STATE_CREATING) {
-            if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
-            else backup.discardReason = 'creating-too-long';
-        } else if (referencedBackupIds.includes(backup.id)) {
-            backup.keepReason = 'reference';
-        } else if ((backup.preserveSecs === -1) || ((now - backup.creationTime) < (backup.preserveSecs * 1000))) {
-            backup.keepReason = 'preserveSecs';
-        } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) {
-            backup.keepReason = 'keepWithinSecs';
-        }
-    }
-
-    const KEEP_FORMATS = {
-        keepDaily: 'Y-M-D',
-        keepWeekly: 'Y-W',
-        keepMonthly: 'Y-M',
-        keepYearly: 'Y'
-    };
-
-    for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) {
-        if (!(format in policy)) continue;
-
-        const n = policy[format]; // we want to keep "n" backups of format
-        if (!n) continue; // disabled rule
-
-        let lastPeriod = null, keptSoFar = 0;
-        for (const backup of allBackups) {
-            if (backup.discardReason) continue; // already discarded for some reason
-            if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason
-            const period = moment(backup.creationTime).format(KEEP_FORMATS[format]);
-            if (period === lastPeriod) continue; // already kept for this period
-
-            lastPeriod = period;
-            backup.keepReason = backup.keepReason ? `${backup.keepReason}+${format}` : format;
-            if (++keptSoFar === n) break;
-        }
-    }
-
-    if (policy.keepLatest) {
-        let latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL);
-        if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest';
-    }
-
-    for (const backup of allBackups) {
-        debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
-    }
-}
-
-async function removeBackup(backupConfig, backup, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof backup, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const backupFilePath = backupFormat.api(backup.format).getBackupFilePath(backupConfig, backup.remotePath);
-
-    let removeError;
-    if (backup.format ==='tgz') {
-        progressCallback({ message: `${backup.remotePath}: Removing ${backupFilePath}`});
-        [removeError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFilePath));
-    } else {
-        progressCallback({ message: `${backup.remotePath}: Removing directory ${backupFilePath}`});
-        [removeError] = await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath, progressCallback));
-    }
-
-    if (removeError) {
-        debug('removeBackup: error removing backup %j : %s', backup, removeError.message);
-        return;
-    }
-
-    // prune empty directory if possible
-    const [pruneError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath)));
-    if (pruneError) debug('removeBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), pruneError.message);
-
-    const [delError] = await safe(backups.del(backup.id));
-    if (delError) debug(`removeBackup: error removing ${backup.id} from database`, delError);
-    else debug(`removeBackup: removed ${backup.remotePath}`);
-}
-
-async function cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert(Array.isArray(referencedBackupIds));
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const removedAppBackupPaths = [];
-
-    const allApps = await apps.list();
-    const allAppIds = allApps.map(a => a.id);
-
-    const appBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000);
-
-    // collate the backups by app id. note that the app could already have been uninstalled
-    let appBackupsById = {};
-    for (const appBackup of appBackups) {
-        if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = [];
-        appBackupsById[appBackup.identifier].push(appBackup);
-    }
-
-    // apply backup policy per app. keep latest backup only for existing apps
-    let appBackupsToRemove = [];
-    for (const appId of Object.keys(appBackupsById)) {
-        applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedBackupIds);
-        appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
-    }
-
-    for (const appBackup of appBackupsToRemove) {
-        await progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
-        removedAppBackupPaths.push(appBackup.remotePath);
-        await removeBackup(backupConfig, appBackup, progressCallback); // never errors
-    }
-
-    debug('cleanupAppBackups: done');
-
-    return removedAppBackupPaths;
-}
-
-async function cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert(Array.isArray(referencedBackupIds));
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const removedMailBackupPaths = [];
-
-    const mailBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_MAIL, 1, 1000);
-
-    applyBackupRetentionPolicy(mailBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), referencedBackupIds);
-
-    for (const mailBackup of mailBackups) {
-        if (mailBackup.keepReason) continue;
-        await progressCallback({ message: `Removing mail backup ${mailBackup.remotePath}`});
-        removedMailBackupPaths.push(mailBackup.remotePath);
-        await removeBackup(backupConfig, mailBackup, progressCallback); // never errors
-    }
-
-    debug('cleanupMailBackups: done');
-
-    return removedMailBackupPaths;
-}
-
-async function cleanupBoxBackups(backupConfig, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    let referencedBackupIds = [], removedBoxBackupPaths = [];
-
-    const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
-
-    applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */);
-
-    for (const boxBackup of boxBackups) {
-        if (boxBackup.keepReason) {
-            referencedBackupIds = referencedBackupIds.concat(boxBackup.dependsOn);
-            continue;
-        }
-
-        await progressCallback({ message: `Removing box backup ${boxBackup.remotePath}`});
-
-        removedBoxBackupPaths.push(boxBackup.remotePath);
-        await removeBackup(backupConfig, boxBackup, progressCallback);
-    }
-
-    debug('cleanupBoxBackups: done');
-
-    return { removedBoxBackupPaths, referencedBackupIds };
-}
-
-// cleans up the database by checking if backup existsing in the remote
-async function cleanupMissingBackups(backupConfig, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const perPage = 1000;
-    const missingBackupPaths = [];
-
-    if (constants.TEST) return missingBackupPaths;
-
-    let page = 1, result = [];
-    do {
-        result = await backups.list(page, perPage);
-
-        for (const backup of result) {
-            let backupFilePath = backupFormat.api(backup.format).getBackupFilePath(backupConfig, backup.remotePath);
-            if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
-
-            const [existsError, exists] = await safe(storage.api(backupConfig.provider).exists(backupConfig, backupFilePath));
-            if (existsError || exists) continue;
-
-            await progressCallback({ message: `Removing missing backup ${backup.remotePath}`});
-
-            const [delError] = await safe(backups.del(backup.id));
-            if (delError) debug(`cleanupMissingBackups: error removing ${backup.id} from database`, delError);
-
-            missingBackupPaths.push(backup.remotePath);
-        }
-
-        ++ page;
-    } while (result.length === perPage);
-
-    debug('cleanupMissingBackups: done');
-
-    return missingBackupPaths;
-}
-
-// removes the snapshots of apps that have been uninstalled
-async function cleanupSnapshots(backupConfig) {
-    assert.strictEqual(typeof backupConfig, 'object');
-
-    const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
-    const info = safe.JSON.parse(contents);
-    if (!info) return;
-
-    delete info.box;
-
-    const progressCallback = (progress) => { debug(`cleanupSnapshots: ${progress.message}`); };
-
-    for (const appId of Object.keys(info)) {
-        const app = await apps.get(appId);
-        if (app) continue; // app is still installed
-
-        if (info[appId].format ==='tgz') {
-            await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`)), { debug });
-        } else {
-            await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`), progressCallback), { debug });
-        }
-
-        safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
-        safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`));
-
-        await safe(backups.setSnapshotInfo(appId, null /* info */), { debug });
-        debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`);
-    }
-
-    debug('cleanupSnapshots: done');
-}
-
-async function run(progressCallback) {
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const backupConfig = await settings.getBackupConfig();
-
-    if (mounts.isManagedProvider(backupConfig.provider) || backupConfig.provider === 'mountpoint') {
-        const hostPath = mounts.isManagedProvider(backupConfig.provider) ? paths.MANAGED_BACKUP_MOUNT_DIR : backupConfig.mountPoint;
-        const status = await mounts.getStatus(backupConfig.provider, hostPath); // { state, message }
-        debug(`clean: mount point status is ${JSON.stringify(status)}`);
-        if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
-    }
-
-    if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
-        debug('cleanup: keeping all backups');
-        return {};
-    }
-
-    await progressCallback({ percent: 10, message: 'Cleaning box backups' });
-    const { removedBoxBackupPaths, referencedBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback); // references is app or mail backup ids
-
-    await progressCallback({ percent: 20, message: 'Cleaning mail backups' });
-    const removedMailBackupPaths = await cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback);
-
-    await progressCallback({ percent: 40, message: 'Cleaning app backups' });
-    const removedAppBackupPaths = await cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback);
-
-    await progressCallback({ percent: 70, message: 'Cleaning missing backups' });
-    const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback);
-
-    await progressCallback({ percent: 90, message: 'Cleaning snapshots' });
-    await cleanupSnapshots(backupConfig);
-
-    return { removedBoxBackupPaths, removedMailBackupPaths, removedAppBackupPaths, missingBackupPaths };
-}

src/backupdb.js

@@ -0,0 +1,177 @@
+'use strict';
+
+var assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    database = require('./database.js'),
+    safe = require('safetydance'),
+    util = require('util');
+
+var BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
+
+exports = module.exports = {
+    add,
+
+    getByTypePaged,
+    getByIdentifierPaged,
+    getByIdentifierAndStatePaged,
+
+    get,
+    del,
+    update,
+    list,
+
+    _clear: clear
+};
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
+
+    result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
+    delete result.manifestJson;
+}
+
+function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof state, 'string');
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ identifier, state, (page-1)*perPage, perPage ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
+function getByTypePaged(type, page, perPage, callback) {
+    assert.strictEqual(typeof type, 'string');
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ type, (page-1)*perPage, perPage ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
+function getByIdentifierPaged(identifier, page, perPage, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ identifier, (page-1)*perPage, perPage ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
+function list(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups ORDER BY creationTime DESC LIMIT ?,?',
+        [ (page-1)*perPage, perPage ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
+        [ id ], function (error, result) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+            if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
+
+            postProcess(result[0]);
+
+            callback(null, result[0]);
+        });
+}
+
+function add(id, data, callback) {
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof id, 'string');
+    assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number');
+    assert.strictEqual(typeof data.packageVersion, 'string');
+    assert.strictEqual(typeof data.type, 'string');
+    assert.strictEqual(typeof data.identifier, 'string');
+    assert.strictEqual(typeof data.state, 'string');
+    assert(util.isArray(data.dependsOn));
+    assert.strictEqual(typeof data.manifest, 'object');
+    assert.strictEqual(typeof data.format, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var creationTime = data.creationTime || new Date(); // allow tests to set the time
+    var manifestJson = JSON.stringify(data.manifest);
+
+    database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ],
+        function (error) {
+            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            callback(null);
+        });
+}
+
+function update(id, backup, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof backup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var fields = [ ], values = [ ];
+    for (var p in backup) {
+        fields.push(p + ' = ?');
+        values.push(backup[p]);
+    }
+    values.push(id);
+
+    database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values, function (error) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE backups', [], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        callback(null);
+    });
+}

src/backupformat.js

@@ -1,12 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    api
-};
-
-function api(format) {
-    switch (format) {
-    case 'tgz': return require('./backupformat/tgz.js');
-    case 'rsync': return require('./backupformat/rsync.js');
-    }
-}

src/backupformat/rsync.js

@@ -1,245 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    getBackupFilePath,
-    download,
-    upload,
-
-    _saveFsMetadata: saveFsMetadata,
-    _restoreFsMetadata: restoreFsMetadata
-};
-
-const assert = require('assert'),
-    async = require('async'),
-    BoxError = require('../boxerror.js'),
-    DataLayout = require('../datalayout.js'),
-    debug = require('debug')('box:backupformat/rsync'),
-    fs = require('fs'),
-    hush = require('../hush.js'),
-    once = require('../once.js'),
-    path = require('path'),
-    safe = require('safetydance'),
-    storage = require('../storage.js'),
-    syncer = require('../syncer.js'),
-    util = require('util');
-
-function getBackupFilePath(backupConfig, remotePath) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-
-    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
-    return path.join(rootPath, remotePath);
-}
-
-function sync(backupConfig, remotePath, dataLayout, progressCallback, callback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB
-    const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10);
-    const removeDir = util.callbackify(storage.api(backupConfig.provider).removeDir);
-    const remove = util.callbackify(storage.api(backupConfig.provider).remove);
-
-    syncer.sync(dataLayout, function processTask(task, iteratorCallback) {
-        debug('sync: processing task: %j', task);
-        // the empty task.path is special to signify the directory
-        const destPath = task.path && backupConfig.encryption ? hush.encryptFilePath(task.path, backupConfig.encryption) : task.path;
-        const backupFilePath = path.join(getBackupFilePath(backupConfig, remotePath), destPath);
-
-        if (task.operation === 'removedir') {
-            debug(`Removing directory ${backupFilePath}`);
-            return removeDir(backupConfig, backupFilePath, progressCallback, iteratorCallback);
-        } else if (task.operation === 'remove') {
-            debug(`Removing ${backupFilePath}`);
-            return remove(backupConfig, backupFilePath, iteratorCallback);
-        }
-
-        let retryCount = 0;
-        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-            retryCallback = once(retryCallback); // protect again upload() erroring much later after read stream error
-
-            ++retryCount;
-            if (task.operation === 'add') {
-                progressCallback({ message: `Adding ${task.path}` + (retryCount > 1 ?  ` (Try ${retryCount})` : '') });
-                debug(`Adding ${task.path} position ${task.position} try ${retryCount}`);
-                const stream = hush.createReadStream(dataLayout.toLocalPath('./' + task.path), backupConfig.encryption);
-                stream.on('error', (error) => retryCallback(error.message.includes('ENOENT') ? null : error)); // ignore error if file disappears
-                stream.on('progress', function (progress) {
-                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                    if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0MBps looks wrong
-                    progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}MBps` }); // 0M@0MBps looks wrong
-                });
-                // only create the destination path when we have confirmation that the source is available. otherwise, we end up with
-                // files owned as 'root' and the cp later will fail
-                stream.on('open', function () {
-                    storage.api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, function (error) {
-                        debug(error ? `Error uploading ${task.path} try ${retryCount}: ${error.message}` : `Uploaded ${task.path}`);
-                        retryCallback(error);
-                    });
-                });
-            }
-        }, iteratorCallback);
-    }, concurrency, function (error) {
-        if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-
-        callback();
-    });
-}
-
-// this is not part of 'snapshotting' because we need root access to traverse
-async function saveFsMetadata(dataLayout, metadataFile) {
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof metadataFile, 'string');
-
-    // contains paths prefixed with './'
-    const metadata = {
-        emptyDirs: [],
-        execFiles: [],
-        symlinks: []
-    };
-
-    // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
-    for (let lp of dataLayout.localPaths()) {
-        const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
-        if (emptyDirs === null) throw new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`);
-        if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
-
-        const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
-        if (execFiles === null) throw new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`);
-        if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
-
-        const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
-        if (symlinks === null) throw new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`);
-        if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => {
-            const target = safe.fs.readlinkSync(sl);
-            return { path: dataLayout.toRemotePath(sl), target };
-        }));
-    }
-
-    if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) throw new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`);
-}
-
-async function restoreFsMetadata(dataLayout, metadataFile) {
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof metadataFile, 'string');
-
-    debug(`Recreating empty directories in ${dataLayout.toString()}`);
-
-    const metadataJson = safe.fs.readFileSync(metadataFile, 'utf8');
-    if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message);
-    const metadata = safe.JSON.parse(metadataJson);
-    if (metadata === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message);
-
-    for (const emptyDir of metadata.emptyDirs) {
-        const [mkdirError] = await safe(fs.promises.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }));
-        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to create path: ${mkdirError.message}`);
-    }
-
-    for (const execFile of metadata.execFiles) {
-        const [chmodError] = await safe(fs.promises.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8)));
-        if (chmodError) throw new BoxError(BoxError.FS_ERROR, `unable to chmod: ${chmodError.message}`);
-    }
-
-    for (const symlink of (metadata.symlinks || [])) {
-        if (!symlink.target) continue;
-        // the path may not exist if we had a directory full of symlinks
-        const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }));
-        if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink (mkdir): ${mkdirError.message}`);
-        const [symlinkError] = await safe(fs.promises.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file'));
-        if (symlinkError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink: ${symlinkError.message}`);
-    }
-}
-
-function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof backupFilePath, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
-
-    function downloadFile(entry, done) {
-        let relativePath = path.relative(backupFilePath, entry.fullPath);
-        if (backupConfig.encryption) {
-            const { error, result } = hush.decryptFilePath(relativePath, backupConfig.encryption);
-            if (error) return done(new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file'));
-            relativePath = result;
-        }
-        const destFilePath = dataLayout.toLocalPath('./' + relativePath);
-
-        fs.mkdir(path.dirname(destFilePath), { recursive: true }, function (error) {
-            if (error) return done(new BoxError(BoxError.FS_ERROR, error.message));
-
-            async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-                storage.api(backupConfig.provider).download(backupConfig, entry.fullPath, function (error, sourceStream) {
-                    if (error) {
-                        progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
-                        return retryCallback(error);
-                    }
-
-                    let destStream = hush.createWriteStream(destFilePath, backupConfig.encryption);
-
-                    // protect against multiple errors. must destroy the write stream so that a previous retry does not write
-                    let closeAndRetry = once((error) => {
-                        if (error) progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${error.message}` });
-                        else progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} finished` });
-                        sourceStream.destroy();
-                        destStream.destroy();
-                        retryCallback(error);
-                    });
-
-                    destStream.on('progress', function (progress) {
-                        const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                        if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0MBps looks wrong
-                        progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}MBps` });
-                    });
-                    destStream.on('error', closeAndRetry);
-
-                    sourceStream.on('error', closeAndRetry);
-
-                    progressCallback({ message: `Downloading ${entry.fullPath} to ${destFilePath}` });
-
-                    sourceStream.pipe(destStream, { end: true }).on('done', closeAndRetry);
-                });
-            }, done);
-        });
-    }
-
-    storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
-        // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
-        const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
-
-        async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
-    }, callback);
-}
-
-async function download(backupConfig, remotePath, dataLayout, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
-
-    const backupFilePath = getBackupFilePath(backupConfig, remotePath);
-    const downloadDirAsync = util.promisify(downloadDir);
-
-    await downloadDirAsync(backupConfig, backupFilePath, dataLayout, progressCallback);
-    await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
-}
-
-async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-    assert.strictEqual(typeof dataLayout, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const syncAsync = util.promisify(sync);
-
-    await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
-    await syncAsync(backupConfig, remotePath, dataLayout, progressCallback);
-}

src/backupformat/tgz.js

@@ -1,195 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    getBackupFilePath,
-    download,
-    upload
-};
-
-const assert = require('assert'),
-    async = require('async'),
-    BoxError = require('../boxerror.js'),
-    DataLayout = require('../datalayout.js'),
-    debug = require('debug')('box:backupformat/tgz'),
-    { DecryptStream, EncryptStream } = require('../hush.js'),
-    once = require('../once.js'),
-    path = require('path'),
-    progressStream = require('progress-stream'),
-    storage = require('../storage.js'),
-    tar = require('tar-fs'),
-    zlib = require('zlib');
-
-function getBackupFilePath(backupConfig, remotePath) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-
-    const rootPath = storage.api(backupConfig.provider).getRootPath(backupConfig);
-
-    const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz';
-    return path.join(rootPath, remotePath + fileType);
-}
-
-function tarPack(dataLayout, encryption) {
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof encryption, 'object');
-
-    const pack = tar.pack('/', {
-        dereference: false, // pack the symlink and not what it points to
-        entries: dataLayout.localPaths(),
-        ignoreStatError: (path, err) => {
-            debug(`tarPack: error stat'ing ${path} - ${err.code}`);
-            return err.code === 'ENOENT'; // ignore if file or dir got removed (probably some temporary file)
-        },
-        map: function(header) {
-            header.name = dataLayout.toRemotePath(header.name);
-            // the tar pax format allows us to encode filenames > 100 and size > 8GB (see #640)
-            // https://www.systutorials.com/docs/linux/man/5-star/
-            if (header.size > 8589934590 || header.name > 99) header.pax = { size: header.size };
-            return header;
-        },
-        strict: false // do not error for unknown types (skip fifo, char/block devices)
-    });
-
-    const gzip = zlib.createGzip({});
-    const ps = progressStream({ time: 10000 }); // emit 'progress' every 10 seconds
-
-    pack.on('error', function (error) {
-        debug('tarPack: tar stream error.', error);
-        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    gzip.on('error', function (error) {
-        debug('tarPack: gzip stream error.', error);
-        ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    if (encryption) {
-        const encryptStream = new EncryptStream(encryption);
-        encryptStream.on('error', function (error) {
-            debug('tarPack: encrypt stream error.', error);
-            ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-        });
-
-        pack.pipe(gzip).pipe(encryptStream).pipe(ps);
-    } else {
-        pack.pipe(gzip).pipe(ps);
-    }
-
-    return ps;
-}
-
-function tarExtract(inStream, dataLayout, encryption) {
-    assert.strictEqual(typeof inStream, 'object');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof encryption, 'object');
-
-    const gunzip = zlib.createGunzip({});
-    const ps = progressStream({ time: 10000 }); // display a progress every 10 seconds
-    const extract = tar.extract('/', {
-        map: function (header) {
-            header.name = dataLayout.toLocalPath(header.name);
-            return header;
-        },
-        dmode: 500 // ensure directory is writable
-    });
-
-    const emitError = once((error) => {
-        inStream.destroy();
-        ps.emit('error', error);
-    });
-
-    inStream.on('error', function (error) {
-        debug('tarExtract: input stream error.', error);
-        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    gunzip.on('error', function (error) {
-        debug('tarExtract: gunzip stream error.', error);
-        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    extract.on('error', function (error) {
-        debug('tarExtract: extract stream error.', error);
-        emitError(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
-    });
-
-    extract.on('finish', function () {
-        debug('tarExtract: done.');
-        // we use a separate event because ps is a through2 stream which emits 'finish' event indicating end of inStream and not extract
-        ps.emit('done');
-    });
-
-    if (encryption) {
-        const decrypt = new DecryptStream(encryption);
-        decrypt.on('error', function (error) {
-            debug('tarExtract: decrypt stream error.', error);
-            emitError(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to decrypt: ${error.message}`));
-        });
-        inStream.pipe(ps).pipe(decrypt).pipe(gunzip).pipe(extract);
-    } else {
-        inStream.pipe(ps).pipe(gunzip).pipe(extract);
-    }
-
-    return ps;
-}
-
-async function download(backupConfig, remotePath, dataLayout, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    debug(`download: Downloading ${remotePath} to ${dataLayout.toString()}`);
-
-    const backupFilePath = getBackupFilePath(backupConfig, remotePath);
-
-    return new Promise((resolve, reject) => {
-        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-            progressCallback({ message: `Downloading backup ${remotePath}` });
-
-            storage.api(backupConfig.provider).download(backupConfig, backupFilePath, function (error, sourceStream) {
-                if (error) return retryCallback(error);
-
-                const ps = tarExtract(sourceStream, dataLayout, backupConfig.encryption);
-
-                ps.on('progress', function (progress) {
-                    const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                    if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong
-                    progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` });
-                });
-                ps.on('error', retryCallback);
-                ps.on('done', retryCallback);
-            });
-        }, (error) => {
-            if (error) return reject(error);
-            resolve();
-        });
-    });
-}
-
-async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-    assert.strictEqual(typeof dataLayout, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    return new Promise((resolve, reject) => {
-        async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-            retryCallback = once(retryCallback); // protect again upload() erroring much later after tar stream error
-
-            const tarStream = tarPack(dataLayout, backupConfig.encryption);
-
-            tarStream.on('progress', function (progress) {
-                const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0MBps looks wrong
-                progressCallback({ message: `Uploading backup ${transferred}M@${speed}MBps` });
-            });
-            tarStream.on('error', retryCallback); // already returns BoxError
-
-            storage.api(backupConfig.provider).upload(backupConfig, getBackupFilePath(backupConfig, remotePath), tarStream, retryCallback);
-        }, (error) => {
-            if (error) return reject(error);
-            resolve();
-        });
-    });
-}

src/backuptask.js

@@ -1,492 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    fullBackup,
-
-    restore,
-
-    backupApp,
-    downloadApp,
-
-    backupMail,
-    downloadMail,
-
-    upload,
-};
-
-const apps = require('./apps.js'),
-    assert = require('assert'),
-    backupFormat = require('./backupformat.js'),
-    backups = require('./backups.js'),
-    BoxError = require('./boxerror.js'),
-    constants = require('./constants.js'),
-    DataLayout = require('./datalayout.js'),
-    database = require('./database.js'),
-    debug = require('debug')('box:backuptask'),
-    path = require('path'),
-    paths = require('./paths.js'),
-    safe = require('safetydance'),
-    services = require('./services.js'),
-    settings = require('./settings.js'),
-    shell = require('./shell.js'),
-    storage = require('./storage.js');
-
-const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');
-
-function canBackupApp(app) {
-    // only backup apps that are installed or specific pending states
-
-    // stopped apps cannot be backed up because addons might be down (redis)
-    if (app.runState === apps.RSTATE_STOPPED) return false;
-
-    // we used to check the health here but that doesn't work for stopped apps. it's better to just fail
-    // and inform the user if the backup fails and the app addons have not been setup yet.
-    return  app.installationState === apps.ISTATE_INSTALLED ||
-            app.installationState === apps.ISTATE_PENDING_CONFIGURE ||
-            app.installationState === apps.ISTATE_PENDING_BACKUP ||  // called from apptask
-            app.installationState === apps.ISTATE_PENDING_UPDATE; // called from apptask
-}
-
-// this function is called via backupupload (since it needs root to traverse app's directory)
-async function upload(remotePath, format, dataLayoutString, progressCallback) {
-    assert.strictEqual(typeof remotePath, 'string');
-    assert.strictEqual(typeof format, 'string');
-    assert.strictEqual(typeof dataLayoutString, 'string');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    debug(`upload: path ${remotePath} format ${format} dataLayout ${dataLayoutString}`);
-
-    const dataLayout = DataLayout.fromString(dataLayoutString);
-    const backupConfig = await settings.getBackupConfig();
-    await safe(storage.api(backupConfig.provider).checkPreconditions(backupConfig, dataLayout));
-
-    await backupFormat.api(format).upload(backupConfig, remotePath, dataLayout, progressCallback);
-}
-
-async function download(backupConfig, remotePath, format, dataLayout, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-    assert.strictEqual(typeof format, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    debug(`download: Downloading ${remotePath} of format ${format} to ${dataLayout.toString()}`);
-
-    await backupFormat.api(format).download(backupConfig, remotePath, dataLayout, progressCallback);
-}
-
-async function restore(backupConfig, remotePath, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof remotePath, 'string');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
-    if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
-    const dataLayout = new DataLayout(boxDataDir, []);
-
-    await download(backupConfig, remotePath, backupConfig.format, dataLayout, progressCallback);
-
-    debug('restore: download completed, importing database');
-
-    await database.importFromFile(`${dataLayout.localRoot()}/box.mysqldump`);
-
-    debug('restore: database imported');
-
-    await settings.initCache();
-}
-
-async function downloadApp(app, restoreConfig, progressCallback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof restoreConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
-    if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
-    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);
-
-    const startTime = new Date();
-    const backupConfig = restoreConfig.backupConfig || await settings.getBackupConfig();
-
-    await download(backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
-    debug('downloadApp: time: %s', (new Date() - startTime)/1000);
-}
-
-async function runBackupUpload(uploadConfig, progressCallback) {
-    assert.strictEqual(typeof uploadConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const { remotePath, backupConfig, dataLayout, progressTag } = uploadConfig;
-    assert.strictEqual(typeof remotePath, 'string');
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof progressTag, 'string');
-    assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
-
-    // https://stackoverflow.com/questions/48387040/node-js-recommended-max-old-space-size
-    const envCopy = Object.assign({}, process.env);
-    if (backupConfig.memoryLimit && backupConfig.memoryLimit >= 2*1024*1024*1024) {
-        const heapSize = Math.min((backupConfig.memoryLimit/1024/1024) - 256, 8192);
-        debug(`runBackupUpload: adjusting heap size to ${heapSize}M`);
-        envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`;
-    }
-
-    let result = ''; // the script communicates error result as a string
-    function onMessage(progress) { // this is { message } or { result }
-        if ('message' in progress) return progressCallback({ message: `${progress.message} (${progressTag})` });
-        debug(`runBackupUpload: result - ${JSON.stringify(progress)}`);
-        result = progress.result;
-    }
-
-    const [error] = await safe(shell.promises.sudo(`backup-${remotePath}`, [ BACKUP_UPLOAD_CMD, remotePath, backupConfig.format, dataLayout.toString() ], { env: envCopy, preserveEnv: true, ipc: true, onMessage }));
-    if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
-        throw new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed');
-    } else if (error && error.code === 50) { // exited with error
-        throw new BoxError(BoxError.EXTERNAL_ERROR, result);
-    }
-}
-
-async function snapshotBox(progressCallback) {
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    progressCallback({ message: 'Snapshotting box' });
-
-    const startTime = new Date();
-
-    await database.exportToFile(`${paths.BOX_DATA_DIR}/box.mysqldump`);
-    debug(`snapshotBox: took ${(new Date() - startTime)/1000} seconds`);
-}
-
-async function uploadBoxSnapshot(backupConfig, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    await snapshotBox(progressCallback);
-
-    const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
-    if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
-
-    const uploadConfig = {
-        remotePath: 'snapshot/box',
-        backupConfig,
-        dataLayout: new DataLayout(boxDataDir, []),
-        progressTag: 'box'
-    };
-
-    progressCallback({ message: 'Uploading box snapshot' });
-
-    const startTime = new Date();
-
-    await runBackupUpload(uploadConfig, progressCallback);
-
-    debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`);
-
-    await backups.setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format });
-}
-
-async function copy(backupConfig, srcRemotePath, destRemotePath, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof srcRemotePath, 'string');
-    assert.strictEqual(typeof destRemotePath, 'string');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const { provider, format } = backupConfig;
-    const oldFilePath = backupFormat.api(format).getBackupFilePath(backupConfig, srcRemotePath);
-    const newFilePath = backupFormat.api(format).getBackupFilePath(backupConfig, destRemotePath);
-
-    const startTime = new Date();
-    await safe(storage.api(provider).copy(backupConfig, oldFilePath, newFilePath, progressCallback));
-    debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
-}
-
-async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof tag, 'string');
-    assert.strictEqual(typeof options, 'object');
-    assert(Array.isArray(dependsOn));
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const remotePath = `${tag}/box_v${constants.VERSION}`;
-    const format = backupConfig.format;
-
-    debug(`rotateBoxBackup: rotating to id ${remotePath}`);
-
-    const data = {
-        remotePath,
-        encryptionVersion: backupConfig.encryption ? 2 : null,
-        packageVersion: constants.VERSION,
-        type: backups.BACKUP_TYPE_BOX,
-        state: backups.BACKUP_STATE_CREATING,
-        identifier: backups.BACKUP_IDENTIFIER_BOX,
-        dependsOn,
-        manifest: null,
-        format,
-        preserveSecs: options.preserveSecs || 0
-    };
-
-    const id = await backups.add(data);
-    const [error] = await safe(copy(backupConfig, 'snapshot/box', remotePath, progressCallback));
-    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
-    await backups.setState(id, state);
-    if (error) throw error;
-
-    return id;
-}
-
-async function backupBox(dependsOn, tag, options, progressCallback) {
-    assert(Array.isArray(dependsOn));
-    assert.strictEqual(typeof tag, 'string');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const backupConfig = await settings.getBackupConfig();
-
-    await uploadBoxSnapshot(backupConfig, progressCallback);
-    return await rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback);
-}
-
-async function rotateAppBackup(backupConfig, app, tag, options, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof tag, 'string');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const snapshotInfo = backups.getSnapshotInfo(app.id);
-
-    const manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat
-    const remotePath = `${tag}/app_${app.fqdn}_v${manifest.version}`;
-    const format = backupConfig.format;
-
-    debug(`rotateAppBackup: rotating ${app.fqdn} to path ${remotePath}`);
-
-    const data = {
-        remotePath,
-        encryptionVersion: backupConfig.encryption ? 2 : null,
-        packageVersion: manifest.version,
-        type: backups.BACKUP_TYPE_APP,
-        state: backups.BACKUP_STATE_CREATING,
-        identifier: app.id,
-        dependsOn: [],
-        manifest,
-        format,
-        preserveSecs: options.preserveSecs || 0
-    };
-
-    const id = await backups.add(data);
-    const [error] = await safe(copy(backupConfig,  `snapshot/app_${app.id}`, remotePath, progressCallback));
-    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
-    await backups.setState(id, state);
-    if (error) throw error;
-
-    return id;
-}
-
-async function backupApp(app, options, progressCallback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    if (options.snapshotOnly) return await snapshotApp(app, progressCallback);
-
-    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
-
-    debug(`backupApp: backing up ${app.fqdn} with tag ${tag}`);
-
-    return await backupAppWithTag(app, tag, options, progressCallback);
-}
-
-async function snapshotApp(app, progressCallback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const startTime = new Date();
-    progressCallback({ message: `Snapshotting app ${app.fqdn}` });
-
-    await apps.backupConfig(app);
-    await services.backupAddons(app, app.manifest.addons);
-
-    debug(`snapshotApp: ${app.fqdn} took ${(new Date() - startTime)/1000} seconds`);
-}
-
-async function uploadAppSnapshot(backupConfig, app, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    await snapshotApp(app, progressCallback);
-
-    const remotePath = `snapshot/app_${app.id}`;
-    const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
-    if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`);
-
-    const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []);
-
-    progressCallback({ message: `Uploading app snapshot ${app.fqdn}`});
-
-    const uploadConfig = {
-        remotePath,
-        backupConfig,
-        dataLayout,
-        progressTag: app.fqdn
-    };
-
-    const startTime = new Date();
-
-    await runBackupUpload(uploadConfig, progressCallback);
-
-    debug(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`);
-
-    await backups.setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), manifest: app.manifest, format: backupConfig.format });
-}
-
-async function backupAppWithTag(app, tag, options, progressCallback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof tag, 'string');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    if (!canBackupApp(app)) { // if we cannot backup, reuse it's most recent backup
-        const results = await backups.getByIdentifierAndStatePaged(app.id, backups.BACKUP_STATE_NORMAL, 1, 1);
-        if (results.length === 0) return null; // no backup to re-use
-
-        return results[0].id;
-    }
-
-    const backupConfig = await settings.getBackupConfig();
-
-    await uploadAppSnapshot(backupConfig, app, progressCallback);
-    return await rotateAppBackup(backupConfig, app, tag, options, progressCallback);
-}
-
-async function uploadMailSnapshot(backupConfig, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
-    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);
-
-    const uploadConfig = {
-        remotePath: 'snapshot/mail',
-        backupConfig,
-        dataLayout: new DataLayout(mailDataDir, []),
-        progressTag: 'mail'
-    };
-
-    progressCallback({ message: 'Uploading mail snapshot' });
-
-    const startTime = new Date();
-
-    await runBackupUpload(uploadConfig, progressCallback);
-
-    debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`);
-
-    await backups.setSnapshotInfo('mail', { timestamp: new Date().toISOString(), format: backupConfig.format });
-}
-
-async function rotateMailBackup(backupConfig, tag, options, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
-    assert.strictEqual(typeof tag, 'string');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const remotePath = `${tag}/mail_v${constants.VERSION}`;
-    const format = backupConfig.format;
-
-    debug(`rotateMailBackup: rotating to ${remotePath}`);
-
-    const data = {
-        remotePath,
-        encryptionVersion: backupConfig.encryption ? 2 : null,
-        packageVersion: constants.VERSION,
-        type: backups.BACKUP_TYPE_MAIL,
-        state: backups.BACKUP_STATE_CREATING,
-        identifier: backups.BACKUP_IDENTIFIER_MAIL,
-        dependsOn: [],
-        manifest: null,
-        format,
-        preserveSecs: options.preserveSecs || 0
-    };
-
-    const id = await backups.add(data);
-    const [error] = await safe(copy(backupConfig, 'snapshot/mail', remotePath, progressCallback));
-    const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
-    await backups.setState(id, state);
-    if (error) throw error;
-
-    return id;
-}
-
-async function backupMailWithTag(tag, options, progressCallback) {
-    assert.strictEqual(typeof tag, 'string');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    debug(`backupMailWithTag: backing up mail with tag ${tag}`);
-
-    const backupConfig = await settings.getBackupConfig();
-    await uploadMailSnapshot(backupConfig, progressCallback);
-    return await rotateMailBackup(backupConfig, tag, options, progressCallback);
-}
-
-async function backupMail(options, progressCallback) {
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
-    debug(`backupMail: backing up mail with tag ${tag}`);
-    return await backupMailWithTag(tag, options, progressCallback);
-}
-
-async function downloadMail(restoreConfig, progressCallback) {
-    assert.strictEqual(typeof restoreConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
-    if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);
-    const dataLayout = new DataLayout(mailDataDir, []);
-
-    const startTime = new Date();
-
-    await download(restoreConfig.backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
-    debug('downloadMail: time: %s', (new Date() - startTime)/1000);
-}
-
-// this function is called from external process. calling process is expected to have a lock
-async function fullBackup(options, progressCallback) {
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-
-    const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); // unique tag under which all apps/mail/box backs up
-
-    const allApps = await apps.list();
-
-    let percent = 1;
-    let step = 100/(allApps.length+3);
-
-    const appBackupIds = [];
-    for (let i = 0; i < allApps.length; i++) {
-        const app = allApps[i];
-        progressCallback({ percent: percent, message: `Backing up ${app.fqdn} (${i+1}/${allApps.length})` });
-        percent += step;
-
-        if (!app.enableBackup) {
-            debug(`fullBackup: skipped backup ${app.fqdn} (${i+1}/${allApps.length}) since automatic backup disabled`);
-            continue; // nothing to backup
-        }
-
-        const startTime = new Date();
-        const appBackupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
-        debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
-        if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up
-    }
-
-    progressCallback({ percent, message: 'Backing up mail' });
-    percent += step;
-    const mailBackupId = await backupMailWithTag(tag, options, (progress) => progressCallback({ percent, message: progress.message }));
-
-    progressCallback({ percent, message: 'Backing up system data' });
-    percent += step;
-
-    const dependsOn = appBackupIds.concat(mailBackupId);
-    const backupId = await backupBox(dependsOn, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
-    return backupId;
-}

src/blobs.js

@@ -1,64 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-exports = module.exports = {
-    get,
-    getString,
-    set,
-    setString,
-    del,
-
-    ACME_ACCOUNT_KEY: 'acme_account_key',
-    ADDON_TURN_SECRET: 'addon_turn_secret',
-    SFTP_PUBLIC_KEY: 'sftp_public_key',
-    SFTP_PRIVATE_KEY: 'sftp_private_key',
-    PROXY_AUTH_TOKEN_SECRET: 'proxy_auth_token_secret',
-
-    CERT_PREFIX: 'cert',
-
-    _clear: clear
-};
-
-const assert = require('assert'),
-    database = require('./database.js');
-
-const BLOBS_FIELDS = [ 'id', 'value' ].join(',');
-
-async function get(id) {
-    assert.strictEqual(typeof id, 'string');
-
-    const result = await database.query(`SELECT ${BLOBS_FIELDS} FROM blobs WHERE id = ?`, [ id ]);
-    if (result.length === 0) return null;
-    return result[0].value;
-}
-
-async function getString(id) {
-    assert.strictEqual(typeof id, 'string');
-
-    const result = await database.query(`SELECT ${BLOBS_FIELDS} FROM blobs WHERE id = ?`, [ id ]);
-    if (result.length === 0) return null;
-    return result[0].value.toString('utf8');
-}
-
-async function set(id, value) {
-    assert.strictEqual(typeof id, 'string');
-    assert(value === null || Buffer.isBuffer(value));
-
-    await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, value ]);
-}
-
-async function setString(id, value) {
-    assert.strictEqual(typeof id, 'string');
-    assert(value === null || typeof value === 'string');
-
-    await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, Buffer.from(value) ]);
-}
-
-async function del(id) {
-    await database.query('DELETE FROM blobs WHERE id=?', [ id ]);
-}
-
-async function clear() {
-    await database.query('DELETE FROM blobs');
-}

src/boxerror.js

@@ -9,17 +9,17 @@ const assert = require('assert'),
 
 exports = module.exports = BoxError;
 
-function BoxError(reason, errorOrMessage, override) {
+function BoxError(reason, errorOrMessage, details) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
-    assert(typeof override === 'object' || typeof override === 'undefined');
+    assert(typeof details === 'object' || typeof details === 'undefined');
 
     Error.call(this);
     Error.captureStackTrace(this, this.constructor);
 
     this.name = this.constructor.name;
     this.reason = reason;
-    this.details = {};
+    this.details = details || {};
 
     if (typeof errorOrMessage === 'undefined') {
         this.message = reason;
@@ -28,12 +28,11 @@ function BoxError(reason, errorOrMessage, override) {
     } else { // error object
         this.message = errorOrMessage.message;
         this.nestedError = errorOrMessage;
-        _.extend(this, override); // copy enumerable properies
+        _.extend(this.details, errorOrMessage); // copy enumerable properies
     }
 }
 util.inherits(BoxError, Error);
 BoxError.ACCESS_DENIED = 'Access Denied';
-BoxError.ACME_ERROR = 'Acme Error';
 BoxError.ADDONS_ERROR = 'Addons Error';
 BoxError.ALREADY_EXISTS = 'Already Exists';
 BoxError.BAD_FIELD = 'Bad Field';
@@ -48,20 +47,18 @@ BoxError.DOCKER_ERROR = 'Docker Error';
 BoxError.EXTERNAL_ERROR = 'External Error'; // use this for external API errors
 BoxError.FEATURE_DISABLED = 'Feature Disabled';
 BoxError.FS_ERROR = 'FileSystem Error';
-BoxError.INACTIVE = 'Inactive'; // service/volume/mount
+BoxError.INACTIVE = 'Inactive';
 BoxError.INTERNAL_ERROR = 'Internal Error';
 BoxError.INVALID_CREDENTIALS = 'Invalid Credentials';
 BoxError.IPTABLES_ERROR = 'IPTables Error';
-BoxError.LICENSE_ERROR = 'License Error'; // billing or subscription expired
+BoxError.LICENSE_ERROR = 'License Error';
 BoxError.LOGROTATE_ERROR = 'Logrotate Error';
 BoxError.MAIL_ERROR = 'Mail Error';
-BoxError.MOUNT_ERROR = 'Mount Error';
 BoxError.NETWORK_ERROR = 'Network Error';
 BoxError.NGINX_ERROR = 'Nginx Error';
 BoxError.NOT_FOUND = 'Not found';
 BoxError.NOT_IMPLEMENTED = 'Not implemented';
 BoxError.NOT_SIGNED = 'Not Signed';
-BoxError.NOT_SUPPORTED = 'Not Supported';
 BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.PLAN_LIMIT = 'Plan Limit';
 BoxError.SPAWN_ERROR = 'Spawn Error';
@@ -87,15 +84,12 @@ BoxError.toHttpError = function (error) {
     case BoxError.ALREADY_EXISTS:
     case BoxError.BAD_STATE:
     case BoxError.CONFLICT:
-    case BoxError.NOT_SUPPORTED:
         return new HttpError(409, error);
     case BoxError.INVALID_CREDENTIALS:
         return new HttpError(412, error);
     case BoxError.EXTERNAL_ERROR:
-    case BoxError.ACME_ERROR:
     case BoxError.NETWORK_ERROR:
     case BoxError.FS_ERROR:
-    case BoxError.MOUNT_ERROR:
     case BoxError.MAIL_ERROR:
     case BoxError.DOCKER_ERROR:
     case BoxError.ADDONS_ERROR:

src/cert/acme2.js

@@ -0,0 +1,630 @@
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    BoxError = require('../boxerror.js'),
+    crypto = require('crypto'),
+    debug = require('debug')('box:cert/acme2'),
+    domains = require('../domains.js'),
+    fs = require('fs'),
+    path = require('path'),
+    paths = require('../paths.js'),
+    request = require('request'),
+    safe = require('safetydance'),
+    util = require('util'),
+    _ = require('underscore');
+
+const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
+    CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory';
+
+exports = module.exports = {
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'acme',
+    _getChallengeSubdomain: getChallengeSubdomain
+};
+
+// http://jose.readthedocs.org/en/latest/
+// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
+// https://community.letsencrypt.org/t/list-of-client-implementations/2103
+
+function Acme2(options) {
+    assert.strictEqual(typeof options, 'object');
+
+    this.accountKeyPem = null; // Buffer
+    this.email = options.email;
+    this.keyId = null;
+    this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
+    this.directory = {};
+    this.performHttpAuthorization = !!options.performHttpAuthorization;
+    this.wildcard = !!options.wildcard;
+}
+
+// urlsafe base64 encoding (jose)
+function urlBase64Encode(string) {
+    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
+}
+
+function b64(str) {
+    var buf = util.isBuffer(str) ? str : Buffer.from(str);
+    return urlBase64Encode(buf.toString('base64'));
+}
+
+function getModulus(pem) {
+    assert(util.isBuffer(pem));
+
+    var stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
+    if (!stdout) return null;
+    var match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
+    if (!match) return null;
+    return Buffer.from(match[1], 'hex');
+}
+
+Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof payload, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    assert(util.isBuffer(this.accountKeyPem));
+
+    const that = this;
+    let header = {
+        url: url,
+        alg: 'RS256'
+    };
+
+    // keyId is null when registering account
+    if (this.keyId) {
+        header.kid = this.keyId;
+    } else {
+        header.jwk = {
+            e: b64(Buffer.from([0x01, 0x00, 0x01])), // exponent - 65537
+            kty: 'RSA',
+            n: b64(getModulus(this.accountKeyPem))
+        };
+    }
+
+    var payload64 = b64(payload);
+
+    request.get(this.directory.newNonce, { json: true, timeout: 30000 }, function (error, response) {
+        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`));
+        if (response.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching nonce : ' + response.statusCode));
+
+        const nonce = response.headers['Replay-Nonce'.toLowerCase()];
+        if (!nonce) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response'));
+
+        debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
+
+        var protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
+
+        var signer = crypto.createSign('RSA-SHA256');
+        signer.update(protected64 + '.' + payload64, 'utf8');
+        var signature64 = urlBase64Encode(signer.sign(that.accountKeyPem, 'base64'));
+
+        var data = {
+            protected: protected64,
+            payload: payload64,
+            signature: signature64
+        };
+
+        request.post(url, { headers: { 'Content-Type': 'application/jose+json', 'User-Agent': 'acme-cloudron' }, body: JSON.stringify(data), timeout: 30000 }, function (error, response) {
+            if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`)); // network error
+
+            // we don't set json: true in request because it ends up mangling the content-type
+            // we don't set json: true in request because it ends up mangling the content-type
+            if (response.headers['content-type'] === 'application/json') response.body = safe.JSON.parse(response.body);
+
+            callback(null, response);
+        });
+    });
+};
+
+// https://tools.ietf.org/html/rfc8555#section-6.3
+Acme2.prototype.postAsGet = function (url, callback) {
+    this.sendSignedRequest(url, '', callback);
+};
+
+Acme2.prototype.updateContact = function (registrationUri, callback) {
+    assert.strictEqual(typeof registrationUri, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
+
+    // https://github.com/ietf-wg-acme/acme/issues/30
+    const payload = {
+        contact: [ 'mailto:' + this.email ]
+    };
+
+    const that = this;
+    this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(error);
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+
+        debug(`updateContact: contact of user updated to ${that.email}`);
+
+        callback();
+    });
+};
+
+Acme2.prototype.registerUser = function (callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        termsOfServiceAgreed: true
+    };
+
+    debug('registerUser: registering user');
+
+    var that = this;
+    this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(error);
+        // 200 if already exists. 201 for new accounts
+        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+
+        debug(`registerUser: user registered keyid: ${result.headers.location}`);
+
+        that.keyId = result.headers.location;
+
+        that.updateContact(result.headers.location, callback);
+    });
+};
+
+Acme2.prototype.newOrder = function (domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        identifiers: [{
+            type: 'dns',
+            value: domain
+        }]
+    };
+
+    debug('newOrder: %s', domain);
+
+    this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(error);
+        if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+
+        debug('newOrder: created order %s %j', domain, result.body);
+
+        const order = result.body, orderUrl = result.headers.location;
+
+        if (!Array.isArray(order.authorizations)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid authorizations in order'));
+        if (typeof order.finalize !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid finalize in order'));
+        if (typeof orderUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid order location in order header'));
+
+        callback(null, order, orderUrl);
+    });
+};
+
+Acme2.prototype.waitForOrder = function (orderUrl, callback) {
+    assert.strictEqual(typeof orderUrl, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`waitForOrder: ${orderUrl}`);
+    const that = this;
+
+    async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
+        debug('waitForOrder: getting status');
+
+        that.postAsGet(orderUrl, function (error, result) {
+            if (error) {
+                debug('waitForOrder: network error getting uri %s', orderUrl);
+                return retryCallback(error);
+            }
+            if (result.statusCode !== 200) {
+                debug('waitForOrder: invalid response code getting uri %s', result.statusCode);
+                return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
+            }
+
+            debug('waitForOrder: status is "%s %j', result.body.status, result.body);
+
+            if (result.body.status === 'pending' || result.body.status === 'processing') return retryCallback(new BoxError(BoxError.TRY_AGAIN, `Request is in ${result.body.status} state`));
+            else if (result.body.status === 'valid' && result.body.certificate) return retryCallback(null, result.body.certificate);
+            else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status or invalid response: ' + result.body));
+        });
+    }, callback);
+};
+
+Acme2.prototype.getKeyAuthorization = function (token) {
+    assert(util.isBuffer(this.accountKeyPem));
+
+    let jwk = {
+        e: b64(Buffer.from([0x01, 0x00, 0x01])), // Exponent - 65537
+        kty: 'RSA',
+        n: b64(getModulus(this.accountKeyPem))
+    };
+
+    let shasum = crypto.createHash('sha256');
+    shasum.update(JSON.stringify(jwk));
+    let thumbprint = urlBase64Encode(shasum.digest('base64'));
+    return token + '.' + thumbprint;
+};
+
+Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token }
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('notifyChallengeReady: %s was met', challenge.url);
+
+    const keyAuthorization = this.getKeyAuthorization(challenge.token);
+
+    var payload = {
+        resource: 'challenge',
+        keyAuthorization: keyAuthorization
+    };
+
+    this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(error);
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+
+        callback();
+    });
+};
+
+Acme2.prototype.waitForChallenge = function (challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('waitingForChallenge: %j', challenge);
+    const that = this;
+
+    async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
+        debug('waitingForChallenge: getting status');
+
+        that.postAsGet(challenge.url, function (error, result) {
+            if (error) {
+                debug('waitForChallenge: network error getting uri %s', challenge.url);
+                return retryCallback(error);
+            }
+            if (result.statusCode !== 200) {
+                debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
+                return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
+            }
+
+            debug('waitForChallenge: status is "%s" %j', result.body.status, result.body);
+
+            if (result.body.status === 'pending') return retryCallback(new BoxError(BoxError.TRY_AGAIN));
+            else if (result.body.status === 'valid') return retryCallback();
+            else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
+        });
+    }, function retryFinished(error) {
+        // async.retry will pass 'undefined' as second arg making it unusable with async.waterfall()
+        callback(error);
+    });
+};
+
+// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
+Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof finalizationUrl, 'string');
+    assert(util.isBuffer(csrDer));
+    assert.strictEqual(typeof callback, 'function');
+
+    const payload = {
+        csr: b64(csrDer)
+    };
+
+    debug('signCertificate: sending sign request');
+
+    this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(error);
+        // 429 means we reached the cert limit for this domain
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+
+        return callback(null);
+    });
+};
+
+Acme2.prototype.createKeyAndCsr = function (hostname, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var outdir = paths.APP_CERTS_DIR;
+    const certName = hostname.replace('*.', '_.');
+    var csrFile = path.join(outdir, `${certName}.csr`);
+    var privateKeyFile = path.join(outdir, `${certName}.key`);
+
+    if (safe.fs.existsSync(privateKeyFile)) {
+        // in some old releases, csr file was corrupt. so always regenerate it
+        debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
+    } else {
+        var key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
+        if (!key) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
+        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error));
+
+        debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
+    }
+
+    var csrDer = safe.child_process.execSync(`openssl req -new -key ${privateKeyFile} -outform DER -subj /CN=${hostname}`);
+    if (!csrDer) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
+    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new BoxError(BoxError.FS_ERROR, safe.error)); // bookkeeping
+
+    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
+
+    callback(null, csrDer);
+};
+
+Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof certUrl, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var outdir = paths.APP_CERTS_DIR;
+    const that = this;
+
+    async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
+        debug('downloadCertificate: downloading certificate');
+
+        that.postAsGet(certUrl, function (error, result) {
+            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
+            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
+            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+
+            const fullChainPem = result.body; // buffer
+
+            const certName = hostname.replace('*.', '_.');
+            var certificateFile = path.join(outdir, `${certName}.cert`);
+            if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return retryCallback(new BoxError(BoxError.FS_ERROR, safe.error));
+
+            debug('downloadCertificate: cert file for %s saved at %s', hostname, certificateFile);
+
+            retryCallback(null);
+        });
+    }, callback);
+};
+
+Acme2.prototype.prepareHttpChallenge = function (hostname, domain, authorization, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof authorization, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('acmeFlow: challenges: %j', authorization);
+    let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
+    if (httpChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no http challenges'));
+    let challenge = httpChallenges[0];
+
+    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
+
+    let keyAuthorization = this.getKeyAuthorization(challenge.token);
+
+    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
+
+    fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), keyAuthorization, function (error) {
+        if (error) return callback(new BoxError(BoxError.FS_ERROR, error));
+
+        callback(null, challenge);
+    });
+};
+
+Acme2.prototype.cleanupHttpChallenge = function (hostname, domain, challenge, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('cleanupHttpChallenge: unlinking %s', path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
+
+    fs.unlink(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), callback);
+};
+
+function getChallengeSubdomain(hostname, domain) {
+    let challengeSubdomain;
+
+    if (hostname === domain) {
+        challengeSubdomain = '_acme-challenge';
+    } else if (hostname.includes('*')) { // wildcard
+        let subdomain = hostname.slice(0, -domain.length - 1);
+        challengeSubdomain = subdomain ? subdomain.replace('*', '_acme-challenge') : '_acme-challenge';
+    } else {
+        challengeSubdomain = '_acme-challenge.' + hostname.slice(0, -domain.length - 1);
+    }
+
+    debug(`getChallengeSubdomain: challenge subdomain for hostname ${hostname} at domain ${domain} is ${challengeSubdomain}`);
+
+    return challengeSubdomain;
+}
+
+Acme2.prototype.prepareDnsChallenge = function (hostname, domain, authorization, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof authorization, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('acmeFlow: challenges: %j', authorization);
+    let dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
+    if (dnsChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no dns challenges'));
+    let challenge = dnsChallenges[0];
+
+    const keyAuthorization = this.getKeyAuthorization(challenge.token);
+    let shasum = crypto.createHash('sha256');
+    shasum.update(keyAuthorization);
+
+    const txtValue = urlBase64Encode(shasum.digest('base64'));
+    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
+
+    debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
+
+    domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
+        if (error) return callback(error);
+
+        domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 }, function (error) {
+            if (error) return callback(error);
+
+            callback(null, challenge);
+        });
+    });
+};
+
+Acme2.prototype.cleanupDnsChallenge = function (hostname, domain, challenge, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const keyAuthorization = this.getKeyAuthorization(challenge.token);
+    let shasum = crypto.createHash('sha256');
+    shasum.update(keyAuthorization);
+
+    const txtValue = urlBase64Encode(shasum.digest('base64'));
+    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
+
+    debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
+
+    domains.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
+        if (error) return callback(error);
+
+        callback(null);
+    });
+};
+
+Acme2.prototype.prepareChallenge = function (hostname, domain, authorizationUrl, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof authorizationUrl, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
+
+    const that = this;
+    this.postAsGet(authorizationUrl, function (error, response) {
+        if (error) return callback(error);
+        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code getting authorization : ' + response.statusCode));
+
+        const authorization = response.body;
+
+        if (that.performHttpAuthorization) {
+            that.prepareHttpChallenge(hostname, domain, authorization, callback);
+        } else {
+            that.prepareDnsChallenge(hostname, domain, authorization, callback);
+        }
+    });
+};
+
+Acme2.prototype.cleanupChallenge = function (hostname, domain, challenge, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`cleanupChallenge: http: ${this.performHttpAuthorization}`);
+
+    if (this.performHttpAuthorization) {
+        this.cleanupHttpChallenge(hostname, domain, challenge, callback);
+    } else {
+        this.cleanupDnsChallenge(hostname, domain, challenge, callback);
+    }
+};
+
+Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
+        debug('getCertificate: generating acme account key on first run');
+        this.accountKeyPem = safe.child_process.execSync('openssl genrsa 4096');
+        if (!this.accountKeyPem) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
+
+        safe.fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, this.accountKeyPem);
+    } else {
+        debug('getCertificate: using existing acme account key');
+        this.accountKeyPem = fs.readFileSync(paths.ACME_ACCOUNT_KEY_FILE);
+    }
+
+    var that = this;
+    this.registerUser(function (error) {
+        if (error) return callback(error);
+
+        that.newOrder(hostname, function (error, order, orderUrl) {
+            if (error) return callback(error);
+
+            async.eachSeries(order.authorizations, function (authorizationUrl, iteratorCallback) {
+                debug(`acmeFlow: authorizing ${authorizationUrl}`);
+
+                that.prepareChallenge(hostname, domain, authorizationUrl, function (error, challenge) {
+                    if (error) return iteratorCallback(error);
+
+                    async.waterfall([
+                        that.notifyChallengeReady.bind(that, challenge),
+                        that.waitForChallenge.bind(that, challenge),
+                        that.createKeyAndCsr.bind(that, hostname),
+                        that.signCertificate.bind(that, hostname, order.finalize),
+                        that.waitForOrder.bind(that, orderUrl),
+                        that.downloadCertificate.bind(that, hostname)
+                    ], function (error) {
+                        that.cleanupChallenge(hostname, domain, challenge, function (cleanupError) {
+                            if (cleanupError) debug('acmeFlow: ignoring error when cleaning up challenge:', cleanupError);
+
+                            iteratorCallback(error);
+                        });
+                    });
+                });
+            }, callback);
+        });
+    });
+};
+
+Acme2.prototype.getDirectory = function (callback) {
+    const that = this;
+
+    async.retry({ times: 3, interval: 20000 }, function (retryCallback) {
+        request.get(that.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
+            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
+            if (response.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
+
+            if (typeof response.body.newNonce !== 'string' ||
+                typeof response.body.newOrder !== 'string' ||
+                typeof response.body.newAccount !== 'string') return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));
+
+            that.directory = response.body;
+
+            retryCallback(null);
+        });
+    }, callback);
+};
+
+Acme2.prototype.getCertificate = function (vhost, domain, callback) {
+    assert.strictEqual(typeof vhost, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
+
+    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
+        vhost = domains.makeWildcard(vhost);
+        debug(`getCertificate: will get wildcard cert for ${vhost}`);
+    }
+
+    const that = this;
+    this.getDirectory(function (error) {
+        if (error) return callback(error);
+
+        that.acmeFlow(vhost, domain, function (error) {
+            if (error) return callback(error);
+
+            var outdir = paths.APP_CERTS_DIR;
+            const certName = vhost.replace('*.', '_.');
+            callback(null, path.join(outdir, `${certName}.cert`), path.join(outdir, `${certName}.key`));
+        });
+    });
+};
+
+function getCertificate(vhost, domain, options, callback) {
+    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    let attempt = 1;
+    async.retry({ times: 3, interval: 0 }, function (retryCallback) {
+        debug(`getCertificate: attempt ${attempt++}`);
+
+        let acme = new Acme2(options || { });
+        acme.getCertificate(vhost, domain, retryCallback);
+    }, callback);
+}

src/changelog.js

@@ -1,17 +1,17 @@
 'use strict';
 
-const assert = require('assert'),
+let assert = require('assert'),
     fs = require('fs'),
     path = require('path');
 
 exports = module.exports = {
-    getChanges
+    getChanges: getChanges
 };
 
 function getChanges(version) {
     assert.strictEqual(typeof version, 'string');
 
-    const changelog = [];
+    let changelog = [ ];
     const lines = fs.readFileSync(path.join(__dirname, '../CHANGES'), 'utf8').split('\n');
 
     version = version.replace(/[+-].*/, ''); // strip prerelease