make app password work with sftp

(cherry picked from commit 9faae96d61) (cherry picked from commit f3b979f112)
5.0.6 changelog
2020-03-26 21:56:45 -07:00 · 2020-03-26 19:30:52 -07:00 · 2020-03-26 19:08:43 -07:00 · 2020-03-25 07:57:39 -07:00
224 changed files with 7389 additions and 12328 deletions
@@ -1854,340 +1854,3 @@
 * Make mail eventlog only visible to owners
 * Make app password work with sftp

-[5.1.0]
-* Add turn addon
-* Fix disk usage display
-* Drop support for TLSv1 and TLSv1.1
-* Make cert validation work for ECC certs
-* Add type filter to mail eventlog
-* mail: Fix listing of mailboxes and aliases in the UI
-* branding: fix login page title
-* Only a Cloudron owner can install/update/exec apps with the docker addon
-* security: reset tokens are only valid for a day
-* mail: fix eventlog db perms
-* Fix various bugs in the disk graphs
-
-[5.1.1]
-* Add turn addon
-* Fix disk usage display
-* Drop support for TLSv1 and TLSv1.1
-* Make cert validation work for ECC certs
-* Add type filter to mail eventlog
-* mail: Fix listing of mailboxes and aliases in the UI
-* branding: fix login page title
-* Only a Cloudron owner can install/update/exec apps with the docker addon
-* security: reset tokens are only valid for a day
-* mail: fix eventlog db perms
-* Fix various bugs in the disk graphs
-* Fix collectd installation
-* graphs: sort disk contents by usage
-* backups: show apps that are not automatically backed up in backup view
-
-[5.1.2]
-* Add turn addon
-* Fix disk usage display
-* Drop support for TLSv1 and TLSv1.1
-* Make cert validation work for ECC certs
-* Add type filter to mail eventlog
-* mail: Fix listing of mailboxes and aliases in the UI
-* branding: fix login page title
-* Only a Cloudron owner can install/update/exec apps with the docker addon
-* security: reset tokens are only valid for a day
-* mail: fix eventlog db perms
-* Fix various bugs in the disk graphs
-* Fix collectd installation
-* graphs: sort disk contents by usage
-* backups: show apps that are not automatically backed up in backup view
-* turn: deny local address peers https://www.rtcsec.com/2020/04/01-slack-webrtc-turn-compromise/
-
-[5.1.3]
-* Fix crash with misconfigured reverse proxy
-* Fix issue where invitation links are not working anymore
-
-[5.1.4]
-* Add support for custom .well-known documents to be served
-* Add ECDHE-RSA-AES128-SHA256 to cipher list
-* Fix GPG signature verification
-
-[5.1.5]
-* Check for .well-known routes upstream as fallback. This broke nextcloud's caldav/carddav
-
-[5.2.0]
-* acme: request ECC certs
-* less-strict DKIM check to allow users to set a stronger DKIM key
-* Add members only flag to mailing list
-* oauth: add backward compat layer for backup and uninstall
-* fix bug in disk usage sorting
-* mail: aliases can be across domains
-* mail: allow an external MX to be set
-* Add UI to download backup config as JSON (and import it)
-* Ensure stopped apps are getting backed up
-* Add OVH Object Storage backend
-* Add per-app redis status and configuration to Services
-* spam: large emails were not scanned
-* mail relay: fix delivery event log
-* manual update check always gets the latest updates
-* graphs: fix issue where large number of apps would crash the box code (query param limit exceeded)
-* backups: fix various security issues in encypted backups (thanks @mehdi)
-* graphs: add app graphs
-* older encrypted backups cannot be used in this version
-* Add backup listing UI
-* stopping an app will stop dependent services
-* Add new wasabi s3 storage region us-east-2
-* mail: Fix bug where SRS translation was done on the main domain instead of mailing list domain
-* backups: add retention policy
-* Drop `NET_RAW` caps from container preventing sniffing of network traffic
-
-[5.2.1]
-* Fix app disk graphs
-* restart apps on addon container change
-
-[5.2.2]
-* regression: import UI
-* Mbps -> MBps
-* Remove verbose logs
-* Set dmode in tar extract
-* mail: fix crash in audit logs
-* import: fix crash because encryption is unset
-* create redis with the correct label
-
-[5.2.3]
-* Do not restart stopped apps
-
-[5.2.4]
-* mail: enable/disable incoming mail was showing an error
-* Do not trigger backup of stopped apps. Instead, we will just retain it's existing backups
-  based on retention policy
-* remove broken disk graphs
-* fix OVH backups
-
-[5.3.0]
-* better nginx config for higher loads
-* backups: add CIFS storage provider
-* backups: add SSHFS storage provider
-* backups: add NFS storage provider
-* s3: use vhost style
-* Fix crash when redis config was set
-* Update schedule was unselected in the UI
-* cloudron-setup: --provider is now optional
-* show warning for unstable updates
-* add forumUrl to app manifest
-* postgresql: add unaccent extension for peertube
-* mail: Add Auto-Submitted header to NDRs
-* backups: ensure that the latest backup of installed apps is always preserved
-* add nginx logs
-* mail: make authentication case insensitive
-* Fix timeout issues in postgresql and mysql addon
-* Do not count stopped apps for memory use
-* LDAP group synchronization
-
-[5.3.1]
-* better nginx config for higher loads
-* backups: add CIFS storage provider
-* backups: add SSHFS storage provider
-* backups: add NFS storage provider
-* s3: use vhost style
-* Fix crash when redis config was set
-* Update schedule was unselected in the UI
-* cloudron-setup: --provider is now optional
-* show warning for unstable updates
-* add forumUrl to app manifest
-* postgresql: add unaccent extension for peertube
-* mail: Add Auto-Submitted header to NDRs
-* backups: ensure that the latest backup of installed apps is always preserved
-* add nginx logs
-* mail: make authentication case insensitive
-* Fix timeout issues in postgresql and mysql addon
-* Do not count stopped apps for memory use
-* LDAP group synchronization
-
-[5.3.2]
-* Do not install sshfs package
-* 'provider' is not required anymore in various API calls
-* redis: Set maxmemory and maxmemory-policy
-* Add mlock capability to manifest (for vault app)
-
-[5.3.3]
-* Fix issue where some postinstall messages where causing angular to infinite loop
-
-[5.3.4]
-* Fix issue in database error handling
-
-[5.4.0]
-* Update nginx to 1.18 for various security fixes
-* Add ping capability (for statping app)
-* Fix bug where aliases were displayed incorrectly in SOGo
-* Add univention as LDAP provider
-* Bump max_connection for postgres addon to 200
-* mail: Add pagination to mailing list API
-* Allow admin to lock email and display name of users
-* Allow admin to ensure all users have 2FA setup
-* ami: fix regression where we didn't send provider as part of get status call
-* nginx: hide version
-* backups: add b2 provider
-* Add filemanager webinterface
-* Add darkmode
-* Add note that password reset and invite links expire in 24 hours
-
-[5.4.1]
-* Update nginx to 1.18 for various security fixes
-* Add ping capability (for statping app)
-* Fix bug where aliases were displayed incorrectly in SOGo
-* Add univention as LDAP provider
-* Bump max_connection for postgres addon to 200
-* mail: Add pagination to mailing list API
-* Allow admin to lock email and display name of users
-* Allow admin to ensure all users have 2FA setup
-* ami: fix regression where we didn't send provider as part of get status call
-* nginx: hide version
-* backups: add b2 provider
-* Add filemanager webinterface
-* Add darkmode
-* Add note that password reset and invite links expire in 24 hours
-
-[5.5.0]
-* postgresql: update to PostgreSQL 11
-* postgresql: add citext extension to whitelist for loomio
-* postgresql: add btree_gist,postgres_fdw,pg_stat_statements,plpgsql extensions for gitlab
-* SFTP/Filebrowser: fix access of external data directories
-* Fix contrast issues in dark mode
-* Add option to delete mailbox data when mailbox is delete
-* Allow days/hours of backups and updates to be configurable
-* backup cleaner: fix issue where referenced backups where not counted against time periods
-* route53: fix issue where verification failed if user had more than 100 zones
-* rework task workers to run them in a separate cgroup
-* backups: now much faster thanks to reworking of task worker
-* When custom fallback cert is set, make sure it's used over LE certs
-* mongodb: update to MongoDB 4.0.19
-* List groups ordered by name
-* Invite links are now valid for a week
-* Update release GPG key
-* Add pre-defined variables ($CLOUDRON_APPID) for better post install messages
-* filemanager: show folder first
-
-[5.6.0]
-* Remove IP nginx configuration that redirects to dashboard after activation
-* dashboard: looks for search string in app title as well
-* Add vaapi caps for transcoding
-* Fix issue where the long mongodb database names where causing app indices of rocket.chat to overflow (> 127)
-* Do not resize swap if swap file exists. This means that users can now control how swap is allocated on their own.
-* SFTP: fix issue where parallel rebuilds would cause an error
-* backups: make part size configurable
-* mail: set max email size
-* mail: allow mail server location to be set
-* spamassassin: custom configs and wl/bl
-* Do not automatically update to unstable release
-* scheduler: reduce container churn
-* mail: add API to set banner
-* Fix bug where systemd 237 ignores --nice value in systemd-run
-* postgresql: enable uuid-ossp extension
-* firewall: add blocklist
-* HTTP URLs now redirect directly to the HTTPS of the final domain
-* linode: Add singapore region
-* ovh: add sydney region
-* s3: makes multi-part copies in parallel
-
-[5.6.1]
-* Blocklists are now stored in a text file instead of json
-* regenerate nginx configs
-
-[5.6.2]
-* Update docker to 19.03.12
-* Fix sorting of user listing in the UI
-* namecheap: fix crash when server returns invalid response
-* unlink ghost file automatically on successful login
-* Bump mysql addon connection limit to 200
-* Fix install issue where `/dev/dri` may not be present
-* import: when importing filesystem backups, the input box is a path
-* firewall: fix race condition where blocklist was not added in correct position in the FORWARD chain
-* services: fix issue where services where scaled up/down too fast
-* turn: realm variable was not updated properly on dashboard change
-* nginx: add splash pages for IP based browser access
-* Give services panel a separate top-level view
-* Add app state filter
-* gcs: copy concurrency was not used
-* Mention why an app update cannot be applied and provide shortcut to start the app if stopped
-* Remove version from footer into the setting view
-* Give services panel a separate top-level view
-* postgresql: set collation order explicity when creating database to C.UTF-8 (for confluence)
-* rsync: fix error while goes missing when syncing
-* Pre-select app domain by default in the redirection drop down
-* robots: preseve leading and trailing whitespaces/newlines
-
-[5.6.3]
-* Fix postgres locale issue
-
-[6.0.0]
-* Focal support
-* Reduce duration of self-signed certs to 800 days
-* Better backup config filename when downloading
-* branding: footer can have template variables like %YEAR% and %VERSION%
-* sftp: secure the API with a token
-* filemanager: Add extract context menu item
-* Do not download docker images if present locally
-* sftp: disable access to non-admins by default
-* postgresql: whitelist pgcrypto extension for loomio
-* filemanager: Add new file creation action and collapse new and upload actions
-* rsync: add warning to remove lifecycle rules
-* Add volume management
-* backups: adjust node's heap size based on memory limit
-* s3: diasble per-chunk timeout
-* logs: more descriptive log file names on download
-* collectd: remove collectd config when app stopped (and add it back when started)
-* Apps can optionally request an authwall to be installed in front of them
-* mailbox can now owned by a group
-* linode: enable dns provider in setup view
-* dns: apps can now use the dns port
-* httpPaths: allow apps to specify forwarding from custom paths to container ports (for OLS)
-* add elasticemail smtp relay option
-* mail: add option to fts using solr
-* mail: change the namespace separator of new installations to /
-* mail: enable acl
-* Disable THP
-* filemanager: allow download dirs as zip files
-* aws: add china region
-* security: fix issue where apps could send with any username (but valid password)
-* i18n support
-
-[6.0.1]
-* app: add export route
-* mail: on location change, fix lock up when one or more domains have invalid credentials
-* mail: fix crash because of write after timeout closure
-* scaleway: fix installation issue where THP is not enabled in kernel
-
-[6.1.0]
-* mail: update haraka to 2.8.27. this fixes zero-length queue file crash
-* update: set/unset appStoreId from the update route
-* proxyauth: Do not follow redirects
-* proxyauth: add 2FA
-* appstore: add category translations
-* appstore: add media category
-* prepend the version to assets when sourcing to avoid cache hits on update
-* filemanger: list volumes of the app
-* Display upload size and size progress
-* nfs: chown the backups for hardlinks to work
-* remove user add/remove/role change email notifications
-* persist update indicator across restarts
-* cloudron-setup: add --generate-setup-token
-* dashboard: pass accessToken query param to automatically login
-* wellknown: add a way to set well known docs
-* oom: notification mails have links to dashboard
-* collectd: do not install xorg* packages
-* apptask: backup/restore tasks now use the backup memory limit configuration
-* eventlog: add logout event
-* mailbox: include alias in mailbox search
-* proxyAuth: add path exclusion
-* turn: fix for CVE-2020-26262
-* app password: fix regression where apps are not listed anymore in the UI
-* Support for multiDomain apps (domain aliases)
-* netcup: add dns provider
-* Container swap size is now dynamically determined based on system RAM/swap ratio
-
-[6.1.1]
-* Fix bug where platform does not start if memory limits could not be applied
-
-[6.1.2]
-* App disk usage was not shown in graphs
-* Email autoconfig
-* Fix SOGo login
-
@@ -29,9 +29,9 @@ anyone to effortlessly host web applications on their server on their own terms.
 * Trivially migrate to another server keeping your apps and data (for example, switch your
  infrastructure provider or move to a bigger server).

-* Comprehensive [REST API](https://docs.cloudron.io/api/).
+* Comprehensive [REST API](https://cloudron.io/documentation/developer/api/).

-* [CLI](https://docs.cloudron.io/custom-apps/cli/) to configure apps.
+* [CLI](https://cloudron.io/documentation/cli/) to configure apps.

 * Alerts, audit logs, graphs, dns management ... and much more

@@ -41,37 +41,25 @@ Try our demo at https://my.demo.cloudron.io (username: cloudron password: cloudr

 ## Installing

-[Install script](https://docs.cloudron.io/installation/) - [Pricing](https://cloudron.io/pricing.html)
+[Install script](https://cloudron.io/documentation/installation/) - [Pricing](https://cloudron.io/pricing.html)

 **Note:** This repo is a small part of what gets installed on your server - there is
 the dashboard, database addons, graph container, base image etc. Cloudron also relies
 on external services such as the App Store for apps to be installed. As such, don't
 clone this repo and npm install and expect something to work.

-## Development
+## Documentation

-This is the backend code of Cloudron. The frontend code is [here](https://git.cloudron.io/cloudron/dashboard).
+* [Documentation](https://cloudron.io/documentation/)

-The way to develop is to first install a full instance of Cloudron in a VM. Then you can use the [hotfix](https://git.cloudron.io/cloudron/cloudron-machine)
-tool to patch the VM with the latest code.
+## Related repos

-```
-SSH_PASSPHRASE=sshkeypassword cloudron-machine hotfix --cloudron my.example.com --release 6.0.0 --ssh-key keyname
-```
+The [base image repo](https://git.cloudron.io/cloudron/docker-base-image) is the parent image of all
+the containers in the Cloudron.

-## License
+## Community

-Please note that the Cloudron code is under a source-available license. This is not the same as an
-open source license but ensures the code is available for introspection (and hacking!).
-
-## Contributions
-
-Just to give some heads up, we are a bit restrictive in merging changes. We are a small team and
-would like to keep our maintenance burden low. It might be best to discuss features first in the [forum](https://forum.cloudron.io),
-to also figure out how many other people will use it to justify maintenance for a feature.
-
-## Support
-
-* [Documentation](https://docs.cloudron.io/)
+* [Chat](https://chat.cloudron.io)
 * [Forum](https://forum.cloudron.io/)
+* [Support](mailto:support@cloudron.io)

@@ -4,7 +4,8 @@ set -euv -o pipefail

 readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

-readonly arg_infraversionpath="${SOURCE_DIR}/../src"
+readonly arg_provider="${1:-generic}"
+readonly arg_infraversionpath="${SOURCE_DIR}/${2:-}"

 function die {
    echo $1
@@ -13,9 +14,6 @@ function die {

 export DEBIAN_FRONTEND=noninteractive

-readonly ubuntu_codename=$(lsb_release -cs)
-readonly ubuntu_version=$(lsb_release -rs)
-
 # hold grub since updating it breaks on some VPS providers. also, dist-upgrade will trigger it
 apt-mark hold grub* >/dev/null
 apt-get -o Dpkg::Options::="--force-confdef" update -y
@@ -29,12 +27,11 @@ debconf-set-selections <<< 'mysql-server mysql-server/root_password_again passwo

 # this enables automatic security upgrades (https://help.ubuntu.com/community/AutomaticSecurityUpdates)
 # resolvconf is needed for unbound to work property after disabling systemd-resolved in 18.04
-
+ubuntu_version=$(lsb_release -rs)
+ubuntu_codename=$(lsb_release -cs)
 gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
-mysql_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "mysql-server-8.0" || echo "mysql-server-5.7")
-apt-get -y install --no-install-recommends \
+apt-get -y install \
    acl \
-    apparmor \
    build-essential \
    cifs-utils \
    cron \
@@ -42,12 +39,12 @@ apt-get -y install --no-install-recommends \
    debconf-utils \
    dmsetup \
    $gpg_package \
-    ipset \
    iptables \
    libpython2.7 \
    linux-generic \
    logrotate \
-    $mysql_package \
+    mysql-server-5.7 \
+    nginx-full \
    openssh-server \
    pwgen \
    resolvconf \
@@ -55,17 +52,10 @@ apt-get -y install --no-install-recommends \
    tzdata \
    unattended-upgrades \
    unbound \
-    unzip \
    xfsprogs

-echo "==> installing nginx for xenial for TLSv3 support"
-curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
-# apt install with install deps (as opposed to dpkg -i)
-apt install -y /tmp/nginx.deb
-rm /tmp/nginx.deb
-
 # on some providers like scaleway the sudo file is changed and we want to keep the old one
-apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends sudo
+apt-get -o Dpkg::Options::="--force-confold" install -y sudo

 # this ensures that unattended upgades are enabled, if it was disabled during ubuntu install time (see #346)
 # debconf-set-selection of unattended-upgrades/enable_auto_updates + dpkg-reconfigure does not work
@@ -73,10 +63,10 @@ cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upg

 echo "==> Installing node.js"
 mkdir -p /usr/local/node-10.18.1
-curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-10.18.1
+curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
 ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
 ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
-apt-get install -y --no-install-recommends python   # Install python which is required for npm rebuild
+apt-get install -y python   # Install python which is required for npm rebuild
 [[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"

 # https://docs.docker.com/engine/installation/linux/ubuntulinux/
@@ -87,9 +77,9 @@ mkdir -p /etc/systemd/system/docker.service.d
 echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf

 # there are 3 packages for docker - containerd, CLI and the daemon
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.13-2_amd64.deb" -o /tmp/containerd.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.2-3_amd64.deb" -o /tmp/containerd.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_18.09.2~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_18.09.2~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 # apt install with install deps (as opposed to dpkg -i)
 apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
 rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
@@ -102,7 +92,7 @@ fi

 # do not upgrade grub because it might prompt user and break this script
 echo "==> Enable memory accounting"
-apt-get -y --no-upgrade --no-install-recommends install grub2-common
+apt-get -y --no-upgrade install grub2-common
 sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
 update-grub

@@ -121,37 +111,26 @@ for image in ${images}; do
 done

 echo "==> Install collectd"
-# without this, libnotify4 will install gnome-shell
-apt-get install -y libnotify4 --no-install-recommends
-if ! apt-get install -y --no-install-recommends libcurl3-gnutls collectd collectd-utils; then
+if ! apt-get install -y collectd collectd-utils; then
    # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
    echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
    sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
 fi
-# https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
-[[ "${ubuntu_version}" == "20.04" ]] && echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd

-# some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
 echo "==> Configuring host"
 sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
-if systemctl is-active ntp; then
-    systemctl stop ntp
-    apt purge -y ntp
-fi
 timedatectl set-ntp 1
 # mysql follows the system timezone
 timedatectl set-timezone UTC

 echo "==> Adding sshd configuration warning"
-sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://docs.cloudron.io/security/#securing-ssh-access' -i /etc/ssh/sshd_config
+sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://cloudron.io/documentation/security/#securing-ssh-access' -i /etc/ssh/sshd_config

 # https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
 echo "==> Disabling motd news"
-if [ -f "/etc/default/motd-news" ]; then
-    sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
-fi
+sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news

-# Disable bind for good measure (on online.net, kimsufi servers these are pre-installed)
+# Disable bind for good measure (on online.net, kimsufi servers these are pre-installed and conflicts with unbound)
 systemctl stop bind9 || true
 systemctl disable bind9 || true

@@ -163,7 +142,7 @@ systemctl disable dnsmasq || true
 systemctl stop postfix || true
 systemctl disable postfix || true

-# on ubuntu 18.04 and 20.04, this is the default. this requires resolvconf for DNS to work further after the disable
+# on ubuntu 18.04, this is the default. this requires resolvconf for DNS to work further after the disable
 systemctl stop systemd-resolved || true
 systemctl disable systemd-resolved || true

@@ -172,3 +151,4 @@ systemctl disable systemd-resolved || true
 ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
 echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: ${ip6}" > /etc/unbound/unbound.conf.d/cloudron-network.conf
 systemctl restart unbound
+
@@ -2,65 +2,57 @@

 'use strict';

+// prefix all output with a timestamp
+// debug() already prefixes and uses process.stderr NOT console.*
+['log', 'info', 'warn', 'debug', 'error'].forEach(function (log) {
+    var orig = console[log];
+    console[log] = function () {
+        orig.apply(console, [new Date().toISOString()].concat(Array.prototype.slice.call(arguments)));
+    };
+});
+
+require('supererror')({ splatchError: true });
+
 let async = require('async'),
+    constants = require('./src/constants.js'),
    dockerProxy = require('./src/dockerproxy.js'),
-    fs = require('fs'),
    ldap = require('./src/ldap.js'),
-    paths = require('./src/paths.js'),
-    proxyAuth = require('./src/proxyauth.js'),
    server = require('./src/server.js');

-const NOOP_CALLBACK = function () { };
-
-function setupLogging(callback) {
-    if (process.env.BOX_ENV === 'test') return callback();
-
-    var logfileStream = fs.createWriteStream(paths.BOX_LOG_FILE, { flags:'a' });
-    process.stdout.write = process.stderr.write = logfileStream.write.bind(logfileStream);
-
-    callback();
-}
+console.log();
+console.log('==========================================');
+console.log(`           Cloudron ${constants.VERSION}  `);
+console.log('==========================================');
+console.log();

 async.series([
-    setupLogging,
-    server.start, // do this first since it also inits the database
-    proxyAuth.start,
+    server.start,
    ldap.start,
    dockerProxy.start
 ], function (error) {
    if (error) {
-        console.log('Error starting server', error);
+        console.error('Error starting server', error);
        process.exit(1);
    }
-
-    // require those here so that logging handler is already setup
-    require('supererror');
-    const debug = require('debug')('box:box');
-
-    process.on('SIGINT', function () {
-        debug('Received SIGINT. Shutting down.');
-
-        proxyAuth.stop(NOOP_CALLBACK);
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
-        setTimeout(process.exit.bind(process), 3000);
-    });
-
-    process.on('SIGTERM', function () {
-        debug('Received SIGTERM. Shutting down.');
-
-        proxyAuth.stop(NOOP_CALLBACK);
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
-        setTimeout(process.exit.bind(process), 3000);
-    });
-
-    process.on('uncaughtException', function (error) {
-        console.error((error && error.stack) ? error.stack : error);
-        setTimeout(process.exit.bind(process, 1), 3000);
-    });
-
-    console.log(`Cloudron is up and running. Logs are at ${paths.BOX_LOG_FILE}`); // this goes to journalctl
+    console.log('Cloudron is up and running');
+});
+
+var NOOP_CALLBACK = function () { };
+
+process.on('SIGINT', function () {
+    console.log('Received SIGINT. Shutting down.');
+
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    dockerProxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});
+
+process.on('SIGTERM', function () {
+    console.log('Received SIGTERM. Shutting down.');
+
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    dockerProxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
 });
@@ -12,6 +12,8 @@ exports.up = function(db, callback) {
 			db.all('SELECT * FROM users WHERE admin=1', function (error, results) {
 				if (error) return done(error);

+				console.dir(results);
+
 				async.eachSeries(results, function (r, next) {
 					db.runSql('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ ADMIN_GROUP_ID, r.id ], next);
 				}, done);
@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN resetTokenCreationTime', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,28 +0,0 @@
-'use strict';
-
-let async = require('async');
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps MODIFY mailboxDomain VARCHAR(128)', [], function (error) { // make it nullable
-        if (error) console.error(error);
-
-        // clear mailboxName/Domain for apps that do not use mail addons
-        db.all('SELECT * FROM apps', function (error, apps) {
-            if (error) return callback(error);
-
-            async.eachSeries(apps, function (app, iteratorDone) {
-                var manifest = JSON.parse(app.manifestJson);
-                if (manifest.addons['sendmail'] || manifest.addons['recvmail']) return iteratorDone();
-
-                db.runSql('UPDATE apps SET mailboxName=?, mailboxDomain=? WHERE id=?', [ null, null, app.id ], iteratorDone);
-            }, callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps MODIFY manifestJson VARCHAR(128) NOT NULL', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,17 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE mailboxes ADD COLUMN membersOnly BOOLEAN DEFAULT 0', function (error) {
-        if (error) return callback(error);
-
-        callback();
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN membersOnly', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
@@ -1,28 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN aliasDomain VARCHAR(128)'),
-        function setAliasDomain(done) {
-            db.all('SELECT * FROM mailboxes', function (error, mailboxes) {
-                async.eachSeries(mailboxes, function (mailbox, iteratorDone) {
-                    if (!mailbox.aliasTarget) return iteratorDone();
-
-                    db.runSql('UPDATE mailboxes SET aliasDomain=? WHERE name=? AND domain=?', [ mailbox.domain, mailbox.name, mailbox.domain ], iteratorDone);
-                }, done);
-            });
-        },
-        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD CONSTRAINT mailboxes_aliasDomain_constraint FOREIGN KEY(aliasDomain) REFERENCES mail(domain)'),
-        db.runSql.bind(db, 'ALTER TABLE mailboxes CHANGE aliasTarget aliasName VARCHAR(128)')
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE mailboxes DROP FOREIGN KEY mailboxes_aliasDomain_constraint'),
-        db.runSql.bind(db, 'ALTER TABLE mailboxes DROP COLUMN aliasDomain'),
-        db.runSql.bind(db, 'ALTER TABLE mailboxes CHANGE aliasName aliasTarget VARCHAR(128)')
-    ], callback);
-};
@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN servicesConfigJson TEXT', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN servicesConfigJson', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN bindsJson TEXT', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN bindsJson', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,35 +0,0 @@
-'use strict';
-
-const backups = require('../src/backups.js'),
-    fs = require('fs');
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        var backupConfig = JSON.parse(results[0].value);
-        if (backupConfig.key) {
-            backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.key);
-            backups.cleanupCacheFilesSync();
-
-            fs.writeFileSync('/home/yellowtent/platformdata/BACKUP_PASSWORD',
-                'This file contains your Cloudron backup password.\nBefore Cloudron v5.2, this was saved in the database.' +
-                'From Cloudron 5.2, this password is not required anymore. We generate strong keys based off this password and use those keys to encrypt the backups.\n' +
-                'This means that the password is only required at decryption/restore time.\n\n' +
-                'This file can be safely removed and only exists for the off-chance that you do not remember your backup password.\n\n' +
-                `Password: ${backupConfig.key}\n`,
-                'utf8');
-
-        } else {
-            backupConfig.encryption = null;
-        }
-
-        delete backupConfig.key;
-
-        db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [ JSON.stringify(backupConfig) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE backups CHANGE version packageVersion VARCHAR(128) NOT NULL', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE backups CHANGE packageVersion version VARCHAR(128) NOT NULL', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,24 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE backups ADD COLUMN encryptionVersion INTEGER', function (error) {
-        if (error) return callback(error);
-
-        db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-            if (error || results.length === 0) return callback(error);
-
-            var backupConfig = JSON.parse(results[0].value);
-            if (!backupConfig.encryption) return callback(null);
-
-            // mark old encrypted backups as v1
-            db.runSql('UPDATE backups SET encryptionVersion=1', callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE backups DROP COLUMN encryptionVersion', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,18 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        var backupConfig = JSON.parse(results[0].value);
-        backupConfig.retentionPolicy = { keepWithinSecs: backupConfig.retentionSecs };
-        delete backupConfig.retentionSecs;
-
-        // mark old encrypted backups as v1
-        db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [ JSON.stringify(backupConfig) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,18 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        var backupConfig = JSON.parse(results[0].value);
-        if (backupConfig.provider !== 'minio' && backupConfig.provider !== 's3-v4-compat') return callback();
-
-        backupConfig.s3ForcePathStyle = true; // usually minio is self-hosted. s3 v4 compat, we don't know
-
-        db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [ JSON.stringify(backupConfig) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,17 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-
-    // http://stackoverflow.com/questions/386294/what-is-the-maximum-length-of-a-valid-email-address
-
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE appPasswords DROP INDEX name'),
-        db.runSql.bind(db, 'ALTER TABLE appPasswords ADD CONSTRAINT appPasswords_name_userId_identifier UNIQUE (name, userId, identifier)'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,17 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE userGroups ADD COLUMN source VARCHAR(128) DEFAULT ""', function (error) {
-        if (error) return callback(error);
-
-        callback();
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE userGroups DROP COLUMN source', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
@@ -1,38 +0,0 @@
-'use strict';
-
-const async = require('async');
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE backups ADD COLUMN identifier VARCHAR(128)', function (error) {
-        if (error) return callback(error);
-
-
-        db.all('SELECT * FROM backups', function (error, backups) {
-            if (error) return callback(error);
-
-            async.eachSeries(backups, function (backup, next) {
-                let identifier = 'unknown';
-
-                if (backup.type === 'box') {
-                    identifier = 'box';
-                } else {
-                    const match = backup.id.match(/app_(.+?)_.+/);
-                    if (match) identifier = match[1];
-                }
-
-                db.runSql('UPDATE backups SET identifier=? WHERE id=?', [ identifier, backup.id ], next);
-            }, function (error) {
-                if (error) return callback(error);
-
-                db.runSql('ALTER TABLE backups MODIFY COLUMN identifier VARCHAR(128) NOT NULL', callback);
-            });
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE backups DROP COLUMN identifier', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,16 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
-        if (error) console.error(error);
-
-        db.runSql('ALTER TABLE users DROP COLUMN modifiedAt', callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN ts', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,29 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        var backupConfig = JSON.parse(results[0].value);
-        if (backupConfig.intervalSecs === 6 * 60 * 60) { // every 6 hours
-            backupConfig.schedulePattern = '00 00 5,11,17,23 * * *';
-        } else if (backupConfig.intervalSecs === 12 * 60 * 60) { // every 12 hours
-            backupConfig.schedulePattern = '00 00 5,17 * * *';
-        } else if (backupConfig.intervalSecs === 24 * 60 * 60) { // every day
-            backupConfig.schedulePattern = '00 00 23 * * *';
-        } else if (backupConfig.intervalSecs === 3 * 24 * 60 * 60) { // every 3 days (based on day)
-            backupConfig.schedulePattern = '00 00 23 * * 1,3,5';
-        } else if (backupConfig.intervalSecs === 7 * 24 * 60 * 60) { // every week (saturday)
-            backupConfig.schedulePattern = '00 00 23 * * 6';
-        } else { // default to everyday
-            backupConfig.schedulePattern = '00 00 23 * * *';
-        }
-
-        delete backupConfig.intervalSecs;
-        db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [ JSON.stringify(backupConfig) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,23 +0,0 @@
-'use strict';
-
-const async = require('async');
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="admin_domain"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        const adminDomain = results[0].value;
-
-        async.series([
-            db.runSql.bind(db, 'INSERT INTO settings (name, value) VALUES (?, ?)', [ 'mail_domain', adminDomain ]),
-            db.runSql.bind(db, 'INSERT INTO settings (name, value) VALUES (?, ?)', [ 'mail_fqdn', `my.${adminDomain}` ])
-        ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'DELETE FROM settings WHERE name="mail_domain"'),
-        db.runSql.bind(db, 'DELETE FROM settings WHERE name="mail_fqdn"'),
-    ], callback);
-};
@@ -1,22 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-    db.runSql('SELECT * FROM settings WHERE name=?', ['app_autoupdate_pattern'], function (error, results) {
-        if (error || results.length === 0) return callback(error); // will use defaults from box code
-
-        var updatePattern = results[0].value; // use app auto update patter for the box as well
-
-        async.series([
-            db.runSql.bind(db, 'START TRANSACTION;'),
-            db.runSql.bind(db, 'DELETE FROM settings WHERE name=? OR name=?', ['app_autoupdate_pattern', 'box_autoupdate_pattern']),
-            db.runSql.bind(db, 'INSERT settings (name, value) VALUES(?, ?)', ['autoupdate_pattern', updatePattern]),
-            db.runSql.bind(db, 'COMMIT')
-        ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE mail ADD COLUMN bannerJson TEXT', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE mail DROP COLUMN bannerJson', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,27 +0,0 @@
-'use strict';
-
-const OLD_FIREWALL_CONFIG_JSON = '/home/yellowtent/boxdata/firewall-config.json';
-const PORTS_FILE = '/home/yellowtent/boxdata/firewall/ports.json';
-const BLOCKLIST_FILE = '/home/yellowtent/boxdata/firewall/blocklist.txt';
-
-const fs = require('fs');
-
-exports.up = function (db, callback) {
-    if (!fs.existsSync(OLD_FIREWALL_CONFIG_JSON)) return callback();
-
-    try {
-        const dataJson = fs.readFileSync(OLD_FIREWALL_CONFIG_JSON, 'utf8');
-        const data = JSON.parse(dataJson);
-        fs.writeFileSync(BLOCKLIST_FILE, data.blocklist.join('\n') + '\n', 'utf8');
-        fs.writeFileSync(PORTS_FILE, JSON.stringify({ allowed_tcp_ports: data.allowed_tcp_ports }, null, 4), 'utf8');
-        fs.unlinkSync(OLD_FIREWALL_CONFIG_JSON);
-    } catch (error) {
-        console.log('Error migrating old firewall config', error);
-    }
-
-    callback();
-};
-
-exports.down = function (db, callback) {
-    callback();
-};
@@ -1,40 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    var cmd1 = 'CREATE TABLE volumes(' +
-                'id VARCHAR(128) NOT NULL UNIQUE,' +
-                'name VARCHAR(256) NOT NULL UNIQUE,' +
-                'hostPath VARCHAR(1024) NOT NULL UNIQUE,' +
-                'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' +
-                'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin';
-
-    var cmd2 = 'CREATE TABLE appMounts(' +
-                 'appId VARCHAR(128) NOT NULL,' +
-                 'volumeId VARCHAR(128) NOT NULL,' +
-                 'readOnly BOOLEAN DEFAULT 1,' +
-                 'UNIQUE KEY appMounts_appId_volumeId (appId, volumeId),' +
-                 'FOREIGN KEY(appId) REFERENCES apps(id),' +
-                 'FOREIGN KEY(volumeId) REFERENCES volumes(id)) CHARACTER SET utf8 COLLATE utf8_bin;';
-
-    db.runSql(cmd1, function (error) {
-        if (error) console.error(error);
-
-        db.runSql(cmd2, function (error) {
-            if (error) console.error(error);
-
-            db.runSql('ALTER TABLE apps DROP COLUMN bindsJson', callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('DROP TABLE appMounts', function (error) {
-        if (error) console.error(error);
-
-        db.runSql('DROP TABLE volumes', function (error) {
-            if (error) console.error(error);
-            callback(error);
-        });
-    });
-};
-
@@ -1,16 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN proxyAuth BOOLEAN DEFAULT 0', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN proxyAuth', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
@@ -1,18 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN ownerType VARCHAR(16)'),
-        db.runSql.bind(db, 'UPDATE mailboxes SET ownerType=?', [ 'user' ]),
-        db.runSql.bind(db, 'ALTER TABLE mailboxes MODIFY ownerType VARCHAR(16) NOT NULL'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE mailboxes DROP COLUMN ownerType', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,13 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN httpPort')
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,29 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    iputils = require('../src/iputils.js');
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN containerIp VARCHAR(16) UNIQUE', function (error) {
-        if (error) console.error(error);
-
-        let baseIp = iputils.intFromIp('172.18.16.0');
-
-        db.all('SELECT * FROM apps', function (error, apps) {
-            if (error) return callback(error);
-
-            async.eachSeries(apps, function (app, iteratorDone) {
-                const nextIp = iputils.ipFromInt(++baseIp);
-                db.runSql('UPDATE apps SET containerIp=? WHERE id=?', [ nextIp, app.id ], iteratorDone);
-            }, callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN containerIp', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
@@ -1,21 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT * FROM settings WHERE name=?', ['platform_config'], function (error, results) {
-        let value;
-        if (error || results.length === 0) {
-            value = { sftp: { requireAdmin: true } };
-        } else {
-            value = JSON.parse(results[0].value);
-            if (!value.sftp) value.sftp = {};
-            value.sftp.requireAdmin = true;
-        }
-
-        // existing installations may not even have the key. so use REPLACE instead of UPDATE
-        db.runSql('REPLACE INTO settings (name, value) VALUES (?, ?)', [ 'platform_config', JSON.stringify(value) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,18 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'CREATE TABLE groupMembers_copy(groupId VARCHAR(128) NOT NULL, userId VARCHAR(128) NOT NULL, FOREIGN KEY(groupId) REFERENCES userGroups(id), FOREIGN KEY(userId) REFERENCES users(id), UNIQUE (groupId, userId)) CHARACTER SET utf8 COLLATE utf8_bin'),    // In mysql CREATE TABLE.. LIKE does not copy indexes
-        db.runSql.bind(db, 'INSERT INTO groupMembers_copy SELECT * FROM groupMembers GROUP BY groupId, userId'),
-        db.runSql.bind(db, 'DROP TABLE groupMembers'),
-        db.runSql.bind(db, 'ALTER TABLE groupMembers_copy RENAME TO groupMembers')
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE groupMembers DROP INDEX groupMembers_member'),
-    ], callback);
-};
@@ -1,51 +0,0 @@
-'use strict';
-
-const async = require('async'),
-    safe = require('safetydance');
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE domains ADD COLUMN wellKnownJson TEXT', function (error) {
-        if (error) return callback(error);
-
-        // keep the paths around, so that we don't need to trigger a re-configure. the old nginx config will use the paths
-        // the new one will proxy calls to the box code
-        const WELLKNOWN_DIR = '/home/yellowtent/boxdata/well-known';
-        const output = safe.child_process.execSync('find . -type f -printf "%P\n"', { cwd: WELLKNOWN_DIR, encoding: 'utf8' });
-        if (!output) return callback();
-        const paths = output.trim().split('\n');
-        if (paths.length === 0) return callback(); // user didn't configure any well-known
-
-        let wellKnown = {};
-        for (let path of paths) {
-            const fqdn = path.split('/', 1)[0];
-            const loc = path.slice(fqdn.length+1);
-            const doc = safe.fs.readFileSync(`${WELLKNOWN_DIR}/${path}`, { encoding: 'utf8' });
-            if (!doc) continue;
-
-            wellKnown[fqdn] = {};
-            wellKnown[fqdn][loc] = doc;
-        }
-
-        console.log('Migrating well-known', JSON.stringify(wellKnown, null, 4));
-
-        async.eachSeries(Object.keys(wellKnown), function (fqdn, iteratorDone) {
-            db.runSql('UPDATE domains SET wellKnownJson=? WHERE domain=?', [ JSON.stringify(wellKnown[fqdn]), fqdn ], function (error, result) {
-                if (error) {
-                    console.error(error); // maybe the domain does not exist anymore
-                } else if (result.affectedRows === 0) {
-                    console.log(`Could not migrate wellknown as domain ${fqdn} is missing`);
-                }
-                iteratorDone();
-            });
-        }, function (error) {
-            callback(error);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE domains DROP COLUMN wellKnownJson', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
@@ -1,23 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT * FROM settings WHERE name=?', ['platform_config'], function (error, results) {
-        if (error || results.length === 0) return callback(null);
-
-        let value = JSON.parse(results[0].value);
-
-        for (const serviceName of Object.keys(value)) {
-            const service = value[serviceName];
-            if (!service.memorySwap) continue;
-            service.memoryLimit = service.memorySwap;
-            delete service.memorySwap;
-            delete service.memory;
-        }
-
-        db.runSql('UPDATE settings SET value=? WHERE name=?', [ JSON.stringify(value), 'platform_config' ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,28 +0,0 @@
-'use strict';
-
-const async = require('async');
-
-exports.up = function(db, callback) {
-    db.all('SELECT * FROM apps', function (error, apps) {
-        if (error) return callback(error);
-
-        async.eachSeries(apps, function (app, iteratorDone) {
-            if (!app.servicesConfigJson) return iteratorDone();
-
-            let servicesConfig = JSON.parse(app.servicesConfigJson);
-            for (const serviceName of Object.keys(servicesConfig)) {
-                const service = servicesConfig[serviceName];
-                if (!service.memorySwap) continue;
-                service.memoryLimit = service.memorySwap;
-                delete service.memorySwap;
-                delete service.memory;
-            }
-
-            db.runSql('UPDATE apps SET servicesConfigJson=? WHERE id=?', [ JSON.stringify(servicesConfig), app.id ], iteratorDone);
-        }, callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};
@@ -1,9 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'services_config', 'platform_config' ], callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'platform_config', 'services_config' ], callback);
-};
@@ -21,31 +21,26 @@ CREATE TABLE IF NOT EXISTS users(
    password VARCHAR(1024) NOT NULL,
    salt VARCHAR(512) NOT NULL,
    createdAt VARCHAR(512) NOT NULL,
-    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    modifiedAt VARCHAR(512) NOT NULL,
    displayName VARCHAR(512) DEFAULT "",
    fallbackEmail VARCHAR(512) DEFAULT "",
    twoFactorAuthenticationSecret VARCHAR(128) DEFAULT "",
    twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
    source VARCHAR(128) DEFAULT "",
    role VARCHAR(32),
-    resetToken VARCHAR(128) DEFAULT "",
-    resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    active BOOLEAN DEFAULT 1,

    PRIMARY KEY(id));

 CREATE TABLE IF NOT EXISTS userGroups(
    id VARCHAR(128) NOT NULL UNIQUE,
    name VARCHAR(254) NOT NULL UNIQUE,
-    source VARCHAR(128) DEFAULT "",
    PRIMARY KEY(id));

 CREATE TABLE IF NOT EXISTS groupMembers(
    groupId VARCHAR(128) NOT NULL,
    userId VARCHAR(128) NOT NULL,
    FOREIGN KEY(groupId) REFERENCES userGroups(id),
-    FOREIGN KEY(userId) REFERENCES users(id),
-    UNIQUE (groupId, userId));
+    FOREIGN KEY(userId) REFERENCES users(id));

 CREATE TABLE IF NOT EXISTS tokens(
    id VARCHAR(128) NOT NULL UNIQUE,
@@ -66,6 +61,9 @@ CREATE TABLE IF NOT EXISTS apps(
    healthTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, // when the app last responded
    containerId VARCHAR(128),
    manifestJson TEXT,
+    httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
+    location VARCHAR(128) NOT NULL,
+    domain VARCHAR(128) NOT NULL,
    accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, // when the app was installed
    updateTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,   // when the last app update was done
@@ -78,15 +76,13 @@ CREATE TABLE IF NOT EXISTS apps(
    reverseProxyConfigJson TEXT, // { robotsTxt, csp }
    enableBackup BOOLEAN DEFAULT 1, // misnomer: controls automatic daily backups
    enableAutomaticUpdate BOOLEAN DEFAULT 1,
-    mailboxName VARCHAR(128), // mailbox of this app
-    mailboxDomain VARCHAR(128), // mailbox domain of this apps
+    mailboxName VARCHAR(128), // mailbox of this app. default allocated as '.app'
+    mailboxDomain VARCHAR(128) NOT NULL, // mailbox domain of this apps
    label VARCHAR(128),     // display name
    tagsJson VARCHAR(2048), // array of tags
    dataDir VARCHAR(256) UNIQUE,
    taskId INTEGER, // current task
    errorJson TEXT,
-    servicesConfigJson TEXT, // app services configuration
-    containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair'

    FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
    FOREIGN KEY(taskId) REFERENCES tasks(id),
@@ -121,10 +117,8 @@ CREATE TABLE IF NOT EXISTS appEnvVars(
 CREATE TABLE IF NOT EXISTS backups(
    id VARCHAR(128) NOT NULL,
    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    packageVersion VARCHAR(128) NOT NULL, /* app version or box version */
-    encryptionVersion INTEGER, /* when null, unencrypted backup */
+    version VARCHAR(128) NOT NULL, /* app version or box version */
    type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
-    identifier VARCHAR(128) NOT NULL, /* 'box' or the app id */
    dependsOn TEXT, /* comma separate list of objects this backup depends on */
    state VARCHAR(16) NOT NULL,
    manifestJson TEXT, /* to validate if the app can be installed in this version of box */
@@ -148,7 +142,6 @@ CREATE TABLE IF NOT EXISTS domains(
    provider VARCHAR(16) NOT NULL,
    configJson TEXT, /* JSON containing the dns backend provider config */
    tlsConfigJson TEXT, /* JSON containing the tls provider config */
-    wellKnownJson TEXT, /* JSON containing well known docs for this domain */

    PRIMARY KEY (domain))

@@ -162,7 +155,6 @@ CREATE TABLE IF NOT EXISTS mail(
    mailFromValidation BOOLEAN DEFAULT 1,
    catchAllJson TEXT,
    relayJson TEXT,
-    bannerJson TEXT,

    dkimSelector VARCHAR(128) NOT NULL DEFAULT "cloudron",

@@ -182,16 +174,12 @@ CREATE TABLE IF NOT EXISTS mailboxes(
    name VARCHAR(128) NOT NULL,
    type VARCHAR(16) NOT NULL, /* 'mailbox', 'alias', 'list' */
    ownerId VARCHAR(128) NOT NULL, /* user id */
-    ownerType VARCHAR(16) NOT NULL,
-    aliasName VARCHAR(128), /* the target name type is an alias */
-    aliasDomain VARCHAR(128), /* the target domain */
+    aliasTarget VARCHAR(128), /* the target name type is an alias */
    membersJson TEXT, /* members of a group. fully qualified */
-    membersOnly BOOLEAN DEFAULT false,
    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    domain VARCHAR(128),

    FOREIGN KEY(domain) REFERENCES mail(domain),
-    FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
    UNIQUE (name, domain));

 CREATE TABLE IF NOT EXISTS subdomains(
@@ -223,7 +211,7 @@ CREATE TABLE IF NOT EXISTS notifications(
    message TEXT,
    acknowledged BOOLEAN DEFAULT false,
    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    UNIQUE KEY appPasswords_name_appId_identifier (name, userId, identifier),
+
    PRIMARY KEY (id)
 );

@@ -235,24 +223,8 @@ CREATE TABLE IF NOT EXISTS appPasswords(
    hashedPassword VARCHAR(1024) NOT NULL,
    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    FOREIGN KEY(userId) REFERENCES users(id),
-
+    UNIQUE (name, userId),
    PRIMARY KEY (id)
 );

-CREATE TABLE IF NOT EXISTS volumes(
-    id VARCHAR(128) NOT NULL UNIQUE,
-    name VARCHAR(256) NOT NULL UNIQUE,
-    hostPath VARCHAR(1024) NOT NULL UNIQUE,
-    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    PRIMARY KEY (id)
-);
-
-CREATE TABLE IF NOT EXISTS appMounts(
-    appId VARCHAR(128) NOT NULL,
-    volumeId VARCHAR(128) NOT NULL,
-    readOnly BOOLEAN DEFAULT 1,
-    UNIQUE KEY appMounts_appId_volumeId (appId, volumeId),
-    FOREIGN KEY(appId) REFERENCES apps(id),
-    FOREIGN KEY(volumeId) REFERENCES volumes(id));
-
 CHARACTER SET utf8 COLLATE utf8_bin;
@@ -14,76 +14,71 @@
    "node": ">=4.0.0 <=4.1.1"
  },
  "dependencies": {
-    "@google-cloud/dns": "^1.2.9",
+    "@google-cloud/dns": "^1.1.0",
    "@google-cloud/storage": "^2.5.0",
    "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
    "async": "^2.6.3",
-    "aws-sdk": "^2.828.0",
-    "basic-auth": "^2.0.1",
+    "aws-sdk": "^2.610.0",
    "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.10.1",
+    "cloudron-manifestformat": "^4.0.0",
    "connect": "^3.7.0",
-    "connect-lastmile": "^2.0.0",
+    "connect-lastmile": "^1.2.2",
    "connect-timeout": "^1.9.0",
-    "cookie-parser": "^1.4.5",
    "cookie-session": "^1.4.0",
    "cron": "^1.8.2",
-    "db-migrate": "^0.11.12",
-    "db-migrate-mysql": "^2.1.2",
-    "debug": "^4.3.1",
+    "db-migrate": "^0.11.6",
+    "db-migrate-mysql": "^1.1.10",
+    "debug": "^4.1.1",
    "dockerode": "^2.5.8",
    "ejs": "^2.6.1",
-    "ejs-cli": "^2.2.1",
+    "ejs-cli": "^2.1.1",
    "express": "^4.17.1",
-    "ipaddr.js": "^2.0.0",
-    "js-yaml": "^3.14.0",
+    "js-yaml": "^3.13.1",
    "json": "^9.0.6",
-    "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.2.3",
-    "lodash": "^4.17.20",
+    "ldapjs": "^1.0.2",
+    "lodash": "^4.17.15",
    "lodash.chunk": "^4.2.0",
-    "mime": "^2.5.0",
-    "moment": "^2.29.1",
-    "moment-timezone": "^0.5.32",
-    "morgan": "^1.10.0",
-    "multiparty": "^4.2.2",
-    "mustache-express": "^1.3.0",
+    "mime": "^2.4.4",
+    "moment-timezone": "^0.5.27",
+    "morgan": "^1.9.1",
+    "multiparty": "^4.2.1",
    "mysql": "^2.18.1",
-    "nodemailer": "^6.4.17",
+    "nodemailer": "^6.4.2",
    "nodemailer-smtp-transport": "^2.7.4",
    "once": "^1.4.0",
-    "pretty-bytes": "^5.5.0",
+    "parse-links": "^0.1.0",
+    "pretty-bytes": "^5.3.0",
    "progress-stream": "^2.0.0",
    "proxy-middleware": "^0.15.0",
    "qrcode": "^1.4.4",
-    "readdirp": "^3.5.0",
-    "request": "^2.88.2",
+    "readdirp": "^3.3.0",
+    "request": "^2.88.0",
    "rimraf": "^2.6.3",
    "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^1.1.1",
+    "safetydance": "^1.0.0",
    "semver": "^6.1.1",
    "showdown": "^1.9.1",
    "speakeasy": "^2.0.0",
    "split": "^1.0.1",
-    "superagent": "^5.3.1",
+    "superagent": "^5.2.1",
    "supererror": "^0.7.2",
    "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
-    "tar-stream": "^2.2.0",
+    "tar-stream": "^2.1.0",
    "tldjs": "^2.3.1",
-    "underscore": "^1.12.0",
+    "underscore": "^1.9.2",
    "uuid": "^3.4.0",
    "validator": "^11.0.0",
-    "ws": "^7.4.2",
+    "ws": "^7.2.1",
    "xml2js": "^0.4.23"
  },
  "devDependencies": {
    "expect.js": "*",
-    "hock": "^1.4.1",
-    "js2xmlparser": "^4.0.1",
-    "mocha": "^6.2.3",
+    "hock": "^1.3.3",
+    "js2xmlparser": "^4.0.0",
+    "mocha": "^6.1.4",
    "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
    "nock": "^10.0.6",
-    "node-sass": "^4.14.1",
+    "node-sass": "^4.12.0",
    "recursive-readdir": "^2.2.2"
  },
  "scripts": {
@@ -2,11 +2,11 @@

 set -eu

-readonly source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SOURCE_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly DATA_DIR="${HOME}/.cloudron_test"
 readonly DEFAULT_TESTS="./src/test/*-test.js ./src/routes/test/*-test.js"

-! "${source_dir}/src/test/checkInstall" && exit 1
+! "${SOURCE_dir}/src/test/checkInstall" && exit 1

 # cleanup old data dirs some of those docker container data requires sudo to be removed
 echo "=> Provide root password to purge any leftover data in ${DATA_DIR} and load apparmor profile:"
@@ -22,27 +22,19 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com boxdata/sftp/ssh
-mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
-sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
-
-# translations
-mkdir -p box/dashboard/dist/translation
-cp -r ${source_dir}/../dashboard/dist/translation/* box/dashboard/dist/translation
+mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p platformdata/addons/mail platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks

 # put cert
 echo "=> Generating a localhost selfsigned cert"
 openssl req -x509 -newkey rsa:2048 -keyout platformdata/nginx/cert/host.key -out platformdata/nginx/cert/host.cert -days 3650 -subj '/CN=localhost' -nodes -config <(cat /etc/ssl/openssl.cnf <(printf "\n[SAN]\nsubjectAltName=DNS:*.localhost"))

-# generate legacy key format for sftp
-ssh-keygen -m PEM -t rsa -f boxdata/sftp/ssh/ssh_host_rsa_key -q -N ""
-
 # clear out any containers
 echo "=> Delete all docker containers first"
 docker ps -qa | xargs --no-run-if-empty docker rm -f

 # create docker network (while the infra code does this, most tests skip infra setup)
-docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
+docker network create --subnet=172.18.0.0/16 cloudron || true

 # create the same mysql server version to test with
 OUT=`docker inspect mysql-server` || true
@@ -67,7 +59,7 @@ echo "=> Ensure database"
 mysql -h"${MYSQL_IP}" -uroot -ppassword -e 'CREATE DATABASE IF NOT EXISTS box'

 echo "=> Run database migrations"
-cd "${source_dir}"
+cd "${SOURCE_dir}"
 BOX_ENV=test DATABASE_URL=mysql://root:password@${MYSQL_IP}/box node_modules/.bin/db-migrate up

 echo "=> Run tests with mocha"
@@ -41,39 +41,35 @@ if systemctl -q is-active box; then
 fi

 initBaseImage="true"
-provider="generic"
+# provisioning data
+provider=""
 requestedVersion=""
-installServerOrigin="https://api.cloudron.io"
 apiServerOrigin="https://api.cloudron.io"
 webServerOrigin="https://cloudron.io"
 sourceTarballUrl=""
 rebootServer="true"
-setupToken=""
+license=""

-args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot,generate-setup-token" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot,license:" -n "$0" -- "$@")
 eval set -- "${args}"

 while true; do
    case "$1" in
-    --help) echo "See https://docs.cloudron.io/installation/ on how to install Cloudron"; exit 0;;
+    --help) echo "See https://cloudron.io/documentation/installation/ on how to install Cloudron"; exit 0;;
    --provider) provider="$2"; shift 2;;
    --version) requestedVersion="$2"; shift 2;;
    --env)
        if [[ "$2" == "dev" ]]; then
            apiServerOrigin="https://api.dev.cloudron.io"
            webServerOrigin="https://dev.cloudron.io"
-            installServerOrigin="https://api.dev.cloudron.io"
        elif [[ "$2" == "staging" ]]; then
            apiServerOrigin="https://api.staging.cloudron.io"
            webServerOrigin="https://staging.cloudron.io"
-            installServerOrigin="https://api.staging.cloudron.io"
-        elif [[ "$2" == "unstable" ]]; then
-            installServerOrigin="https://api.dev.cloudron.io"
        fi
        shift 2;;
+    --license) license="$2"; shift 2;;
    --skip-baseimage-init) initBaseImage="false"; shift;;
    --skip-reboot) rebootServer="false"; shift;;
-    --generate-setup-token) setupToken="$(openssl rand -hex 10)"; shift;;
    --) break;;
    *) echo "Unknown option $1"; exit 1;;
    esac
@@ -87,14 +83,56 @@ fi

 # Only --help works with mismatched ubuntu
 ubuntu_version=$(lsb_release -rs)
-if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubuntu_version}" != "20.04" ]]; then
-    echo "Cloudron requires Ubuntu 16.04, 18.04 or 20.04" > /dev/stderr
+if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" ]]; then
+    echo "Cloudron requires Ubuntu 16.04 or 18.04" > /dev/stderr
    exit 1
 fi

 # Can only write after we have confirmed script has root access
 echo "Running cloudron-setup with args : $@" > "${LOG_FILE}"

+# validate arguments in the absence of data
+readonly AVAILABLE_PROVIDERS="azure, caas, cloudscale, contabo, digitalocean, ec2, exoscale, gce, hetzner, interox, lightsail, linode, netcup, ovh, rosehosting, scaleway, skysilk, time4vps, upcloud, vultr or generic"
+if [[ -z "${provider}" ]]; then
+    echo "--provider is required ($AVAILABLE_PROVIDERS)"
+    exit 1
+elif [[  \
+            "${provider}" != "ami" && \
+            "${provider}" != "azure" && \
+            "${provider}" != "azure-image" && \
+            "${provider}" != "caas" && \
+            "${provider}" != "cloudscale" && \
+            "${provider}" != "contabo" && \
+            "${provider}" != "digitalocean" && \
+            "${provider}" != "digitalocean-mp" && \
+            "${provider}" != "ec2" && \
+            "${provider}" != "exoscale" && \
+            "${provider}" != "gce" && \
+            "${provider}" != "hetzner" && \
+            "${provider}" != "interox" && \
+            "${provider}" != "interox-image" && \
+            "${provider}" != "lightsail" && \
+            "${provider}" != "linode" && \
+            "${provider}" != "linode-oneclick" && \
+            "${provider}" != "linode-stackscript" && \
+            "${provider}" != "netcup" && \
+            "${provider}" != "netcup-image" && \
+            "${provider}" != "ovh" && \
+            "${provider}" != "rosehosting" && \
+            "${provider}" != "scaleway" && \
+            "${provider}" != "skysilk" && \
+            "${provider}" != "skysilk-image" && \
+            "${provider}" != "time4vps" && \
+            "${provider}" != "time4vps-image" && \
+            "${provider}" != "upcloud" && \
+            "${provider}" != "upcloud-image" && \
+            "${provider}" != "vultr" && \
+            "${provider}" != "generic" \
+        ]]; then
+    echo "--provider must be one of: $AVAILABLE_PROVIDERS"
+    exit 1
+fi
+
 echo ""
 echo "##############################################"
 echo "         Cloudron Setup (${requestedVersion:-latest})"
@@ -107,20 +145,32 @@ echo " Join us at https://forum.cloudron.io for any questions."
 echo ""

 if [[ "${initBaseImage}" == "true" ]]; then
+    echo "=> Installing software-properties-common"
+    if ! apt-get install -y software-properties-common &>> "${LOG_FILE}"; then
+        echo "Could not install software-properties-common (for add-apt-repository below). See ${LOG_FILE}"
+        exit 1
+    fi
+
+    echo "=> Ensure required apt sources"
+    if ! add-apt-repository universe &>> "${LOG_FILE}"; then
+        echo "Could not add required apt sources (for nginx-full). See ${LOG_FILE}"
+        exit 1
+    fi
+
    echo "=> Updating apt and installing script dependencies"
    if ! apt-get update &>> "${LOG_FILE}"; then
        echo "Could not update package repositories. See ${LOG_FILE}"
        exit 1
    fi

-    if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install --no-install-recommends curl python3 ubuntu-standard software-properties-common -y &>> "${LOG_FILE}"; then
+    if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install curl python3 ubuntu-standard -y &>> "${LOG_FILE}"; then
        echo "Could not install setup dependencies (curl). See ${LOG_FILE}"
        exit 1
    fi
 fi

 echo "=> Checking version"
-if ! releaseJson=$($curl -s "${installServerOrigin}/api/v1/releases?boxVersion=${requestedVersion}"); then
+if ! releaseJson=$($curl -s "${apiServerOrigin}/api/v1/releases?boxVersion=${requestedVersion}"); then
    echo "Failed to get release information"
    exit 1
 fi
@@ -146,19 +196,19 @@ fi

 if [[ "${initBaseImage}" == "true" ]]; then
    echo -n "=> Installing base dependencies and downloading docker images (this takes some time) ..."
-    # initializeBaseUbuntuImage.sh args (provider, infraversion path) are only to support installation of pre 5.3 Cloudrons
-    if ! /bin/bash "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "generic" "../src" &>> "${LOG_FILE}"; then
+    if ! /bin/bash "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "${provider}" "../src" &>> "${LOG_FILE}"; then
        echo "Init script failed. See ${LOG_FILE} for details"
        exit 1
    fi
    echo ""
 fi

-# The provider flag is still used for marketplace images
+# NOTE: this install script only supports 4.2 and above
 echo "=> Installing version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
 echo "${provider}" > /etc/cloudron/PROVIDER
-[[ ! -z "${setupToken}" ]] && echo "${setupToken}" > /etc/cloudron/SETUP_TOKEN
+
+[[ -n "${license}" ]] && echo -n "$license" > /etc/cloudron/LICENSE

 if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
    echo "Failed to install cloudron. See ${LOG_FILE} for details"
@@ -171,21 +221,16 @@ mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('web
 echo -n "=> Waiting for cloudron to be ready (this takes some time) ..."
 while true; do
    echo -n "."
-    if status=$($curl -s -f "http://localhost:3000/api/v1/cloudron/status" 2>/dev/null); then
+    if status=$($curl -q -f "http://localhost:3000/api/v1/cloudron/status" 2>/dev/null); then
        break # we are up and running
    fi
    sleep 10
 done

-if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+if ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
    ip='<IP>'
 fi
-if [[ -z "${setupToken}" ]]; then
-    url="https://${ip}"
-else
-    url="https://${ip}/?setupToken=${setupToken}"
-fi
-echo -e "\n\n${GREEN}After reboot, visit ${url} and accept the self-signed certificate to finish setup.${DONE}\n"
+echo -e "\n\n${GREEN}Visit https://${ip} and accept the self-signed certificate to finish setup.${DONE}\n"

 if [[ "${rebootServer}" == "true" ]]; then
    systemctl stop box mysql # sometimes mysql ends up having corrupt privilege tables
@@ -13,7 +13,7 @@ HELP_MESSAGE="
 This script collects diagnostic information to help debug server related issues

 Options:
-   --owner-login   Login as owner
+   --admin-login   Login as administrator
   --enable-ssh    Enable SSH access for the Cloudron support team
   --help          Show this message
 "
@@ -26,7 +26,7 @@ fi

 enableSSH="false"

-args=$(getopt -o "" -l "help,enable-ssh,admin-login,owner-login" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,enable-ssh,admin-login" -n "$0" -- "$@")
 eval set -- "${args}"

 while true; do
@@ -34,15 +34,12 @@ while true; do
    --help) echo -e "${HELP_MESSAGE}"; exit 0;;
    --enable-ssh) enableSSH="true"; shift;;
    --admin-login)
-        # fall through
-        ;&
-    --owner-login)
-        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY createdAt LIMIT 1" 2>/dev/null)
+        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' LIMIT 1" 2>/dev/null)
        admin_password=$(pwgen -1s 12)
        ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
        printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > "${ghost_file}"
        chown yellowtent:yellowtent "${ghost_file}" && chmod o-r,g-r "${ghost_file}"
-        echo "Login as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
+        echo "Login as ${admin_username} / ${admin_password} . Remove ${ghost_file} when done."
        exit 0
        ;;
    --) break;;
@@ -57,7 +54,7 @@ if [[ "`df --output="avail" / | sed -n 2p`" -lt "10240" ]]; then
    echo ""
    df -h
    echo ""
-    echo "To recover from a full disk, follow the guide at https://docs.cloudron.io/troubleshooting/#recovery-after-disk-full"
+    echo "To recover from a full disk, follow the guide at https://cloudron.io/documentation/troubleshooting/#recovery-after-disk-full"
    exit 1
 fi

@@ -94,7 +91,7 @@ echo -e $LINE"Backup stats (possibly misleading)"$LINE >> $OUT
 du -hcsL /var/backups/* &>> $OUT || true

 echo -e $LINE"System daemon status"$LINE >> $OUT
-systemctl status --lines=100 box mysql unbound cloudron-syslog nginx collectd docker &>> $OUT
+systemctl status --lines=100 cloudron.target box mysql unbound cloudron-syslog nginx collectd docker &>> $OUT

 echo -e $LINE"Box logs"$LINE >> $OUT
 tail -n 100 /home/yellowtent/platformdata/logs/box.log &>> $OUT
@@ -112,7 +109,7 @@ if [[ "${enableSSH}" == "true" ]]; then
    permit_root_login=$(grep -q ^PermitRootLogin.*yes /etc/ssh/sshd_config && echo "yes" || echo "no")

    # support.js uses similar logic
-    if [[ -d /home/ubuntu ]]; then
+    if $(grep -q "ec2\|lightsail\|ami" /etc/cloudron/PROVIDER); then
        ssh_user="ubuntu"
        keys_file="/home/ubuntu/.ssh/authorized_keys"
    else
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-# This script downloads new translation data from weblate at https://translate.cloudron.io
-
-OUT="/home/yellowtent/box/dashboard/dist/translation"
-
-# We require root
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root. Run with sudo"
-    exit 1
-fi
-
-echo "=> Downloading new translation files..."
-curl https://translate.cloudron.io/download/cloudron/dashboard/?format=zip -o /tmp/lang.zip
-
-echo "=> Unpacking..."
-unzip -jo /tmp/lang.zip -d $OUT
-chown -R yellowtent:yellowtent $OUT
-# unzip put very restrictive permissions
-chmod ua+r $OUT/*
-
-echo "=> Cleanup..."
-rm /tmp/lang.zip
-
-echo "=> Done"
-
-echo ""
-echo "Reload the dashboard to see the new translations"
-echo ""
@@ -11,8 +11,9 @@ if [[ ${EUID} -ne 0 ]]; then
    exit 1
 fi

-readonly user=yellowtent
-readonly box_src_dir=/home/${user}/box
+readonly USER=yellowtent
+readonly BOX_SRC_DIR=/home/${USER}/box
+readonly BASE_DATA_DIR=/home/${USER}

 readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -23,15 +24,13 @@ readonly ubuntu_codename=$(lsb_release -cs)

 readonly is_update=$(systemctl is-active box && echo "yes" || echo "no")

-echo "==> installer: Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION) <=="
-
 echo "==> installer: updating docker"

-if [[ $(docker version --format {{.Client.Version}}) != "19.03.12" ]]; then
+if [[ $(docker version --format {{.Client.Version}}) != "18.09.2" ]]; then
    # there are 3 packages for docker - containerd, CLI and the daemon
-    curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.13-2_amd64.deb" -o /tmp/containerd.deb
-    curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
-    curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.2-3_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_18.09.2~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_18.09.2~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb

    echo "==> installer: Waiting for all dpkg tasks to finish..."
    while fuser /var/lib/dpkg/lock; do
@@ -57,19 +56,6 @@ if [[ $(docker version --format {{.Client.Version}}) != "19.03.12" ]]; then
    rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
 fi

-readonly nginx_version=$(nginx -v 2>&1)
-if [[ "${nginx_version}" != *"1.18."* ]]; then
-    echo "==> installer: installing nginx 1.18"
-    curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
-    # apt install with install deps (as opposed to dpkg -i)
-    apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
-    rm /tmp/nginx.deb
-fi
-
-# Cloudron 6 on ubuntu 20 installed recommended packages of collectd -> libinotify -> gnome->shell
-apt remove -y gnome-shell || true
-apt -y autoremove || true
-
 echo "==> installer: updating node"
 if [[ "$(node --version)" != "v10.18.1" ]]; then
    mkdir -p /usr/local/node-10.18.1
@@ -123,22 +109,22 @@ while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLO
    sleep 5
 done

-if ! id "${user}" 2>/dev/null; then
-    useradd "${user}" -m
+if ! id "${USER}" 2>/dev/null; then
+    useradd "${USER}" -m
 fi

 if [[ "${is_update}" == "yes" ]]; then
-    echo "==> installer: stop box service for update"
-    ${box_src_dir}/setup/stop.sh
+    echo "==> installer: stop cloudron.target service for update"
+    ${BOX_SRC_DIR}/setup/stop.sh
 fi

 # ensure we are not inside the source directory, which we will remove now
 cd /root

 echo "==> installer: switching the box code"
-rm -rf "${box_src_dir}"
-mv "${box_src_tmp_dir}" "${box_src_dir}"
-chown -R "${user}:${user}" "${box_src_dir}"
+rm -rf "${BOX_SRC_DIR}"
+mv "${box_src_tmp_dir}" "${BOX_SRC_DIR}"
+chown -R "${USER}:${USER}" "${BOX_SRC_DIR}"

 echo "==> installer: calling box setup script"
-"${box_src_dir}/setup/start.sh"
+"${BOX_SRC_DIR}/setup/start.sh"
@@ -19,12 +19,6 @@ readonly json="$(realpath ${script_dir}/../node_modules/.bin/json)"
 readonly ubuntu_version=$(lsb_release -rs)

 cp -f "${script_dir}/../scripts/cloudron-support" /usr/bin/cloudron-support
-cp -f "${script_dir}/../scripts/cloudron-translation-update" /usr/bin/cloudron-translation-update
-
-# this needs to match the cloudron/base:2.0.0 gid
-if ! getent group media; then
-    addgroup --gid 500 --system media
-fi

 echo "==> Configuring docker"
 cp "${script_dir}/start/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
@@ -32,8 +26,7 @@ systemctl enable apparmor
 systemctl restart apparmor

 usermod ${USER} -a -G docker
-# unbound (which starts after box code) relies on this interface to exist. dockerproxy also relies on this.
-docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
+docker network create --subnet=172.18.0.0/16 cloudron || true

 mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
@@ -46,7 +39,7 @@ mkdir -p "${PLATFORM_DATA_DIR}/mysql"
 mkdir -p "${PLATFORM_DATA_DIR}/postgresql"
 mkdir -p "${PLATFORM_DATA_DIR}/mongodb"
 mkdir -p "${PLATFORM_DATA_DIR}/redis"
-mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner"
+mkdir -p "${PLATFORM_DATA_DIR}/addons/mail"
 mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
 mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
 mkdir -p "${PLATFORM_DATA_DIR}/acme"
@@ -59,13 +52,10 @@ mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
 mkdir -p "${PLATFORM_DATA_DIR}/update"

 mkdir -p "${BOX_DATA_DIR}/appicons"
-mkdir -p "${BOX_DATA_DIR}/firewall"
 mkdir -p "${BOX_DATA_DIR}/profileicons"
 mkdir -p "${BOX_DATA_DIR}/certs"
 mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
 mkdir -p "${BOX_DATA_DIR}/mail/dkim"
-mkdir -p "${BOX_DATA_DIR}/well-known" # .well-known documents
-mkdir -p "${BOX_DATA_DIR}/sftp/ssh" # sftp keys

 # ensure backups folder exists and is writeable
 mkdir -p /var/backups
@@ -89,9 +79,6 @@ systemctl daemon-reload
 systemctl restart systemd-journald
 setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal

-# Give user access to nginx logs (uses adm group)
-usermod -a -G adm ${USER}
-
 echo "==> Setting up unbound"
 # DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org)
 # We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
@@ -106,11 +93,10 @@ echo "==> Adding systemd services"
 cp -r "${script_dir}/start/systemd/." /etc/systemd/system/
 [[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/MemoryMax/MemoryLimit/g' -i /etc/systemd/system/box.service
 systemctl daemon-reload
-systemctl enable --now cloudron-syslog
 systemctl enable unbound
-systemctl enable box
+systemctl enable cloudron-syslog
+systemctl enable cloudron.target
 systemctl enable cloudron-firewall
-systemctl enable --now cloudron-disable-thp

 # update firewall rules
 systemctl restart cloudron-firewall
@@ -132,12 +118,6 @@ echo "==> Configuring collectd"
 rm -rf /etc/collectd /var/log/collectd.log
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
-if [[ "${ubuntu_version}" == "20.04" ]]; then
-    # https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
-    if ! grep -q LD_PRELOAD /etc/default/collectd; then
-        echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
-    fi
-fi
 systemctl restart collectd

 echo "==> Configuring logrotate"
@@ -164,15 +144,8 @@ cp "${script_dir}/start/nginx/mime.types" "${PLATFORM_DATA_DIR}/nginx/mime.types
 if ! grep -q "^Restart=" /etc/systemd/system/multi-user.target.wants/nginx.service; then
    # default nginx service file does not restart on crash
    echo -e "\n[Service]\nRestart=always\n" >> /etc/systemd/system/multi-user.target.wants/nginx.service
+    systemctl daemon-reload
 fi
-
-# worker_rlimit_nofile in nginx config can be max this number
-mkdir -p /etc/systemd/system/nginx.service.d
-if ! grep -q "^LimitNOFILE=" /etc/systemd/system/nginx.service.d/cloudron.conf; then
-    echo -e "[Service]\nLimitNOFILE=16384\n" > /etc/systemd/system/nginx.service.d/cloudron.conf
-fi
-
-systemctl daemon-reload
 systemctl start nginx

 # restart mysql to make sure it has latest config
@@ -195,17 +168,11 @@ fi

 readonly mysql_root_password="password"
 mysqladmin -u root -ppassword password password # reset default root password
-if [[ "${ubuntu_version}" == "20.04" ]]; then
-    # mysql 8 added a new caching_sha2_password scheme which mysqljs does not support
-    mysql -u root -p${mysql_root_password} -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '${mysql_root_password}';"
-fi
 mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'

-# set HOME explicity, because it's not set when the installer calls it. this is done because
-# paths.js uses this env var and some of the migrate code requires box code
 echo "==> Migrating data"
 cd "${BOX_SRC_DIR}"
-if ! HOME=${HOME_DIR} BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@127.0.0.1/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up; then
+if ! BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@127.0.0.1/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up; then
    echo "DB migration failed"
    exit 1
 fi
@@ -220,15 +187,9 @@ else
    cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem"
 fi

-if [[ ! -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" ]]; then
-    # the key format in Ubuntu 20 changed, so we create keys in legacy format. for older ubuntu, just re-use the host keys
-    # see https://github.com/proftpd/proftpd/issues/793
-    if [[ "${ubuntu_version}" == "20.04" ]]; then
-        ssh-keygen -m PEM -t rsa -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" -q -N ""
-    else
-        cp /etc/ssh/ssh_host_rsa_key* ${BOX_DATA_DIR}/sftp/ssh
-    fi
-fi
+# old installations used to create appdata/<app>/redis which is now part of old backups and prevents restore
+echo "==> Cleaning up stale redis directories"
+find "${APPS_DATA_DIR}" -maxdepth 2 -type d -name redis -exec rm -rf {} +

 echo "==> Changing ownership"
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
@@ -245,7 +206,7 @@ chown "${USER}:${USER}" "${BOX_DATA_DIR}/mail"
 chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys

 echo "==> Starting Cloudron"
-systemctl start box
+systemctl start cloudron.target

 sleep 2 # give systemd sometime to start the processes

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-echo "==> Disabling THP"
-
-# https://docs.couchbase.com/server/current/install/thp-disable.html
-if [[ -d /sys/kernel/mm/transparent_hugepage ]]; then
-    echo "never" > /sys/kernel/mm/transparent_hugepage/enabled
-    echo "never" > /sys/kernel/mm/transparent_hugepage/defrag
-else
-    echo "==> kernel does not have THP"
-fi
-
@@ -6,34 +6,11 @@ echo "==> Setting up firewall"
 iptables -t filter -N CLOUDRON || true
 iptables -t filter -F CLOUDRON # empty any existing rules

-# first setup any user IP block lists
-ipset create cloudron_blocklist hash:net || true
-/home/yellowtent/box/src/scripts/setblocklist.sh
-
-iptables -t filter -A CLOUDRON -m set --match-set cloudron_blocklist src -j DROP
-# the DOCKER-USER chain is not cleared on docker restart
-if ! iptables -t filter -C DOCKER-USER -m set --match-set cloudron_blocklist src -j DROP; then
-    iptables -t filter -I DOCKER-USER 1 -m set --match-set cloudron_blocklist src -j DROP
-fi
-
-# allow related and establisted connections
-iptables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
-iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,25,80,202,443 -j ACCEPT # 202 is the alternate ssh port
-
-# whitelist any user ports
-ports_json="/home/yellowtent/boxdata/firewall/ports.json"
-if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(','))" 2>/dev/null); then
-    [[ -n "${allowed_tcp_ports}" ]] && iptables -A CLOUDRON -p tcp -m tcp -m multiport --dports "${allowed_tcp_ports}" -j ACCEPT
-fi
-
-if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(','))" 2>/dev/null); then
-    [[ -n "${allowed_tcp_ports}" ]] && iptables -A CLOUDRON -p udp -m udp -m multiport --dports "${allowed_tcp_ports}" -j ACCEPT
-fi
-
-# turn and stun service
-iptables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
+# NOTE: keep these in sync with src/apps.js validatePortBindings
+# allow ssh, http, https, ping, dns
+iptables -t filter -I CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
+# ssh is allowed alternately on port 202
+iptables -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,25,80,202,443,587,993,4190 -j ACCEPT

 iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
 iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
@@ -70,6 +47,8 @@ for port in 22 202; do
    iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
 done

+# TODO: move docker platform rules to platform.js so it can be specialized to rate limit only when destination is the mail container
+
 # docker translates (dnat) 25, 587, 993, 4190 in the PREROUTING step
 for port in 2525 4190 9993; do
    iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
@@ -85,10 +64,12 @@ for port in 3306 5432 6379 27017; do
    iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done

+# For ssh, http, https
 if ! iptables -t filter -C INPUT -j CLOUDRON_RATELIMIT 2>/dev/null; then
    iptables -t filter -I INPUT 1 -j CLOUDRON_RATELIMIT
 fi

-# Workaround issue where Docker insists on adding itself first in FORWARD table
+# For smtp, imap etc routed via docker/nat
+# Workaroud issue where Docker insists on adding itself first in FORWARD table
 iptables -D FORWARD -j CLOUDRON_RATELIMIT || true
 iptables -I FORWARD 1 -j CLOUDRON_RATELIMIT
@@ -10,19 +10,12 @@ if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
    fi
    echo "${ip}" > /tmp/.cloudron-motd-cache

-    if [[ ! -f /etc/cloudron/SETUP_TOKEN ]]; then
-        url="https://${ip}"
-    else
-        setupToken="$(cat /etc/cloudron/SETUP_TOKEN)"
-        url="https://${ip}/?setupToken=${setupToken}"
-    fi
-
    printf "\t\t\tWELCOME TO CLOUDRON\n"
    printf "\t\t\t-------------------\n"

-    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit ${url} on your browser and accept the self-signed certificate to finish setup."
-    printf "Cloudron overview - https://docs.cloudron.io/ \n"
-    printf "Cloudron setup - https://docs.cloudron.io/installation/#setup \n"
+    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit https://${ip} on your browser and accept the self-signed certificate to finish setup."
+    printf "Cloudron overview - https://cloudron.io/documentation/ \n"
+    printf "Cloudron setup - https://cloudron.io/documentation/installation/#setup \n"
 else
    printf "\t\t\tNOTE TO CLOUDRON ADMINS\n"
    printf "\t\t\t-----------------------\n"
@@ -30,7 +23,7 @@ else
    printf "Cloudron relies on and may break your installation. Ubuntu security updates\n"
    printf "are automatically installed on this server every night.\n"
    printf "\n"
-    printf "Read more at https://docs.cloudron.io/security/#os-updates\n"
+    printf "Read more at https://cloudron.io/documentation/security/#os-updates\n"
 fi

 printf "\nFor help and more information, visit https://forum.cloudron.io\n\n"
@@ -4,11 +4,6 @@ set -eu -o pipefail

 readonly APPS_SWAP_FILE="/apps.swap"

-if [[ -f "${APPS_SWAP_FILE}" ]]; then
-    echo "Swap file already exists at /apps.swap . Skipping"
-    exit
-fi
-
 # all sizes are in mb
 readonly physical_memory=$(LC_ALL=C free -m | awk '/Mem:/ { print $2 }')
 readonly swap_size=$((${physical_memory} > 4096 ? 4096 : ${physical_memory})) # min(RAM, 4GB) if you change this, fix enoughResourcesAvailable() in client.js
@@ -121,7 +121,7 @@ LoadPlugin memory
 #LoadPlugin netlink
 #LoadPlugin network
 #LoadPlugin nfs
-#LoadPlugin nginx
+LoadPlugin nginx
 #LoadPlugin notify_desktop
 #LoadPlugin notify_email
 #LoadPlugin ntpd
@@ -149,7 +149,7 @@ LoadPlugin memory
 #LoadPlugin statsd
 LoadPlugin swap
 #LoadPlugin table
-#LoadPlugin tail
+LoadPlugin tail
 #LoadPlugin tail_csv
 #LoadPlugin tcpconns
 #LoadPlugin teamspeak2
@@ -197,11 +197,42 @@ LoadPlugin write_graphite
   IgnoreSelected false
 </Plugin>

+<Plugin nginx>
+   URL "http://127.0.0.1/nginx_status"
+</Plugin>
+
 <Plugin swap>
   ReportByDevice false
   ReportBytes true
 </Plugin>

+<Plugin "tail">
+   <File "/var/log/nginx/error.log">
+     Instance "nginx"
+     <Match>
+       Regex ".*"
+       DSType "CounterInc"
+       Type counter
+       Instance "errors"
+     </Match>
+   </File>
+   <File "/var/log/nginx/access.log">
+     Instance "nginx"
+     <Match>
+       Regex ".*"
+       DSType "CounterInc"
+       Type counter
+       Instance "requests"
+     </Match>
+    <Match>
+      Regex " \".*\" [0-9]+ [0-9]+ ([0-9]+)"
+      DSType GaugeAverage
+      Type delay
+      Instance "response"
+    </Match>
+   </File>
+</Plugin>
+
 <Plugin python>
    # https://blog.dbrgn.ch/2017/3/10/write-a-collectd-python-plugin/
    ModulePath "/home/yellowtent/box/setup/start/collectd/"
@@ -6,7 +6,7 @@ disks = []

 def init():
    global disks
-    lines = [s.split() for s in subprocess.check_output(["df", "--type=ext4", "--output=source,target,size,used,avail"]).decode('utf-8').splitlines()]
+    lines = [s.split() for s in subprocess.check_output(["df", "--type=ext4", "--output=source,target,size,used,avail"]).splitlines()]
    disks = lines[1:] # strip header
    collectd.info('custom df plugin initialized with %s' % disks)

@@ -3,12 +3,10 @@ import collectd,os,subprocess,sys,re,time
 # https://www.programcreek.com/python/example/106897/collectd.register_read

 PATHS = [] # { name, dir, exclude }
-# there is a pattern in carbon/storage-schemas.conf which stores values every 12h for a year
 INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this

 def du(pathinfo):
-    # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
-    cmd = 'timeout 1800 du -DsB1 "{}"'.format(pathinfo['dir'])
+    cmd = 'timeout 1800 du -Dsb "{}"'.format(pathinfo['dir'])
    if pathinfo['exclude'] != '':
        cmd += ' --exclude "{}"'.format(pathinfo['exclude'])

@@ -28,7 +26,6 @@ def parseSize(size):

 def dockerSize():
    # use --format '{{json .}}' to dump the string. '{{if eq .Type "Images"}}{{.Size}}{{end}}' still creates newlines
-    # https://godoc.org/github.com/docker/go-units#HumanSize is used. so it's 1000 (KB) and not 1024 (KiB)
    cmd = 'timeout 1800 docker system df --format "{{.Size}}" | head -n1'
    try:
        size = subprocess.check_output(cmd, shell=True).strip().decode('utf-8')
@@ -10,7 +10,6 @@
 /home/yellowtent/platformdata/logs/redis-*/*.log
 /home/yellowtent/platformdata/logs/crash/*.log
 /home/yellowtent/platformdata/logs/collectd/*.log
-/home/yellowtent/platformdata/logs/turn/*.log
 /home/yellowtent/platformdata/logs/updater/*.log {
    # only keep one rotated file, we currently do not send that over the api
    rotate 1
@@ -18,7 +17,6 @@
    missingok
    # we never compress so we can simply tail the files
    nocompress
-    # this truncates the original log file and not the rotated one
    copytruncate
 }

@@ -6,7 +6,7 @@
 performance_schema=OFF
 max_connections=50
 # on ec2, without this we get a sporadic connection drop when doing the initial migration
-max_allowed_packet=64M
+max_allowed_packet=32M

 # https://mathiasbynens.be/notes/mysql-utf8mb4
 character-set-server = utf8mb4
@@ -1,18 +1,11 @@
 user www-data;

-# detect based on available CPU cores
-worker_processes  auto;
-
-# this is 4096 by default. See /proc/<PID>/limits and /etc/security/limits.conf
-# usually twice the worker_connections (one for uptsream, one for downstream)
-# see also LimitNOFILE=16384 in systemd drop-in
-worker_rlimit_nofile 8192;
+worker_processes  1;

 pid /run/nginx.pid;

 events {
-    # a single worker has these many simultaneous connections max
-    worker_connections  4096;
+    worker_connections  1024;
 }

 http {
@@ -43,5 +36,23 @@ http {
    # zones for rate limiting
    limit_req_zone $binary_remote_addr zone=admin_login:10m rate=10r/s; # 10 request a second

+
+    # default http server that returns 404 for any domain we are not listening on
+    server {
+        listen      80 default_server;
+        listen      [::]:80 default_server;
+        server_name does_not_match_anything;
+
+        # acme challenges (for app installation and re-configure when the vhost config does not exist)
+        location /.well-known/acme-challenge/ {
+            default_type text/plain;
+            alias /home/yellowtent/platformdata/acme/;
+        }
+
+        location / {
+            return 404;
+        }
+    }
+
    include applications/*.conf;
 }
@@ -50,15 +50,3 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartdocker.s
 Defaults!/home/yellowtent/box/src/scripts/restartunbound.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartunbound.sh

-Defaults!/home/yellowtent/box/src/scripts/rmmailbox.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmailbox.sh
-
-Defaults!/home/yellowtent/box/src/scripts/starttask.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD:SETENV: /home/yellowtent/box/src/scripts/starttask.sh
-
-Defaults!/home/yellowtent/box/src/scripts/stoptask.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/stoptask.sh
-
-Defaults!/home/yellowtent/box/src/scripts/setblocklist.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setblocklist.sh
-
@@ -1,21 +1,20 @@
 [Unit]
 Description=Cloudron Admin
 OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
 ; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
 BindsTo=systemd-journald.service
 After=mysql.service nginx.service
 ; As cloudron-resize-fs is a one-shot, the Wants= automatically ensures that the service *finishes*
 Wants=cloudron-resize-fs.service

-[Install]
-WantedBy=multi-user.target
-
 [Service]
 Type=idle
 WorkingDirectory=/home/yellowtent/box
 Restart=always
-ExecStart=/home/yellowtent/box/box.js
-Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production"
+; Systemd does not append logs when logging to files, we spawn a shell first and exec to replace it after setting up the pipes
+ExecStart=/bin/sh -c 'echo "Logging to /home/yellowtent/platformdata/logs/box.log"; exec /usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js >> /home/yellowtent/platformdata/logs/box.log 2>&1'
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
 ; kill apptask processes as well
 KillMode=control-group
 ; Do not kill this process on OOM. Children inherit this score. Do not set it to -1000 so that MemoryMax can keep working
@@ -1,15 +0,0 @@
-# https://docs.mongodb.com/manual/tutorial/transparent-huge-pages/
-[Unit]
-Description=Disable Transparent Huge Pages (THP)
-DefaultDependencies=no
-After=sysinit.target local-fs.target
-Before=docker.service
-
-[Service]
-Type=oneshot
-ExecStart="/home/yellowtent/box/setup/start/cloudron-disable-thp.sh"
-RemainAfterExit=yes
-
-[Install]
-WantedBy=basic.target
-
@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Smartserver
+Documentation=https://cloudron.io/documentation.html
+StopWhenUnneeded=true
+Requires=box.service
+After=box.service
+# AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target
@@ -2,7 +2,7 @@

 [Unit]
 Description=Unbound DNS Resolver
-After=network.target docker.service
+After=network.target

 [Service]
 PIDFile=/run/unbound.pid
@@ -1,7 +1,5 @@
 server:
-        port: 53
-        interface: 127.0.0.1
-        interface: 172.18.0.1
+        interface: 0.0.0.0
        do-ip6: no
        access-control: 127.0.0.1 allow
        access-control: 172.18.0.1/16 allow
@@ -4,4 +4,4 @@ set -eu -o pipefail

 echo "Stopping cloudron"

-systemctl stop box
+systemctl stop cloudron.target
@@ -1,32 +1,32 @@
 'use strict';

 exports = module.exports = {
-    get,
-    add,
-    exists,
-    del,
-    update,
-    getAll,
-    getPortBindings,
-    delPortBinding,
+    get: get,
+    getByHttpPort: getByHttpPort,
+    getByContainerId: getByContainerId,
+    add: add,
+    exists: exists,
+    del: del,
+    update: update,
+    getAll: getAll,
+    getPortBindings: getPortBindings,
+    delPortBinding: delPortBinding,

-    setAddonConfig,
-    getAddonConfig,
-    getAddonConfigByAppId,
-    getAddonConfigByName,
-    unsetAddonConfig,
-    unsetAddonConfigByAppId,
-    getAppIdByAddonConfigValue,
-    getByIpAddress,
+    setAddonConfig: setAddonConfig,
+    getAddonConfig: getAddonConfig,
+    getAddonConfigByAppId: getAddonConfigByAppId,
+    getAddonConfigByName: getAddonConfigByName,
+    unsetAddonConfig: unsetAddonConfig,
+    unsetAddonConfigByAppId: unsetAddonConfigByAppId,
+    getAppIdByAddonConfigValue: getAppIdByAddonConfigValue,

-    setHealth,
-    setTask,
-    getAppStoreIds,
+    setHealth: setHealth,
+    setTask: setTask,
+    getAppStoreIds: getAppStoreIds,

    // subdomain table types
    SUBDOMAIN_TYPE_PRIMARY: 'primary',
    SUBDOMAIN_TYPE_REDIRECT: 'redirect',
-    SUBDOMAIN_TYPE_ALIAS: 'alias',

    _clear: clear
 };
@@ -38,14 +38,17 @@ var assert = require('assert'),
    safe = require('safetydance'),
    util = require('util');

-const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
-    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
-    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson',
-    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp',
+var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
+    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'subdomains.subdomain AS location', 'subdomains.domain',
+    'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
+    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson',
+    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup',
    'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
    'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(',');

-const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
+var PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
+
+const SUBDOMAIN_FIELDS = [ 'appId', 'domain', 'subdomain', 'type' ].join(',');

 function postProcess(result) {
    assert.strictEqual(typeof result, 'object');
@@ -86,33 +89,16 @@ function postProcess(result) {
    result.sso = !!result.sso; // make it bool
    result.enableBackup = !!result.enableBackup; // make it bool
    result.enableAutomaticUpdate = !!result.enableAutomaticUpdate; // make it bool
-    result.proxyAuth = !!result.proxyAuth;

    assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
    result.debugMode = safe.JSON.parse(result.debugModeJson);
    delete result.debugModeJson;

-    assert(result.servicesConfigJson === null || typeof result.servicesConfigJson === 'string');
-    result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
-    delete result.servicesConfigJson;
-
-    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
-    delete result.subdomains;
-    delete result.domains;
-    delete result.subdomainTypes;
-
-    result.alternateDomains = [];
-    result.aliasDomains = [];
-    for (let i = 0; i < subdomainTypes.length; i++) {
-        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
-            result.location = subdomains[i];
-            result.domain = domains[i];
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
-            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
-            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
-        }
-    }
+    result.alternateDomains = result.alternateDomains || [];
+    result.alternateDomains.forEach(function (d) {
+        delete d.appId;
+        delete d.type;
+    });

    let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
    delete result.envNames;
@@ -122,67 +108,119 @@ function postProcess(result) {
        if (envNames[i]) result.env[envNames[i]] = envValues[i];
    }

-    let volumeIds = JSON.parse(result.volumeIds);
-    delete result.volumeIds;
-    let volumeReadOnlys = JSON.parse(result.volumeReadOnlys);
-    delete result.volumeReadOnlys;
-
-    result.mounts = volumeIds[0] === null ? [] : volumeIds.map((v, idx) => { return { volumeId: v, readOnly: !!volumeReadOnlys[idx] }; }); // NOTE: volumeIds is [null] when volumes of an app is empty
-
    result.error = safe.JSON.parse(result.errorJson);
    delete result.errorJson;

    result.taskId = result.taskId ? String(result.taskId) : null;
 }

-// each query simply join apps table with another table by id. we then join the full result together
-const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
-const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
-const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
-const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
-const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
-    + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
-    + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
-    + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
-    + ` LEFT JOIN (${MOUNTS_QUERY}) AS q4 on q4.id = apps.id`;
-
 function get(id, callback) {
    assert.strictEqual(typeof id, 'string');
    assert.strictEqual(typeof callback, 'function');

-    database.query(`${APPS_QUERY} WHERE apps.id = ?`, [ id ], function (error, result) {
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes, '
+        + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues'
+        + ' FROM apps'
+        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
+        + '  LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId'
+        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?'
+        + ' WHERE apps.id = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, id ], function (error, result) {
        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));

-        postProcess(result[0]);
+        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE appId = ? AND type = ?', [ id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));

-        callback(null, result[0]);
+            result[0].alternateDomains = alternateDomains;
+
+            postProcess(result[0]);
+
+            callback(null, result[0]);
+        });
    });
 }

-function getByIpAddress(ip, callback) {
-    assert.strictEqual(typeof ip, 'string');
+function getByHttpPort(httpPort, callback) {
+    assert.strictEqual(typeof httpPort, 'number');
    assert.strictEqual(typeof callback, 'function');

-    database.query(`${APPS_QUERY} WHERE apps.containerIp = ?`, [ ip ], function (error, result) {
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+    + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes,'
+    + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues'
+        + ' FROM apps'
+        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
+        + '  LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId'
+        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?'
+        + ' WHERE httpPort = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, httpPort ], function (error, result) {
        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));

-        postProcess(result[0]);
+        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE appId = ? AND type = ?', [ result[0].id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));

-        callback(null, result[0]);
+            result[0].alternateDomains = alternateDomains;
+            postProcess(result[0]);
+
+            callback(null, result[0]);
+        });
+    });
+}
+
+function getByContainerId(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes,'
+        + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues'
+        + ' FROM apps'
+        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
+        + '  LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId'
+        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?'
+        + ' WHERE containerId = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, containerId ], function (error, result) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+
+        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE appId = ? AND type = ?', [ result[0].id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            result[0].alternateDomains = alternateDomains;
+            postProcess(result[0]);
+
+            callback(null, result[0]);
+        });
    });
 }

 function getAll(callback) {
    assert.strictEqual(typeof callback, 'function');

-    database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ], function (error, results) {
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes,'
+        + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues'
+        + ' FROM apps'
+        + '  LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
+        + '  LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId'
+        + '  LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?'
+        + ' GROUP BY apps.id ORDER BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY ], function (error, results) {
        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));

-        results.forEach(postProcess);
+        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE type = ?', [ exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));

-        callback(null, results);
+            alternateDomains.forEach(function (d) {
+                var domain = results.find(function (a) { return d.appId === a.id; });
+                if (!domain) return;
+
+                domain.alternateDomains = domain.alternateDomains || [];
+                domain.alternateDomains.push(d);
+            });
+
+            results.forEach(postProcess);
+
+            callback(null, results);
+        });
    });
 }

@@ -254,15 +292,6 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
        });
    }

-    if (data.aliasDomains) {
-        data.aliasDomains.forEach(function (d) {
-            queries.push({
-                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
-            });
-        });
-    }
-
    database.transaction(queries, function (error) {
        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'no such domain'));
@@ -321,13 +350,12 @@ function del(id, callback) {
        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
        { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
-        { query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ] },
        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
    ];

    database.transaction(queries, function (error, results) {
        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[5].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
+        if (results[4].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));

        callback(null);
    });
@@ -360,7 +388,6 @@ function updateWithConstraints(id, app, constraints, callback) {
    assert(!('portBindings' in app) || typeof app.portBindings === 'object');
    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
    assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
-    assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
    assert(!('tags' in app) || Array.isArray(app.tags));
    assert(!('env' in app) || typeof app.env === 'object');

@@ -388,35 +415,22 @@ function updateWithConstraints(id, app, constraints, callback) {
    }

    if ('location' in app && 'domain' in app) { // must be updated together as they are unique together
-        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ]}); // all locations of an app must be updated together
-        queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, app.domain, app.location, exports.SUBDOMAIN_TYPE_PRIMARY ]});
-
-        if ('alternateDomains' in app) {
-            app.alternateDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
-            });
-        }
-
-        if ('aliasDomains' in app) {
-            app.aliasDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
-            });
-        }
+        queries.push({ query: 'UPDATE subdomains SET subdomain = ?, domain = ? WHERE appId = ? AND type = ?', args: [ app.location, app.domain, id, exports.SUBDOMAIN_TYPE_PRIMARY ]});
    }

-    if ('mounts' in app) {
-        queries.push({ query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ]});
-        app.mounts.forEach(function (m) {
-            queries.push({ query: 'INSERT INTO appMounts (appId, volumeId, readOnly) VALUES (?, ?, ?)', args: [ id, m.volumeId, m.readOnly ]});
+    if ('alternateDomains' in app) {
+        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ? AND type = ?', args: [ id, exports.SUBDOMAIN_TYPE_REDIRECT ]});
+        app.alternateDomains.forEach(function (d) {
+            queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
        });
    }

    var fields = [ ], values = [ ];
    for (var p in app) {
-        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig') {
+        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig') {
            fields.push(`${p}Json = ?`);
            values.push(JSON.stringify(app[p]));
-        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
+        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'env') {
            fields.push(p + ' = ?');
            values.push(app[p]);
        }
@@ -14,14 +14,13 @@ var appdb = require('./appdb.js'),
    util = require('util');

 exports = module.exports = {
-    run
+    run: run
 };

 const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
 const UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes

 const OOM_EVENT_LIMIT = 60 * 60 * 1000; // 60 minutes
-const gStartTime = new Date(); // time when apphealthmonitor was started
 let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago

 function debugApp(app) {
@@ -35,8 +34,7 @@ function setHealth(app, health, callback) {
    assert.strictEqual(typeof health, 'string');
    assert.strictEqual(typeof callback, 'function');

-    let now = new Date(), curHealth = app.health;
-    let healthTime = gStartTime > app.healthTime ? gStartTime : app.healthTime; // on box restart, clamp value to start time
+    let now = new Date(), healthTime = app.healthTime, curHealth = app.health;

    if (health === apps.HEALTH_HEALTHY) {
        healthTime = now;
@@ -75,19 +73,28 @@ function checkAppHealth(app, callback) {
    assert.strictEqual(typeof callback, 'function');

    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) {
+        debugApp(app, 'skipped. istate:%s rstate:%s', app.installationState, app.runState);
        return callback(null);
    }

    const manifest = app.manifest;

    docker.inspect(app.containerId, function (error, data) {
-        if (error || !data || !data.State) return setHealth(app, apps.HEALTH_ERROR, callback);
-        if (data.State.Running !== true) return setHealth(app, apps.HEALTH_DEAD, callback);
+        if (error || !data || !data.State) {
+            debugApp(app, 'Error inspecting container');
+            return setHealth(app, apps.HEALTH_ERROR, callback);
+        }
+
+        if (data.State.Running !== true) {
+            debugApp(app, 'exited');
+            return setHealth(app, apps.HEALTH_DEAD, callback);
+        }

        // non-appstore apps may not have healthCheckPath
        if (!manifest.healthCheckPath) return setHealth(app, apps.HEALTH_HEALTHY, callback);

-        const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
+        // poll through docker network instead of nginx to bypass any potential oauth proxy
+        var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
        superagent
            .get(healthCheckUrl)
            .set('Host', app.fqdn) // required for some apache configs with rewrite rules
@@ -96,8 +103,10 @@ function checkAppHealth(app, callback) {
            .timeout(HEALTHCHECK_INTERVAL)
            .end(function (error, res) {
                if (error && !error.response) {
+                    debugApp(app, 'not alive (network error): %s', error.message);
                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
-                } else if (res.statusCode >= 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
+                } else if (res.statusCode >= 400) { // 2xx and 3xx are ok
+                    debugApp(app, 'not alive : %s', error || res.status);
                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
                } else {
                    setHealth(app, apps.HEALTH_HEALTHY, callback);
@@ -120,7 +129,7 @@ function getContainerInfo(containerId, callback) {

 /*
    OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:2.0.0 /bin/bash
+        docker run -ti -m 100M cloudron/base:0.10.0 /bin/bash
        apt-get update && apt-get install stress
        stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
@@ -171,14 +180,18 @@ function processDockerEvents(intervalSecs, callback) {
 function processApp(callback) {
    assert.strictEqual(typeof callback, 'function');

-    apps.getAll(function (error, allApps) {
+    apps.getAll(function (error, result) {
        if (error) return callback(error);

-        async.each(allApps, checkAppHealth, function (error) {
-            const alive = allApps
-                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
+        async.each(result, checkAppHealth, function (error) {
+            if (error) console.error(error);

-            debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.` + (error ? ` ${error.reason}` : ''));
+            const alive = result
+                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; })
+                .map(a => a.fqdn)
+                .join(', ');
+
+            debug('apps alive: [%s]', alive);

            callback(null);
        });
@@ -193,7 +206,7 @@ function run(intervalSecs, callback) {
        processApp, // this is first because docker.getEvents seems to get 'stuck' sometimes
        processDockerEvents.bind(null, intervalSecs)
    ], function (error) {
-        if (error) debug(`run: could not check app health. ${error.message}`);
+        if (error) debug(error);

        callback();
    });
@@ -1,28 +1,31 @@
 'use strict';

 exports = module.exports = {
-    getFeatures,
+    getFeatures: getFeatures,

-    getApps,
-    getApp,
-    getAppVersion,
+    getApps: getApps,
+    getApp: getApp,
+    getAppVersion: getAppVersion,

-    trackBeginSetup,
-    trackFinishedSetup,
+    trackBeginSetup: trackBeginSetup,
+    trackFinishedSetup: trackFinishedSetup,

-    registerWithLoginCredentials,
+    registerWithLoginCredentials: registerWithLoginCredentials,
+    registerWithLicense: registerWithLicense,

-    purchaseApp,
-    unpurchaseApp,
+    purchaseApp: purchaseApp,
+    unpurchaseApp: unpurchaseApp,

-    getUserToken,
-    getSubscription,
-    isFreePlan,
+    getUserToken: getUserToken,
+    getSubscription: getSubscription,
+    isFreePlan: isFreePlan,

-    getAppUpdate,
-    getBoxUpdate,
+    sendAliveStatus: sendAliveStatus,

-    createTicket
+    getAppUpdate: getAppUpdate,
+    getBoxUpdate: getBoxUpdate,
+
+    createTicket: createTicket
 };

 var apps = require('./apps.js'),
@@ -31,30 +34,32 @@ var apps = require('./apps.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:appstore'),
+    domains = require('./domains.js'),
    eventlog = require('./eventlog.js'),
-    path = require('path'),
+    groups = require('./groups.js'),
+    mail = require('./mail.js'),
+    os = require('os'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
    semver = require('semver'),
    settings = require('./settings.js'),
    superagent = require('superagent'),
-    support = require('./support.js'),
+    users = require('./users.js'),
    util = require('util');

+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
 // These are the default options and will be adjusted once a subscription state is obtained
 // Keep in sync with appstore/routes/cloudrons.js
 let gFeatures = {
-    userMaxCount: 5,
-    userGroups: false,
-    userRoles: false,
-    domainMaxCount: 1,
-    externalLdap: false,
-    privateDockerRegistry: false,
-    branding: false,
-    support: false,
-    directoryConfig: false,
-    mailboxMaxCount: 5,
-    emailPremium: false
+    userMaxCount: null,
+    externalLdap: true,
+    eventLog: true,
+    privateDockerRegistry: true,
+    branding: true,
+    userManager: true,
+    multiAdmin: true,
+    support: true
 };

 // attempt to load feature cache in case appstore would be down
@@ -122,7 +127,7 @@ function registerUser(email, password, callback) {
    const url = settings.apiServerOrigin() + '/api/v1/register_user';
    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
+        if (result.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS));
        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${result.statusCode}`));

        callback(null);
@@ -230,8 +235,112 @@ function unpurchaseApp(appId, data, callback) {
    });
 }

-function getBoxUpdate(options, callback) {
-    assert.strictEqual(typeof options, 'object');
+function sendAliveStatus(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    let allSettings, allDomains, mailDomains, loginEvents, userCount, groupCount;
+
+    async.series([
+        function (callback) {
+            settings.getAll(function (error, result) {
+                if (error) return callback(error);
+                allSettings = result;
+                callback();
+            });
+        },
+        function (callback) {
+            domains.getAll(function (error, result) {
+                if (error) return callback(error);
+                allDomains = result;
+                callback();
+            });
+        },
+        function (callback) {
+            mail.getDomains(function (error, result) {
+                if (error) return callback(error);
+                mailDomains = result;
+                callback();
+            });
+        },
+        function (callback) {
+            eventlog.getAllPaged([ eventlog.ACTION_USER_LOGIN ], null, 1, 1, function (error, result) {
+                if (error) return callback(error);
+                loginEvents = result;
+                callback();
+            });
+        },
+        function (callback) {
+            users.count(function (error, result) {
+                if (error) return callback(error);
+                userCount = result;
+                callback();
+            });
+        },
+        function (callback) {
+            groups.count(function (error, result) {
+                if (error) return callback(error);
+                groupCount = result;
+                callback();
+            });
+        }
+    ], function (error) {
+        if (error) return callback(error);
+
+        var backendSettings = {
+            backupConfig: {
+                provider: allSettings[settings.BACKUP_CONFIG_KEY].provider,
+                hardlinks: !allSettings[settings.BACKUP_CONFIG_KEY].noHardlinks
+            },
+            domainConfig: {
+                count: allDomains.length,
+                domains: Array.from(new Set(allDomains.map(function (d) { return { domain: d.domain, provider: d.provider }; })))
+            },
+            mailConfig: {
+                outboundCount: mailDomains.length,
+                inboundCount: mailDomains.filter(function (d) { return d.enabled; }).length,
+                catchAllCount: mailDomains.filter(function (d) { return d.catchAll.length !== 0; }).length,
+                relayProviders: Array.from(new Set(mailDomains.map(function (d) { return d.relay.provider; })))
+            },
+            userCount: userCount,
+            groupCount: groupCount,
+            appAutoupdatePattern: allSettings[settings.APP_AUTOUPDATE_PATTERN_KEY],
+            boxAutoupdatePattern: allSettings[settings.BOX_AUTOUPDATE_PATTERN_KEY],
+            timeZone: allSettings[settings.TIME_ZONE_KEY],
+            sysinfoProvider: allSettings[settings.SYSINFO_CONFIG_KEY].provider
+        };
+
+        var data = {
+            version: constants.VERSION,
+            adminFqdn: settings.adminFqdn(),
+            provider: settings.provider(),
+            backendSettings: backendSettings,
+            machine: {
+                cpus: os.cpus(),
+                totalmem: os.totalmem()
+            },
+            events: {
+                lastLogin: loginEvents[0] ? (new Date(loginEvents[0].creationTime).getTime()) : 0
+            }
+        };
+
+        getCloudronToken(function (error, token) {
+            if (error) return callback(error);
+
+            const url = `${settings.apiServerOrigin()}/api/v1/alive`;
+            superagent.post(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
+                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+                if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Sending alive status failed. %s %j', result.status, result.body)));
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function getBoxUpdate(callback) {
    assert.strictEqual(typeof callback, 'function');

    getCloudronToken(function (error, token) {
@@ -239,23 +348,17 @@ function getBoxUpdate(options, callback) {

        const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;

-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            automatic: options.automatic
-        };
-
-        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
+        superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION }).timeout(10 * 1000).end(function (error, result) {
            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 204) return callback(null, null); // no update
+            if (result.statusCode === 204) return callback(null); // no update
            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));

            var updateInfo = result.body;

            if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', result.statusCode, result.text)));
+                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Invalid update version: %s %s', result.statusCode, result.text)));
            }

            // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
@@ -271,24 +374,16 @@ function getBoxUpdate(options, callback) {
    });
 }

-function getAppUpdate(app, options, callback) {
+function getAppUpdate(app, callback) {
    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

    getCloudronToken(function (error, token) {
        if (error) return callback(error);

        const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            appId: app.appStoreId,
-            appVersion: app.manifest.version,
-            automatic: options.automatic
-        };

-        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
+        superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION, appId: app.appStoreId, appVersion: app.manifest.version }).timeout(10 * 1000).end(function (error, result) {
            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
@@ -306,9 +401,7 @@ function getAppUpdate(app, options, callback) {
                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', result.statusCode, result.text)));
            }

-            updateInfo.unstable = !!updateInfo.unstable;
-
-            // { id, creationDate, manifest, unstable }
+            // { id, creationDate, manifest }
            callback(null, updateInfo);
        });
    });
@@ -322,7 +415,7 @@ function registerCloudron(data, callback) {

    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${result.statusCode} ${error.message}`));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${error.message}`));

        // cloudronId, token, licenseKey
        if (!result.body.cloudronId) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id'));
@@ -345,16 +438,18 @@ function registerCloudron(data, callback) {

 // This works without a Cloudron token as this Cloudron was not yet registered
 let gBeginSetupAlreadyTracked = false;
-function trackBeginSetup() {
+function trackBeginSetup(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
    // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
    if (gBeginSetupAlreadyTracked) return;
    gBeginSetupAlreadyTracked = true;

    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;

-    superagent.post(url).send({}).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return debug(`trackBeginSetup: ${error.message}`);
-        if (result.statusCode !== 200) return debug(`trackBeginSetup: ${result.statusCode} ${error.message}`);
+    superagent.post(url).send({ provider }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
    });
 }

@@ -365,8 +460,23 @@ function trackFinishedSetup(domain) {
    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;

    superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return debug(`trackFinishedSetup: ${error.message}`);
-        if (result.statusCode !== 200) return debug(`trackFinishedSetup: ${result.statusCode} ${error.message}`);
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
+function registerWithLicense(license, domain, callback) {
+    assert.strictEqual(typeof license, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getCloudronToken(function (error, token) {
+        if (token) return callback(new BoxError(BoxError.CONFLICT));
+
+        const provider = settings.provider();
+        const version = constants.VERSION;
+
+        registerCloudron({ license, domain, provider, version }, callback);
    });
 }

@@ -381,7 +491,7 @@ function registerWithLoginCredentials(options, callback) {
    }

    getCloudronToken(function (error, token) {
-        if (token) return callback(new BoxError(BoxError.CONFLICT, 'Cloudron is already registered'));
+        if (token) return callback(new BoxError(BoxError.CONFLICT));

        maybeSignup(function (error) {
            if (error) return callback(error);
@@ -389,7 +499,7 @@ function registerWithLoginCredentials(options, callback) {
            login(options.email, options.password, options.totpToken || '', function (error, result) {
                if (error) return callback(error);

-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
+                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, provider: settings.provider(), version: constants.VERSION, purpose: options.purpose || '' }, callback);
            });
        });
    });
@@ -410,55 +520,26 @@ function createTicket(info, auditSource, callback) {
        apps.get(info.appId, callback);
    }

-    function enableSshIfNeeded(callback) {
-        if (!info.enableSshSupport) return callback();
-
-        support.enableRemoteSupport(true, auditSource, function (error) {
-            // ensure we can at least get the ticket through
-            if (error) debug('Unable to enable SSH support.', error);
-
-            callback();
-        });
-    }
-
    getCloudronToken(function (error, token) {
        if (error) return callback(error);

-        enableSshIfNeeded(function (error) {
-            if (error) return callback(error);
+        collectAppInfoIfNeeded(function (error, result) {
+            if (error) console.error('Unable to get app info', error);
+            if (result) info.app = result;

-            collectAppInfoIfNeeded(function (error, app) {
-                if (error) return callback(error);
-                if (app) info.app = app;
+            let url = settings.apiServerOrigin() + '/api/v1/ticket';

-                info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
+            info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets

-                var req = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
-                    .query({ accessToken: token })
-                    .timeout(30 * 1000);
+            superagent.post(url).query({ accessToken: token }).send(info).timeout(10 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));

-                // either send as JSON through body or as multipart, depending on attachments
-                if (info.app) {
-                    req.field('infoJSON', JSON.stringify(info));
+                eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);

-                    apps.getLocalLogfilePaths(info.app).forEach(function (filePath) {
-                        var logs = safe.child_process.execSync(`tail --lines=1000 ${filePath}`);
-                        if (logs) req.attach(path.basename(filePath), logs, path.basename(filePath));
-                    });
-                } else {
-                    req.send(info);
-                }
-
-                req.end(function (error, result) {
-                    if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                    if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                    if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                    if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
-
-                    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
-
-                    callback(null, { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
-                });
+                callback(null, { message: `An email for sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
            });
        });
    });
@@ -474,7 +555,7 @@ function getApps(callback) {
            if (error) return callback(error);

            const url = `${settings.apiServerOrigin()}/api/v1/apps`;
-            superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable }).timeout(30 * 1000).end(function (error, result) {
+            superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable }).timeout(10 * 1000).end(function (error, result) {
                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
@@ -509,7 +590,7 @@ function getAppVersion(appId, version, callback) {
            let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
            if (version !== 'latest') url += `/versions/${version}`;

-            superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+            superagent.get(url).query({ accessToken: token }).timeout(10 * 1000).end(function (error, result) {
                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -6,6 +6,7 @@ exports = module.exports = {
    run: run,

    // exported for testing
+    _reserveHttpPort: reserveHttpPort,
    _configureReverseProxy: configureReverseProxy,
    _unconfigureReverseProxy: unconfigureReverseProxy,
    _createAppDir: createAppDir,
@@ -16,7 +17,10 @@ exports = module.exports = {
    _waitForDnsPropagation: waitForDnsPropagation
 };

-const appdb = require('./appdb.js'),
+require('supererror')({ splatchError: true });
+
+var addons = require('./addons.js'),
+    appdb = require('./appdb.js'),
    apps = require('./apps.js'),
    assert = require('assert'),
    async = require('async'),
@@ -32,15 +36,15 @@ const appdb = require('./appdb.js'),
    ejs = require('ejs'),
    eventlog = require('./eventlog.js'),
    fs = require('fs'),
-    iputils = require('./iputils.js'),
    manifestFormat = require('cloudron-manifestformat'),
+    mkdirp = require('mkdirp'),
+    net = require('net'),
    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    reverseProxy = require('./reverseproxy.js'),
    rimraf = require('rimraf'),
    safe = require('safetydance'),
-    services = require('./services.js'),
    settings = require('./settings.js'),
    shell = require('./shell.js'),
    superagent = require('superagent'),
@@ -88,16 +92,22 @@ function updateApp(app, values, callback) {
    });
 }

-function allocateContainerIp(app, callback) {
+function reserveHttpPort(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

-    async.retry({ times: 10 }, function (retryCallback) {
-        const iprange = iputils.intFromIp('172.18.20.255') - iputils.intFromIp('172.18.16.1');
-        let rnd = Math.floor(Math.random() * iprange);
-        const containerIp = iputils.ipFromInt(iputils.intFromIp('172.18.16.1') + rnd);
-        updateApp(app, { containerIp }, retryCallback);
-    }, callback);
+    let server = net.createServer();
+    server.listen(0, function () {
+        let port = server.address().port;
+
+        updateApp(app, { httpPort: port }, function (error) {
+            server.close(function (/* closeError */) {
+                if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Failed to allocate http port ${port}: ${error.message}`));
+
+                callback(null);
+            });
+        });
+    });
 }

 function configureReverseProxy(app, callback) {
@@ -166,7 +176,7 @@ function createAppDir(app, callback) {
    assert.strictEqual(typeof callback, 'function');

    const appDir = path.join(paths.APPS_DATA_DIR, app.id);
-    fs.mkdir(appDir, { recursive: true }, function (error) {
+    mkdirp(appDir, function (error) {
        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error creating directory: ${error.message}`, { appDir }));

        callback(null);
@@ -341,9 +351,9 @@ function registerSubdomains(app, overwrite, callback) {
    sysinfo.getServerIp(function (error, ip) {
        if (error) return callback(error);

-        const allDomains = [ { subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains);
+        const allDomains = [ { subdomain: app.location, domain: app.domain }].concat(app.alternateDomains);

-        debugApp(app, `registerSubdomain: Will register ${JSON.stringify(allDomains)}`);
+        debug(`registerSubdomain: Will register ${JSON.stringify(allDomains)}`);

        async.eachSeries(allDomains, function (domain, iteratorDone) {
            async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
@@ -363,7 +373,7 @@ function registerSubdomains(app, overwrite, callback) {

                    domains.upsertDnsRecords(domain.subdomain, domain.domain, 'A', [ ip ], function (error) {
                        if (error && (error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                            debugApp(app, 'registerSubdomains: Upsert error. Will retry.', error.message);
+                            debug('registerSubdomains: Upsert error. Will retry.', error.message);
                            return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain })); // try again
                        }

@@ -394,7 +404,7 @@ function unregisterSubdomains(app, allDomains, callback) {
                domains.removeDnsRecords(domain.subdomain, domain.domain, 'A', [ ip ], function (error) {
                    if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, null);
                    if (error && (error.reason === BoxError.SBUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                        debugApp(app, 'registerSubdomains: Remove error. Will retry.', error.message);
+                        debug('registerSubdomains: Remove error. Will retry.', error.message);
                        return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain })); // try again
                    }

@@ -424,8 +434,8 @@ function waitForDnsPropagation(app, callback) {
        domains.waitForDnsRecord(app.location, app.domain, 'A', ip, { times: 240 }, function (error) {
            if (error) return callback(new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: app.location, domain: app.domain }));

-            // now wait for alternateDomains and aliasDomains, if any
-            async.eachSeries(app.alternateDomains.concat(app.aliasDomains), function (domain, iteratorCallback) {
+            // now wait for alternateDomains, if any
+            async.eachSeries(app.alternateDomains, function (domain, iteratorCallback) {
                domains.waitForDnsRecord(domain.subdomain, domain.domain, 'A', ip, { times: 240 }, function (error) {
                    if (error) return callback(new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: domain.subdomain, domain: domain.domain }));

@@ -444,7 +454,7 @@ function moveDataDir(app, targetDir, callback) {
    let resolvedSourceDir = apps.getDataDir(app, app.dataDir);
    let resolvedTargetDir = apps.getDataDir(app, targetDir);

-    debugApp(app, `moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);
+    debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`);

    if (resolvedSourceDir === resolvedTargetDir) return callback();

@@ -477,8 +487,6 @@ function downloadImage(manifest, callback) {
 }

 function startApp(app, callback){
-    debugApp(app, 'startApp: starting container');
-
    if (app.runState === apps.RSTATE_STOPPED) return callback();

    docker.startContainer(app.id, callback);
@@ -512,7 +520,7 @@ function install(app, args, progressCallback, callback) {
                addonsToRemove = app.manifest.addons;
            }

-            services.teardownAddons(app, addonsToRemove, next);
+            addons.teardownAddons(app, addonsToRemove, next);
        },

        function deleteAppDirIfNeeded(done) {
@@ -527,8 +535,7 @@ function install(app, args, progressCallback, callback) {
            docker.deleteImage(oldManifest, done);
        },

-        // allocating container ip here, lets the users "repair" an app if allocation fails at appdb.add time
-        allocateContainerIp.bind(null, app),
+        reserveHttpPort.bind(null, app),

        progressCallback.bind(null, { percent: 20, message: 'Downloading icon' }),
        downloadIcon.bind(null, app),
@@ -546,24 +553,24 @@ function install(app, args, progressCallback, callback) {
            if (!restoreConfig) {
                async.series([
                    progressCallback.bind(null, { percent: 60, message: 'Setting up addons' }),
-                    services.setupAddons.bind(null, app, app.manifest.addons),
+                    addons.setupAddons.bind(null, app, app.manifest.addons),
                ], next);
            } else if (!restoreConfig.backupId) { // in-place import
                async.series([
                    progressCallback.bind(null, { percent: 60, message: 'Importing addons in-place' }),
-                    services.setupAddons.bind(null, app, app.manifest.addons),
-                    services.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
-                    services.restoreAddons.bind(null, app, app.manifest.addons),
+                    addons.setupAddons.bind(null, app, app.manifest.addons),
+                    addons.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
+                    addons.restoreAddons.bind(null, app, app.manifest.addons),
                ], next);
            } else {
                async.series([
                    progressCallback.bind(null, { percent: 65, message: 'Download backup and restoring addons' }),
-                    services.setupAddons.bind(null, app, app.manifest.addons),
-                    services.clearAddons.bind(null, app, app.manifest.addons),
+                    addons.setupAddons.bind(null, app, app.manifest.addons),
+                    addons.clearAddons.bind(null, app, app.manifest.addons),
                    backups.downloadApp.bind(null, app, restoreConfig, (progress) => {
                        progressCallback({ percent: 65, message: progress.message });
                    }),
-                    services.restoreAddons.bind(null, app, app.manifest.addons)
+                    addons.restoreAddons.bind(null, app, app.manifest.addons)
                ], next);
            }
        },
@@ -598,7 +605,7 @@ function backup(app, args, progressCallback, callback) {

    async.series([
        progressCallback.bind(null, { percent: 10, message: 'Backing up' }),
-        backups.backupApp.bind(null, app, { snapshotOnly: !!args.snapshotOnly }, (progress) => {
+        backups.backupApp.bind(null, app, { /* options */ }, (progress) => {
            progressCallback({ percent: 30, message: progress.message });
        }),

@@ -626,7 +633,7 @@ function create(app, args, progressCallback, callback) {

        // FIXME: re-setup addons only because sendmail addon to re-inject env vars on mailboxName change
        progressCallback.bind(null, { percent: 30, message: 'Setting up addons' }),
-        services.setupAddons.bind(null, app, app.manifest.addons),
+        addons.setupAddons.bind(null, app, app.manifest.addons),

        progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
        createContainer.bind(null, app),
@@ -663,12 +670,6 @@ function changeLocation(app, args, progressCallback, callback) {
                return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
            });

-            if (oldConfig.aliasDomains) {
-                obsoleteDomains = obsoleteDomains.concat(oldConfig.aliasDomains.filter(function (o) {
-                    return !app.aliasDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
-                }));
-            }
-
            if (locationChanged) obsoleteDomains.push({ subdomain: oldConfig.location, domain: oldConfig.domain });

            if (obsoleteDomains.length === 0) return next();
@@ -681,7 +682,7 @@ function changeLocation(app, args, progressCallback, callback) {

        // re-setup addons since they rely on the app's fqdn (e.g oauth)
        progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        services.setupAddons.bind(null, app, app.manifest.addons),
+        addons.setupAddons.bind(null, app, app.manifest.addons),

        progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
        createContainer.bind(null, app),
@@ -723,7 +724,7 @@ function migrateDataDir(app, args, progressCallback, callback) {

        // re-setup addons since this creates the localStorage volume
        progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        services.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),
+        addons.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),

        progressCallback.bind(null, { percent: 60, message: 'Moving data dir' }),
        moveDataDir.bind(null, app, newDataDir),
@@ -740,8 +741,7 @@ function migrateDataDir(app, args, progressCallback, callback) {
            debugApp(app, 'error migrating data dir : %s', error);
            return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
        }
-
-        callback();
+        callback(null);
    });
 }

@@ -756,6 +756,7 @@ function configure(app, args, progressCallback, callback) {
        progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }),
        unconfigureReverseProxy.bind(null, app),
        deleteContainers.bind(null, app, { managedOnly: true }),
+        reserveHttpPort.bind(null, app),

        progressCallback.bind(null, { percent: 20, message: 'Downloading icon' }),
        downloadIcon.bind(null, app),
@@ -768,7 +769,7 @@ function configure(app, args, progressCallback, callback) {

        // re-setup addons since they rely on the app's fqdn (e.g oauth)
        progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        services.setupAddons.bind(null, app, app.manifest.addons),
+        addons.setupAddons.bind(null, app, app.manifest.addons),

        progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
        createContainer.bind(null, app),
@@ -785,11 +786,11 @@ function configure(app, args, progressCallback, callback) {
            debugApp(app, 'error reconfiguring : %s', error);
            return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
        }
-
-        callback();
+        callback(null);
    });
 }

+// nginx configuration is skipped because app.httpPort is expected to be available
 function update(app, args, progressCallback, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof args, 'object');
@@ -801,10 +802,7 @@ function update(app, args, progressCallback, callback) {

    // app does not want these addons anymore
    // FIXME: this does not handle option changes (like multipleDatabases)
-    const unusedAddons = _.omit(app.manifest.addons, Object.keys(updateConfig.manifest.addons));
-    const httpPathsChanged = app.manifest.httpPaths !== updateConfig.manifest.httpPaths;
-    const httpPortChanged = app.manifest.httpPort !== updateConfig.manifest.httpPort;
-    const proxyAuthChanged = !_.isEqual(safe.query(app.manifest, 'addons.proxyAuth'), safe.query(updateConfig.manifest, 'addons.proxyAuth'));
+    var unusedAddons = _.omit(app.manifest.addons, Object.keys(updateConfig.manifest.addons));

    async.series([
        // this protects against the theoretical possibility of an app being marked for update from
@@ -843,7 +841,7 @@ function update(app, args, progressCallback, callback) {
        },

        // only delete unused addons after backup
-        services.teardownAddons.bind(null, app, unusedAddons),
+        addons.teardownAddons.bind(null, app, unusedAddons),

        // free unused ports
        function (next) {
@@ -855,7 +853,7 @@ function update(app, args, progressCallback, callback) {
                if (newTcpPorts[portName] || newUdpPorts[portName]) return callback(null); // port still in use

                appdb.delPortBinding(currentPorts[portName], apps.PORT_TYPE_TCP, function (error) {
-                    if (error && error.reason === BoxError.NOT_FOUND) debugApp(app, 'update: portbinding does not exist in database', error);
+                    if (error && error.reason === BoxError.NOT_FOUND) console.error('Portbinding does not exist in database.');
                    else if (error) return next(error);

                    // also delete from app object for further processing (the db is updated in the next step)
@@ -871,21 +869,14 @@ function update(app, args, progressCallback, callback) {
        progressCallback.bind(null, { percent: 45, message: 'Downloading icon' }),
        downloadIcon.bind(null, app),

-        progressCallback.bind(null, { percent: 60, message: 'Updating addons' }),
-        services.setupAddons.bind(null, app, updateConfig.manifest.addons),
+        progressCallback.bind(null, { percent: 70, message: 'Updating addons' }),
+        addons.setupAddons.bind(null, app, updateConfig.manifest.addons),

-        progressCallback.bind(null, { percent: 70, message: 'Creating container' }),
+        progressCallback.bind(null, { percent: 80, message: 'Creating container' }),
        createContainer.bind(null, app),

        startApp.bind(null, app),

-        progressCallback.bind(null, { percent: 90, message: 'Configuring reverse proxy' }),
-        function (next) {
-            if (!httpPathsChanged && !proxyAuthChanged && !httpPortChanged) return next();
-
-            configureReverseProxy(app, next);
-        },
-
        progressCallback.bind(null, { percent: 100, message: 'Done' }),
        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, updateTime: new Date() })
    ], function seriesDone(error) {
@@ -908,17 +899,11 @@ function start(app, args, progressCallback, callback) {
    assert.strictEqual(typeof callback, 'function');

    async.series([
-        progressCallback.bind(null, { percent: 10, message: 'Starting app services' }),
-        services.startAppServices.bind(null, app),
-
-        progressCallback.bind(null, { percent: 35, message: 'Starting container' }),
+        progressCallback.bind(null, { percent: 20, message: 'Starting container' }),
        docker.startContainer.bind(null, app.id),

-        progressCallback.bind(null, { percent: 60, message: 'Adding collectd profile' }),
-        addCollectdProfile.bind(null, app),
-
        // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
-        progressCallback.bind(null, { percent: 80, message: 'Configuring reverse proxy' }),
+        progressCallback.bind(null, { percent: 60, message: 'Configuring reverse proxy' }),
        configureReverseProxy.bind(null, app),

        progressCallback.bind(null, { percent: 100, message: 'Done' }),
@@ -942,12 +927,6 @@ function stop(app, args, progressCallback, callback) {
        progressCallback.bind(null, { percent: 20, message: 'Stopping container' }),
        docker.stopContainers.bind(null, app.id),

-        progressCallback.bind(null, { percent: 50, message: 'Stopping app services' }),
-        services.stopAppServices.bind(null, app),
-
-        progressCallback.bind(null, { percent: 80, message: 'Removing collectd profile' }),
-        removeCollectdProfile.bind(null, app),
-
        progressCallback.bind(null, { percent: 100, message: 'Done' }),
        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
    ], function seriesDone(error) {
@@ -992,20 +971,18 @@ function uninstall(app, args, progressCallback, callback) {
        deleteContainers.bind(null, app, {}),

        progressCallback.bind(null, { percent: 30, message: 'Teardown addons' }),
-        services.teardownAddons.bind(null, app, app.manifest.addons),
+        addons.teardownAddons.bind(null, app, app.manifest.addons),

-        progressCallback.bind(null, { percent: 40, message: 'Cleanup file manager' }),
-
-        progressCallback.bind(null, { percent: 50, message: 'Deleting app data directory' }),
+        progressCallback.bind(null, { percent: 40, message: 'Deleting app data directory' }),
        deleteAppDir.bind(null, app, { removeDirectory: true }),

-        progressCallback.bind(null, { percent: 60, message: 'Deleting image' }),
+        progressCallback.bind(null, { percent: 50, message: 'Deleting image' }),
        docker.deleteImage.bind(null, app.manifest),

-        progressCallback.bind(null, { percent: 70, message: 'Unregistering domains' }),
-        unregisterSubdomains.bind(null, app, [ { subdomain: app.location, domain: app.domain } ].concat(app.alternateDomains).concat(app.aliasDomains)),
+        progressCallback.bind(null, { percent: 60, message: 'Unregistering domains' }),
+        unregisterSubdomains.bind(null, app, [ { subdomain: app.location, domain: app.domain } ].concat(app.alternateDomains)),

-        progressCallback.bind(null, { percent: 80, message: 'Cleanup icon' }),
+        progressCallback.bind(null, { percent: 70, message: 'Cleanup icon' }),
        removeIcon.bind(null, app),

        progressCallback.bind(null, { percent: 90, message: 'Cleanup logs' }),
@@ -1061,8 +1038,6 @@ function run(appId, args, progressCallback, callback) {
            return stop(app, args, progressCallback, callback);
        case apps.ISTATE_PENDING_RESTART:
            return restart(app, args, progressCallback, callback);
-        case apps.ISTATE_INSTALLED: // can only happen when we have a bug in our code while testing/development
-            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, callback);
        default:
            debugApp(app, 'apptask launched with invalid command');
            return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState));
@@ -1,7 +1,7 @@
 'use strict';

 exports = module.exports = {
-    scheduleTask
+    scheduleTask: scheduleTask
 };

 let assert = require('assert'),
@@ -12,8 +12,6 @@ let assert = require('assert'),
    safe = require('safetydance'),
    path = require('path'),
    paths = require('./paths.js'),
-    scheduler = require('./scheduler.js'),
-    services = require('./services.js'),
    tasks = require('./tasks.js');

 let gActiveTasks = { }; // indexed by app id
@@ -37,10 +35,9 @@ function initializeSync() {
 }

 // callback is called when task is finished
-function scheduleTask(appId, taskId, options, callback) {
+function scheduleTask(appId, taskId, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof taskId, 'string');
-    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

    if (!gInitialized) initializeSync();
@@ -52,7 +49,7 @@ function scheduleTask(appId, taskId, options, callback) {
    if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
        debug(`Reached concurrency limit, queueing task id ${taskId}`);
        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, options, callback });
+        gPendingTasks.push({ appId, taskId, callback });
        return;
    }

@@ -61,7 +58,7 @@ function scheduleTask(appId, taskId, options, callback) {
    if (lockError) {
        debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, options, callback });
+        gPendingTasks.push({ appId, taskId, callback });
        return;
    }

@@ -71,17 +68,11 @@ function scheduleTask(appId, taskId, options, callback) {

    if (!fs.existsSync(path.dirname(logFile))) safe.fs.mkdirSync(path.dirname(logFile)); // ensure directory

-    scheduler.suspendJobs(appId);
-
-    tasks.startTask(taskId, Object.assign(options, { logFile }), function (error, result) {
+    tasks.startTask(taskId, { logFile, timeout: 20 * 60 * 60 * 1000 /* 20 hours */ }, function (error, result) {
        callback(error, result);

        delete gActiveTasks[appId];
        locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
-
-        // post app task hooks
-        services.rebuildService('sftp', error => { if (error) debug('Unable to rebuild sftp:', error); });
-        scheduler.resumeJobs(appId);
    });
 }

@@ -91,6 +82,6 @@ function startNextTask() {
    assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);

    const t = gPendingTasks.shift();
-    scheduleTask(t.appId, t.taskId, t.options, t.callback);
+    scheduleTask(t.appId, t.taskId, t.callback);
 }

@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<clientConfig version="1.1">
-  <emailProvider id="<%= domain %>">
-    <domain><%= domain %></domain>
-    <displayName>Cloudron Mail</displayName>
-    <displayShortName>Cloudron</displayShortName>
-    <incomingServer type="imap">
-      <hostname><%= mailFqdn %></hostname>
-      <port>993</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </incomingServer>
-    <outgoingServer type="smtp">
-      <hostname><%= mailFqdn %></hostname>
-      <port>587</port>
-      <socketType>STARTTLS</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-      <addThisServer>true</addThisServer>
-    </outgoingServer>
-
-    <documentation url="http://cloudron.io/email/#autodiscover">
-       <descr lang="en">Cloudron Email</descr>
-    </documentation>
-
-  </emailProvider>
-</clientConfig>
-
@@ -6,20 +6,27 @@ var assert = require('assert'),
    safe = require('safetydance'),
    util = require('util');

-var BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
+var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs' ];

 exports = module.exports = {
-    add,
+    add: add,

-    getByTypePaged,
-    getByIdentifierPaged,
-    getByIdentifierAndStatePaged,
+    getByTypeAndStatePaged: getByTypeAndStatePaged,
+    getByTypePaged: getByTypePaged,

-    get,
-    del,
-    update,
+    get: get,
+    del: del,
+    update: update,
+    getByAppIdPaged: getByAppIdPaged,

-    _clear: clear
+    _clear: clear,
+
+    BACKUP_TYPE_APP: 'app',
+    BACKUP_TYPE_BOX: 'box',
+
+    BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI?
+    BACKUP_STATE_CREATING: 'creating',
+    BACKUP_STATE_ERROR: 'error'
 };

 function postProcess(result) {
@@ -31,15 +38,15 @@ function postProcess(result) {
    delete result.manifestJson;
 }

-function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
+function getByTypeAndStatePaged(type, state, page, perPage, callback) {
+    assert(type === exports.BACKUP_TYPE_APP || type === exports.BACKUP_TYPE_BOX);
    assert.strictEqual(typeof state, 'string');
    assert(typeof page === 'number' && page > 0);
    assert(typeof perPage === 'number' && perPage > 0);
    assert.strictEqual(typeof callback, 'function');

-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, state, (page-1)*perPage, perPage ], function (error, results) {
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ type, state, (page-1)*perPage, perPage ], function (error, results) {
            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));

            results.forEach(function (result) { postProcess(result); });
@@ -49,7 +56,7 @@ function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback
 }

 function getByTypePaged(type, page, perPage, callback) {
-    assert.strictEqual(typeof type, 'string');
+    assert(type === exports.BACKUP_TYPE_APP || type === exports.BACKUP_TYPE_BOX);
    assert(typeof page === 'number' && page > 0);
    assert(typeof perPage === 'number' && perPage > 0);
    assert.strictEqual(typeof callback, 'function');
@@ -64,14 +71,15 @@ function getByTypePaged(type, page, perPage, callback) {
        });
 }

-function getByIdentifierPaged(identifier, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
+function getByAppIdPaged(page, perPage, appId, callback) {
    assert(typeof page === 'number' && page > 0);
    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, (page-1)*perPage, perPage ], function (error, results) {
+    // box versions (0.93.x and below) used to use appbackup_ prefix
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND id LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, '%app%\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) {
            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));

            results.forEach(function (result) { postProcess(result); });
@@ -98,11 +106,8 @@ function get(id, callback) {
 function add(id, data, callback) {
    assert(data && typeof data === 'object');
    assert.strictEqual(typeof id, 'string');
-    assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number');
-    assert.strictEqual(typeof data.packageVersion, 'string');
-    assert.strictEqual(typeof data.type, 'string');
-    assert.strictEqual(typeof data.identifier, 'string');
-    assert.strictEqual(typeof data.state, 'string');
+    assert.strictEqual(typeof data.version, 'string');
+    assert(data.type === exports.BACKUP_TYPE_APP || data.type === exports.BACKUP_TYPE_BOX);
    assert(util.isArray(data.dependsOn));
    assert.strictEqual(typeof data.manifest, 'object');
    assert.strictEqual(typeof data.format, 'string');
@@ -111,8 +116,8 @@ function add(id, data, callback) {
    var creationTime = data.creationTime || new Date(); // allow tests to set the time
    var manifestJson = JSON.stringify(data.manifest);

-    database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ],
+    database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+        [ id, data.version, data.type, creationTime, exports.BACKUP_STATE_NORMAL, data.dependsOn.join(','), manifestJson, data.format ],
        function (error) {
            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
@@ -50,7 +50,6 @@ BoxError.FS_ERROR = 'FileSystem Error';
 BoxError.INACTIVE = 'Inactive';
 BoxError.INTERNAL_ERROR = 'Internal Error';
 BoxError.INVALID_CREDENTIALS = 'Invalid Credentials';
-BoxError.IPTABLES_ERROR = 'IPTables Error';
 BoxError.LICENSE_ERROR = 'License Error';
 BoxError.LOGROTATE_ERROR = 'Logrotate Error';
 BoxError.MAIL_ERROR = 'Mail Error';
@@ -93,7 +92,6 @@ BoxError.toHttpError = function (error) {
    case BoxError.MAIL_ERROR:
    case BoxError.DOCKER_ERROR:
    case BoxError.ADDONS_ERROR:
-    case BoxError.IPTABLES_ERROR:
        return new HttpError(424, error);
    case BoxError.DATABASE_ERROR:
    case BoxError.INTERNAL_ERROR:
@@ -1,18 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    renderFooter
-};
-
-const assert = require('assert'),
-    constants = require('./constants.js');
-
-function renderFooter(footer) {
-    assert.strictEqual(typeof footer, 'string');
-
-    const year = new Date().getFullYear();
-
-    return footer.replace(/%YEAR%/g, year)
-        .replace(/%VERSION%/g, constants.VERSION);
-}
-
@@ -139,7 +139,7 @@ Acme2.prototype.updateContact = function (registrationUri, callback) {
    const that = this;
    this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
        if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 200, got %s %s', result.statusCode, result.text)));

        debug(`updateContact: contact of user updated to ${that.email}`);

@@ -160,7 +160,7 @@ Acme2.prototype.registerUser = function (callback) {
    this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
        if (error) return callback(error);
        // 200 if already exists. 201 for new accounts
-        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register new account. Expecting 200 or 201, got %s %s', result.statusCode, result.text)));

        debug(`registerUser: user registered keyid: ${result.headers.location}`);

@@ -185,8 +185,8 @@ Acme2.prototype.newOrder = function (domain, callback) {

    this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
        if (error) return callback(error);
-        if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+        if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending signed request: ${result.body.detail}`));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));

        debug('newOrder: created order %s %j', domain, result.body);

@@ -259,7 +259,7 @@ Acme2.prototype.notifyChallengeReady = function (challenge, callback) {

    this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
        if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 200, got %s %s', result.statusCode, result.text)));

        callback();
    });
@@ -313,7 +313,7 @@ Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, cal
    this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
        if (error) return callback(error);
        // 429 means we reached the cert limit for this domain
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 200, got %s %s', result.statusCode, result.text)));

        return callback(null);
    });
@@ -332,7 +332,7 @@ Acme2.prototype.createKeyAndCsr = function (hostname, callback) {
        // in some old releases, csr file was corrupt. so always regenerate it
        debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
    } else {
-        var key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
+        var key = safe.child_process.execSync('openssl genrsa 4096');
        if (!key) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error));

@@ -362,7 +362,7 @@ Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
        that.postAsGet(certUrl, function (error, result) {
            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
-            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
+            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));

            const fullChainPem = result.body; // buffer

@@ -586,34 +586,34 @@ Acme2.prototype.getDirectory = function (callback) {
    });
 };

-Acme2.prototype.getCertificate = function (vhost, domain, callback) {
-    assert.strictEqual(typeof vhost, 'string');
+Acme2.prototype.getCertificate = function (hostname, domain, callback) {
+    assert.strictEqual(typeof hostname, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof callback, 'function');

-    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
+    debug(`getCertificate: start acme flow for ${hostname} from ${this.caDirectory}`);

-    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
-        vhost = domains.makeWildcard(vhost);
-        debug(`getCertificate: will get wildcard cert for ${vhost}`);
+    if (hostname !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
+        hostname = domains.makeWildcard(hostname);
+        debug(`getCertificate: will get wildcard cert for ${hostname}`);
    }

    const that = this;
    this.getDirectory(function (error) {
        if (error) return callback(error);

-        that.acmeFlow(vhost, domain, function (error) {
+        that.acmeFlow(hostname, domain, function (error) {
            if (error) return callback(error);

            var outdir = paths.APP_CERTS_DIR;
-            const certName = vhost.replace('*.', '_.');
+            const certName = hostname.replace('*.', '_.');
            callback(null, path.join(outdir, `${certName}.cert`), path.join(outdir, `${certName}.key`));
        });
    });
 };

-function getCertificate(vhost, domain, options, callback) {
-    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
+function getCertificate(hostname, domain, options, callback) {
+    assert.strictEqual(typeof hostname, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');
@@ -623,6 +623,6 @@ function getCertificate(vhost, domain, options, callback) {
        debug(`getCertificate: attempt ${attempt++}`);

        let acme = new Acme2(options || { });
-        acme.getCertificate(vhost, domain, retryCallback);
+        acme.getCertificate(hostname, domain, retryCallback);
    }, callback);
 }
@@ -0,0 +1,22 @@
+'use strict';
+
+exports = module.exports = {
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'caas'
+};
+
+var assert = require('assert'),
+    debug = require('debug')('box:cert/caas.js');
+
+function getCertificate(hostname, domain, options, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getCertificate: using fallback certificate', hostname);
+
+    return callback(null, '', '');
+}
@@ -0,0 +1,22 @@
+'use strict';
+
+exports = module.exports = {
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'fallback'
+};
+
+var assert = require('assert'),
+    debug = require('debug')('box:cert/fallback.js');
+
+function getCertificate(hostname, domain, options, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getCertificate: using fallback certificate', hostname);
+
+    return callback(null, '', '');
+}
@@ -0,0 +1,24 @@
+'use strict';
+
+// -------------------------------------------
+//  This file just describes the interface
+//
+//  New backends can start from here
+// -------------------------------------------
+
+exports = module.exports = {
+    getCertificate: getCertificate
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js');
+
+function getCertificate(hostname, domain, options, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    return callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'getCertificate is not implemented'));
+}
+
@@ -1,34 +1,33 @@
 'use strict';

 exports = module.exports = {
-    initialize,
-    uninitialize,
-    getConfig,
-    getLogs,
+    initialize: initialize,
+    uninitialize: uninitialize,
+    getConfig: getConfig,
+    getLogs: getLogs,

-    reboot,
-    isRebootRequired,
+    reboot: reboot,
+    isRebootRequired: isRebootRequired,

-    onActivated,
+    onActivated: onActivated,

-    setupDnsAndCert,
+    prepareDashboardDomain: prepareDashboardDomain,
+    setDashboardDomain: setDashboardDomain,
+    setDashboardAndMailDomain: setDashboardAndMailDomain,
+    renewCerts: renewCerts,

-    prepareDashboardDomain,
-    setDashboardDomain,
-    updateDashboardDomain,
-    renewCerts,
+    setupDashboard: setupDashboard,

-    runSystemChecks
+    runSystemChecks: runSystemChecks,
 };

-const apps = require('./apps.js'),
+var apps = require('./apps.js'),
    appstore = require('./appstore.js'),
    assert = require('assert'),
    async = require('async'),
    auditSource = require('./auditsource.js'),
    backups = require('./backups.js'),
    BoxError = require('./boxerror.js'),
-    branding = require('./branding.js'),
    constants = require('./constants.js'),
    cron = require('./cron.js'),
    debug = require('debug')('box:cloudron'),
@@ -42,12 +41,10 @@ const apps = require('./apps.js'),
    platform = require('./platform.js'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
-    services = require('./services.js'),
    settings = require('./settings.js'),
    shell = require('./shell.js'),
    spawn = require('child_process').spawn,
    split = require('split'),
-    sysinfo = require('./sysinfo.js'),
    tasks = require('./tasks.js'),
    users = require('./users.js');

@@ -68,7 +65,7 @@ function uninitialize(callback) {

    async.series([
        cron.stopJobs,
-        platform.stopAllTasks
+        platform.stop
    ], callback);
 }

@@ -80,16 +77,7 @@ function onActivated(callback) {
    // 2. the restore code path can run without sudo (since mail/ is non-root)
    async.series([
        platform.start,
-        cron.startJobs,
-        function checkBackupConfiguration(done) {
-            backups.checkConfiguration(function (error, message) {
-                if (error) return done(error);
-                notifications.alert(notifications.ALERT_BACKUP_CONFIG, 'Backup configuration is unsafe', message, done);
-            });
-        },
-        // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
-        // the UI some time to query the dashboard domain in the restore code path
-        (done) => setTimeout(() => reverseProxy.writeDefaultConfig({ activated :true }, done), 30000)
+        cron.startJobs
    ], callback);
 }

@@ -114,49 +102,24 @@ function notifyUpdate(callback) {

 // each of these tasks can fail. we will add some routes to fix/re-run them
 function runStartupTasks() {
-    const tasks = [
-        // stop all the systemd tasks
-        platform.stopAllTasks,
+    // configure nginx to be reachable by IP
+    reverseProxy.writeDefaultConfig(NOOP_CALLBACK);

-        // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
-        function (callback) {
-            settings.getBackupConfig(function (error, backupConfig) {
-                if (error) return callback(error);
+    // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return console.error('Failed to read backup config.', error);
+        backups.configureCollectd(backupConfig, NOOP_CALLBACK);
+    });

-                backups.configureCollectd(backupConfig, callback);
-            });
-        },
+    // always generate webadmin config since we have no versioning mechanism for the ejs
+    if (settings.adminDomain()) reverseProxy.writeAdminConfig(settings.adminDomain(), NOOP_CALLBACK);

-        // always generate webadmin config since we have no versioning mechanism for the ejs
-        function (callback) {
-            if (!settings.adminDomain()) return callback();
+    // check activation state and start the platform
+    users.isActivated(function (error, activated) {
+        if (error) return debug(error);
+        if (!activated) return debug('initialize: not activated yet'); // not activated

-            reverseProxy.writeDashboardConfig(settings.adminDomain(), callback);
-        },
-
-        // check activation state and start the platform
-        function (callback) {
-            users.isActivated(function (error, activated) {
-                if (error) return callback(error);
-
-                // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
-                // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
-                // we remove the config as a simple security measure to not expose IP <-> domain
-                if (!activated) {
-                    debug('runStartupTasks: not activated. generating IP based redirection config');
-                    return reverseProxy.writeDefaultConfig({ activated: false }, callback);
-                }
-
-                onActivated(callback);
-            });
-        }
-    ];
-
-    // we used to run tasks in parallel but simultaneous nginx reloads was causing issues
-    async.series(async.reflectAll(tasks), function (error, results) {
-        results.forEach((result, idx) => {
-            if (result.error) debug(`Startup task at index ${idx} failed: ${result.error.message}`);
-        });
+        onActivated(NOOP_CALLBACK);
    });
 }

@@ -175,18 +138,17 @@ function getConfig(callback) {
            mailFqdn: settings.mailFqdn(),
            version: constants.VERSION,
            isDemo: settings.isDemo(),
+            provider: settings.provider(),
            cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
-            footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
-            features: appstore.getFeatures(),
-            profileLocked: allSettings[settings.DIRECTORY_CONFIG_KEY].lockUserProfiles,
-            mandatory2FA: allSettings[settings.DIRECTORY_CONFIG_KEY].mandatory2FA
+            footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
+            features: appstore.getFeatures()
        });
    });
 }

 function reboot(callback) {
    notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '', function (error) {
-        if (error) debug('reboot: failed to clear reboot notification.', error);
+        if (error) console.error('Failed to clear reboot notification.', error);

        shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
    });
@@ -204,11 +166,24 @@ function runSystemChecks(callback) {
    assert.strictEqual(typeof callback, 'function');

    async.parallel([
+        checkBackupConfiguration,
        checkMailStatus,
        checkRebootRequired
    ], callback);
 }

+function checkBackupConfiguration(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('checking backup configuration');
+
+    backups.checkConfiguration(function (error, message) {
+        if (error) return callback(error);
+
+        notifications.alert(notifications.ALERT_BACKUP_CONFIG, 'Backup configuration is unsafe', message, callback);
+    });
+}
+
 function checkMailStatus(callback) {
    assert.strictEqual(typeof callback, 'function');

@@ -299,7 +274,7 @@ function prepareDashboardDomain(domain, auditSource, callback) {
            const conflict = result.filter(app => app.fqdn === fqdn);
            if (conflict.length) return callback(new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app'));

-            tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.ADMIN_LOCATION, domain, auditSource ], function (error, taskId) {
+            tasks.add(tasks.TASK_PREPARE_DASHBOARD_DOMAIN, [ domain, auditSource ], function (error, taskId) {
                if (error) return callback(error);

                tasks.startTask(taskId, {}, NOOP_CALLBACK);
@@ -321,12 +296,12 @@ function setDashboardDomain(domain, auditSource, callback) {
    domains.get(domain, function (error, domainObject) {
        if (error) return callback(error);

-        reverseProxy.writeDashboardConfig(domain, function (error) {
+        reverseProxy.writeAdminConfig(domain, function (error) {
            if (error) return callback(error);

            const fqdn = domains.fqdn(constants.ADMIN_LOCATION, domainObject);

-            settings.setAdminLocation(domain, fqdn, function (error) {
+            settings.setAdmin(domain, fqdn, function (error) {
                if (error) return callback(error);

                eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain: domain, fqdn: fqdn });
@@ -338,24 +313,35 @@ function setDashboardDomain(domain, auditSource, callback) {
 }

 // call this only post activation because it will restart mail server
-function updateDashboardDomain(domain, auditSource, callback) {
+function setDashboardAndMailDomain(domain, auditSource, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

-    debug(`updateDashboardDomain: ${domain}`);
+    debug(`setDashboardAndMailDomain: ${domain}`);

    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));

    setDashboardDomain(domain, auditSource, function (error) {
        if (error) return callback(error);

-        services.rebuildService('turn', NOOP_CALLBACK); // to update the realm variable
+        mail.onMailFqdnChanged(NOOP_CALLBACK); // this will update dns and re-configure mail server

        callback(null);
    });
 }

+function setupDashboard(auditSource, progressCallback, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        domains.prepareDashboardDomain.bind(null, settings.adminDomain(), auditSource, progressCallback),
+        setDashboardDomain.bind(null, settings.adminDomain(), auditSource)
+    ], callback);
+}
+
 function renewCerts(options, auditSource, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');
@@ -369,34 +355,3 @@ function renewCerts(options, auditSource, callback) {
        callback(null, taskId);
    });
 }
-
-function setupDnsAndCert(subdomain, domain, auditSource, progressCallback, callback) {
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
-
-        const adminFqdn = domains.fqdn(subdomain, domainObject);
-
-        sysinfo.getServerIp(function (error, ip) {
-            if (error) return callback(error);
-
-            async.series([
-                (done) => { progressCallback({ message: `Updating DNS of ${adminFqdn}` }); done(); },
-                domains.upsertDnsRecords.bind(null, subdomain, domain, 'A', [ ip ]),
-                (done) => { progressCallback({ message: `Waiting for DNS of ${adminFqdn}` }); done(); },
-                domains.waitForDnsRecord.bind(null, subdomain, domain, 'A', ip, { interval: 30000, times: 50000 }),
-                (done) => { progressCallback({ message: `Getting certificate of ${adminFqdn}` }); done(); },
-                reverseProxy.ensureCertificate.bind(null, domains.fqdn(subdomain, domainObject), domain, auditSource)
-            ], function (error) {
-                if (error) return callback(error);
-
-                callback(null);
-            });
-        });
-    });
-}
@@ -1,9 +0,0 @@
-<Plugin python>
-  <Module du>
-    <Path>
-        Instance "<%= volumeId %>"
-        Dir "<%= hostPath %>"
-    </Path>
-  </Module>
-</Plugin>
-
@@ -26,7 +26,7 @@ exports = module.exports = {

    PORT: CLOUDRON ? 3000 : 5454,
    INTERNAL_SMTP_PORT: 2525, // this value comes from the mail container
-    AUTHWALL_PORT: 3001,
+    SYSADMIN_PORT: 3001, // unused
    LDAP_PORT: 3002,
    DOCKER_PROXY_PORT: 3003,

@@ -37,26 +37,18 @@ exports = module.exports = {
    DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024), // see also client.js

    DEMO_USERNAME: 'cloudron',
-    DEMO_BLACKLISTED_APPS: [
-        'com.github.cloudtorrent',
-        'net.alltubedownload.cloudronapp',
-        'com.adguard.home.cloudronapp',
-        'com.transmissionbt.cloudronapp',
-        'io.github.sickchill.cloudronapp',
-        'to.couchpota.cloudronapp'
-    ],

    AUTOUPDATE_PATTERN_NEVER: 'never',

-    SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8), // also used in dashboard client.js
+    SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8),

    CLOUDRON: CLOUDRON,
    TEST: TEST,

    SUPPORT_EMAIL: 'support@cloudron.io',

-    FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
+    FOOTER: '&copy; 2020  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',

-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '6.0.1-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '4.2.0-test'
 };

@@ -1,23 +1,16 @@
 'use strict';

-// IMPORTANT: These patterns are together because they spin tasks which acquire a lock
-// If the patterns overlap all the time, then the task may not ever get a chance to run!
-// If you change this change dashboard patterns in settings.html
-const DEFAULT_CLEANUP_BACKUPS_PATTERN = '00 30 1,3,5,23 * * *',
-    DEFAULT_AUTOUPDATE_PATTERN = '00 00 1,3,5,23 * * *';
-
 exports = module.exports = {
-    startJobs,
+    startJobs: startJobs,

-    stopJobs,
+    stopJobs: stopJobs,

-    handleSettingsChanged,
-
-    DEFAULT_AUTOUPDATE_PATTERN,
+    handleSettingsChanged: handleSettingsChanged
 };

 var appHealthMonitor = require('./apphealthmonitor.js'),
    apps = require('./apps.js'),
+    appstore = require('./appstore.js'),
    assert = require('assert'),
    async = require('async'),
    auditSource = require('./auditsource.js'),
@@ -33,13 +26,15 @@ var appHealthMonitor = require('./apphealthmonitor.js'),
    settings = require('./settings.js'),
    system = require('./system.js'),
    updater = require('./updater.js'),
-    updateChecker = require('./updatechecker.js'),
-    _ = require('underscore');
+    updateChecker = require('./updatechecker.js');

-const gJobs = {
-    autoUpdater: null,
+var gJobs = {
+    alive: null, // send periodic stats
+    appAutoUpdater: null,
+    boxAutoUpdater: null,
+    appUpdateChecker: null,
    backup: null,
-    updateChecker: null,
+    boxUpdateChecker: null,
    systemChecks: null,
    diskSpaceChecker: null,
    certificateRenew: null,
@@ -52,7 +47,7 @@ const gJobs = {
    appHealthMonitor: null
 };

-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };

 // cron format
 // Seconds: 0-59
@@ -65,11 +60,15 @@ const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 function startJobs(callback) {
    assert.strictEqual(typeof callback, 'function');

-    debug('startJobs: starting cron jobs');
+    const randomMinute = Math.floor(60*Math.random());
+    gJobs.alive = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // every hour on a random minute
+        onTick: appstore.sendAliveStatus,
+        start: true
+    });

-    const randomTick = Math.floor(60*Math.random());
    gJobs.systemChecks = new CronJob({
-        cronTime: '00 30 2 * * *', // once a day. if you change this interval, change the notification messages with correct duration
+        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
        onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
        start: true
    });
@@ -80,10 +79,15 @@ function startJobs(callback) {
        start: true
    });

-    // this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
-    gJobs.updateCheckerJob = new CronJob({
-        cronTime: `${randomTick} ${randomTick} 1,5,9,13,17,21,23 * * *`,
-        onTick: () => updateChecker.checkForUpdates({ automatic: true }, NOOP_CALLBACK),
+    gJobs.boxUpdateCheckerJob = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkBoxUpdates(NOOP_CALLBACK),
+        start: true
+    });
+
+    gJobs.appUpdateChecker = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkAppUpdates(NOOP_CALLBACK),
        start: true
    });

@@ -94,7 +98,7 @@ function startJobs(callback) {
    });

    gJobs.cleanupBackups = new CronJob({
-        cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN,
+        cronTime: '00 45 1,3,5,23 * * *', // every 6 hours. try not to overlap with ensureBackup job
        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
        start: true
    });
@@ -134,7 +138,8 @@ function startJobs(callback) {

        const tz = allSettings[settings.TIME_ZONE_KEY];
        backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
-        autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz);
+        appAutoupdatePatternChanged(allSettings[settings.APP_AUTOUPDATE_PATTERN_KEY], tz);
+        boxAutoupdatePatternChanged(allSettings[settings.BOX_AUTOUPDATE_PATTERN_KEY], tz);
        dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);

        callback();
@@ -149,7 +154,8 @@ function handleSettingsChanged(key, value) {
    switch (key) {
    case settings.TIME_ZONE_KEY:
    case settings.BACKUP_CONFIG_KEY:
-    case settings.AUTOUPDATE_PATTERN_KEY:
+    case settings.APP_AUTOUPDATE_PATTERN_KEY:
+    case settings.BOX_AUTOUPDATE_PATTERN_KEY:
    case settings.DYNAMIC_DNS_KEY:
        debug('handleSettingsChanged: recreating all jobs');
        async.series([
@@ -166,48 +172,71 @@ function backupConfigChanged(value, tz) {
    assert.strictEqual(typeof value, 'object');
    assert.strictEqual(typeof tz, 'string');

-    debug(`backupConfigChanged: schedule ${value.schedulePattern} (${tz})`);
+    debug(`backupConfigChanged: interval ${value.intervalSecs} (${tz})`);

    if (gJobs.backup) gJobs.backup.stop();
+    let pattern;
+    if (value.intervalSecs <= 6 * 60 * 60) {
+        pattern = '00 00 1,7,13,19 * * *'; // no option but to backup in the middle of the day
+    } else {
+        pattern = '00 00 1,3,5,23 * * *'; // avoid middle of the day backups
+    }

    gJobs.backup = new CronJob({
-        cronTime: value.schedulePattern,
-        onTick: backups.startBackupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        cronTime: pattern,
+        onTick: backups.ensureBackup.bind(null, auditSource.CRON, NOOP_CALLBACK),
        start: true,
        timeZone: tz
    });
 }

-function autoupdatePatternChanged(pattern, tz) {
+function boxAutoupdatePatternChanged(pattern, tz) {
    assert.strictEqual(typeof pattern, 'string');
    assert.strictEqual(typeof tz, 'string');

-    debug(`autoupdatePatternChanged: pattern - ${pattern} (${tz})`);
+    debug(`boxAutoupdatePatternChanged: pattern - ${pattern} (${tz})`);

-    if (gJobs.autoUpdater) gJobs.autoUpdater.stop();
+    if (gJobs.boxAutoUpdater) gJobs.boxAutoUpdater.stop();

    if (pattern === constants.AUTOUPDATE_PATTERN_NEVER) return;

-    gJobs.autoUpdater = new CronJob({
+    gJobs.boxAutoUpdater = new CronJob({
        cronTime: pattern,
        onTick: function() {
-            const updateInfo = updateChecker.getUpdateInfo();
-            // do box before app updates. for the off chance that the box logic fixes some app update logic issue
-            if (updateInfo.box && !updateInfo.box.unstable) {
-                debug('Starting box autoupdate to %j', updateInfo.box);
+            var updateInfo = updateChecker.getUpdateInfo();
+            if (updateInfo.box) {
+                debug('Starting autoupdate to %j', updateInfo.box);
                updater.updateToLatest({ skipBackup: false }, auditSource.CRON, NOOP_CALLBACK);
-                return;
+            } else {
+                debug('No box auto updates available');
            }
+        },
+        start: true,
+        timeZone: tz
+    });
+}

-            const appUpdateInfo = _.omit(updateInfo, 'box');
-            if (Object.keys(appUpdateInfo).length > 0) {
-                debug('Starting app update to %j', appUpdateInfo);
-                apps.autoupdateApps(appUpdateInfo, auditSource.CRON, NOOP_CALLBACK);
+function appAutoupdatePatternChanged(pattern, tz) {
+    assert.strictEqual(typeof pattern, 'string');
+    assert.strictEqual(typeof tz, 'string');
+
+    debug(`appAutoupdatePatternChanged: pattern ${pattern} (${tz})`);
+
+    if (gJobs.appAutoUpdater) gJobs.appAutoUpdater.stop();
+
+    if (pattern === constants.AUTOUPDATE_PATTERN_NEVER) return;
+
+    gJobs.appAutoUpdater = new CronJob({
+        cronTime: pattern,
+        onTick: function() {
+            var updateInfo = updateChecker.getUpdateInfo();
+            if (updateInfo.apps) {
+                debug('Starting app update to %j', updateInfo.apps);
+                apps.autoupdateApps(updateInfo.apps, auditSource.CRON, NOOP_CALLBACK);
            } else {
                debug('No app auto updates available');
            }
        },
-
        start: true,
        timeZone: tz
    });
@@ -17,12 +17,12 @@ var assert = require('assert'),
    BoxError = require('./boxerror.js'),
    child_process = require('child_process'),
    constants = require('./constants.js'),
-    debug = require('debug')('box:database'),
    mysql = require('mysql'),
    once = require('once'),
    util = require('util');

-var gConnectionPool = null;
+var gConnectionPool = null,
+    gDefaultConnection = null;

 const gDatabase = {
    hostname: '127.0.0.1',
@@ -42,37 +42,59 @@ function initialize(callback) {
        gDatabase.hostname = require('child_process').execSync('docker inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" mysql-server').toString().trim();
    }

-    // https://github.com/mysqljs/mysql#pool-options
    gConnectionPool  = mysql.createPool({
-        connectionLimit: 5,
+        connectionLimit: 5, // this has to be > 1 since we store one connection as 'default'. the rest for transactions
        host: gDatabase.hostname,
        user: gDatabase.username,
        password: gDatabase.password,
        port: gDatabase.port,
        database: gDatabase.name,
        multipleStatements: false,
-        waitForConnections: true, // getConnection() will wait until a connection is avaiable
        ssl: false,
        timezone: 'Z' // mysql follows the SYSTEM timezone. on Cloudron, this is UTC
    });

    gConnectionPool.on('connection', function (connection) {
-        // connection objects are re-used. so we have to attach to the event here (once) to prevent crash
-        // note the pool also has an 'acquire' event but that is called whenever we do a getConnection()
-        connection.on('error', (error) => debug(`Connection ${connection.threadId} error: ${error.message} ${error.code}`));
-
        connection.query('USE ' + gDatabase.name);
        connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
    });

-    callback(null);
+    reconnect(callback);
 }

 function uninitialize(callback) {
-    if (!gConnectionPool) return callback(null);
+    if (gConnectionPool) {
+        gConnectionPool.end(callback);
+        gConnectionPool = null;
+    } else {
+        callback(null);
+    }
+}

-    gConnectionPool.end(callback);
-    gConnectionPool = null;
+function reconnect(callback) {
+    callback = callback ? once(callback) : function () {};
+
+    gConnectionPool.getConnection(function (error, connection) {
+        if (error) {
+            console.error('Unable to reestablish connection to database. Try again in a bit.', error.message);
+            return setTimeout(reconnect.bind(null, callback), 1000);
+        }
+
+        connection.on('error', function (error) {
+            // by design, we catch all normal errors by providing callbacks.
+            // this function should be invoked only when we have no callbacks pending and we have a fatal error
+            assert(error.fatal, 'Non-fatal error on connection object');
+
+            console.error('Unhandled mysql connection error.', error);
+
+            // This is most likely an issue an can cause double callbacks from reconnect()
+            setTimeout(reconnect.bind(null, callback), 1000);
+        });
+
+        gDefaultConnection = connection;
+
+        callback(null);
+    });
 }

 function clear(callback) {
@@ -85,43 +107,80 @@ function clear(callback) {
    child_process.exec(cmd, callback);
 }

-function query() {
-    const args = Array.prototype.slice.call(arguments);
-    const callback = args[args.length - 1];
+function beginTransaction(callback) {
    assert.strictEqual(typeof callback, 'function');

-    if (constants.TEST && !gConnectionPool) return callback(new BoxError(BoxError.DATABASE_ERROR, 'database.js not initialized'));
+    if (gConnectionPool === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No database connection pool.'));

-    gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
+    gConnectionPool.getConnection(function (error, connection) {
+        if (error) {
+            console.error('Unable to get connection to database. Try again in a bit.', error.message);
+            return setTimeout(beginTransaction.bind(null, callback), 1000);
+        }
+
+        connection.beginTransaction(function (error) {
+            if (error) return callback(error);
+
+            return callback(null, connection);
+        });
+    });
+}
+
+function rollback(connection, callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    connection.rollback(function (error) {
+        if (error) console.error(error); // can this happen?
+
+        connection.release();
+        callback(null);
+    });
+}
+
+// FIXME: if commit fails, is it supposed to return an error ?
+function commit(connection, callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    connection.commit(function (error) {
+        if (error) return rollback(connection, callback);
+
+        connection.release();
+        return callback(null);
+    });
+}
+
+function query() {
+    var args = Array.prototype.slice.call(arguments);
+    var callback = args[args.length - 1];
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gDefaultConnection === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No connection to database'));
+
+    args[args.length -1 ] = function (error, result) {
+        if (error && error.fatal) {
+            gDefaultConnection = null;
+            setTimeout(reconnect, 1000);
+        }
+
+        callback(error, result);
+    };
+
+    gDefaultConnection.query.apply(gDefaultConnection, args);
 }

 function transaction(queries, callback) {
    assert(util.isArray(queries));
    assert.strictEqual(typeof callback, 'function');

-    callback = once(callback);
-
-    gConnectionPool.getConnection(function (error, connection) {
+    beginTransaction(function (error, conn) {
        if (error) return callback(error);

-        const releaseConnection = (error) => { connection.release(); callback(error); };
+        async.mapSeries(queries, function iterator(query, done) {
+            conn.query(query.query, query.args, done);
+        }, function seriesDone(error, results) {
+            if (error) return rollback(conn, callback.bind(null, error));

-        connection.beginTransaction(function (error) {
-            if (error) return releaseConnection(error);
-
-            async.mapSeries(queries, function iterator(query, done) {
-                connection.query(query.query, query.args, done);
-            }, function seriesDone(error, results) {
-                if (error) return connection.rollback(() => releaseConnection(error));
-
-                connection.commit(function (error) {
-                    if (error) return connection.rollback(() => releaseConnection(error));
-
-                    connection.release();
-
-                    callback(null, results);
-                });
-            });
+            commit(conn, callback.bind(null, null, results));
        });
    });
 }
@@ -144,7 +203,7 @@ function exportToFile(file, callback) {

    // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
    // this option must not be set in production cloudrons which still use the old mysqldump
-    const disableColStats = (constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
+    const disableColStats = (constants.TEST && process.env.DESKTOP_SESSION !== 'ubuntu') ? '--column-statistics=0' : '';

    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${disableColStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;

@@ -0,0 +1,176 @@
+'use strict';
+
+exports = module.exports = {
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    debug = require('debug')('box:dns/caas'),
+    domains = require('../domains.js'),
+    settings = require('../settings.js'),
+    superagent = require('superagent'),
+    util = require('util'),
+    waitForDns = require('./waitfordns.js');
+
+function formatError(response) {
+    return util.format('Caas DNS error [%s] %j', response.statusCode, response.body);
+}
+
+function getFqdn(location, domain) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+
+    return (location === '') ? domain : location + '-' + domain;
+}
+
+function removePrivateFields(domainObject) {
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
+
+    // do not return the 'key'. in caas, this is private
+    delete domainObject.fallbackCertificate.key;
+
+    return domainObject;
+}
+
+function injectPrivateFields(newConfig, currentConfig) {
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+}
+
+function upsert(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config;
+
+    let fqdn = location !== '' && type === 'TXT' ? location + '.' + domainObject.domain : getFqdn(location, domainObject.domain);
+
+    debug('add: %s for zone %s of type %s with values %j', location, domainObject.domain, type, values);
+
+    var data = {
+        type: type,
+        values: values
+    };
+
+    superagent
+        .post(settings.apiServerOrigin() + '/api/v1/caas/domains/' + fqdn)
+        .query({ token: dnsConfig.token })
+        .send(data)
+        .timeout(30 * 1000)
+        .end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, result.body.message));
+            if (result.statusCode === 420) return callback(new BoxError(BoxError.BUSY));
+            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+            return callback(null);
+        });
+}
+
+function get(domainObject, location, type, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config;
+    const fqdn = location !== '' && type === 'TXT' ? location + '.' + domainObject.domain : getFqdn(location, domainObject.domain);
+
+    debug('get: zoneName: %s subdomain: %s type: %s fqdn: %s', domainObject.domain, location, type, fqdn);
+
+    superagent
+        .get(settings.apiServerOrigin() + '/api/v1/caas/domains/' + fqdn)
+        .query({ token: dnsConfig.token, type: type })
+        .timeout(30 * 1000)
+        .end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+            return callback(null, result.body.values);
+        });
+}
+
+function del(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config;
+    debug('del: %s for zone %s of type %s with values %j', location, domainObject.domain, type, values);
+
+    var data = {
+        type: type,
+        values: values
+    };
+
+    superagent
+        .del(settings.apiServerOrigin() + '/api/v1/caas/domains/' + getFqdn(location, domainObject.domain))
+        .query({ token: dnsConfig.token })
+        .send(data)
+        .timeout(30 * 1000)
+        .end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, result.body.message));
+            if (result.statusCode === 420) return callback(new BoxError(BoxError.BUSY));
+            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+            if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+            return callback(null);
+        });
+}
+
+function wait(domainObject, location, type, value, options, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+    assert.strictEqual(typeof callback, 'function');
+
+    const fqdn = domains.fqdn(location, domainObject);
+
+    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+}
+
+function verifyDnsConfig(domainObject, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config;
+
+    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+
+    const ip = '127.0.0.1';
+
+    var credentials = {
+        token: dnsConfig.token,
+        hyphenatedSubdomains: true  // this will ensure we always use them, regardless of passed-in configs
+    };
+
+    const location = 'cloudrontestdns';
+
+    upsert(domainObject, location, 'A', [ ip ], function (error) {
+        if (error) return callback(error);
+
+        debug('verifyDnsConfig: Test A record added');
+
+        del(domainObject, location, 'A', [ ip ], function (error) {
+            if (error) return callback(error);
+
+            debug('verifyDnsConfig: Test A record removed again');
+
+            callback(null, credentials);
+        });
+    });
+}
@@ -13,7 +13,6 @@ exports = module.exports = {
 var assert = require('assert'),
    async = require('async'),
    BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
    debug = require('debug')('box:dns/cloudflare'),
    dns = require('../native-dns.js'),
    domains = require('../domains.js'),
@@ -26,12 +25,12 @@ var assert = require('assert'),
 var CLOUDFLARE_ENDPOINT = 'https://api.cloudflare.com/client/v4';

 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
    return domainObject;
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }

 function translateRequestError(result, callback) {
@@ -40,14 +39,9 @@ function translateRequestError(result, callback) {

    if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, util.format('%s %j', result.statusCode, 'API does not exist')));
    if (result.statusCode === 422) return callback(new BoxError(BoxError.BAD_FIELD, result.body.message));
-    if (result.statusCode === 400 || result.statusCode === 401 || result.statusCode === 403) {
-        let message = 'Unknown error';
-        if (typeof result.body.error === 'string') {
-            message = `message: ${result.body.error} statusCode: ${result.statusCode}`;
-        } else if (Array.isArray(result.body.errors) && result.body.errors.length > 0) {
-            let error = result.body.errors[0];
-            message = `message: ${error.message} statusCode: ${result.statusCode} code:${error.code}`;
-        }
+    if ((result.statusCode === 400 || result.statusCode === 401 || result.statusCode === 403) && result.body.errors.length > 0) {
+        let error = result.body.errors[0];
+        let message = `message: ${error.message} statusCode: ${result.statusCode} code:${error.code}`;
        return callback(new BoxError(BoxError.ACCESS_DENIED, message));
    }

@@ -290,7 +284,7 @@ function verifyDnsConfig(domainObject, callback) {
    if (dnsConfig.tokenType !== 'GlobalApiKey' && dnsConfig.tokenType !== 'ApiToken')  return callback(new BoxError(BoxError.BAD_FIELD, 'tokenType is required', { field: 'tokenType' }));

    if (dnsConfig.tokenType === 'GlobalApiKey') {
-        if (typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+        if ('email' in dnsConfig && typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
    }

    const ip = '127.0.0.1';
@@ -13,7 +13,6 @@ exports = module.exports = {
 var assert = require('assert'),
    async = require('async'),
    BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
    debug = require('debug')('box:dns/digitalocean'),
    dns = require('../native-dns.js'),
    domains = require('../domains.js'),
@@ -29,12 +28,12 @@ function formatError(response) {
 }

 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
    return domainObject;
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }

 function getInternal(dnsConfig, zoneName, name, type, callback) {
@@ -12,7 +12,6 @@ exports = module.exports = {

 var assert = require('assert'),
    BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
    debug = require('debug')('box:dns/gandi'),
    dns = require('../native-dns.js'),
    domains = require('../domains.js'),
@@ -27,12 +26,12 @@ function formatError(response) {
 }

 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
    return domainObject;
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }

 function upsert(domainObject, location, type, values, callback) {
@@ -12,7 +12,6 @@ exports = module.exports = {

 var assert = require('assert'),
    BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
    debug = require('debug')('box:dns/gcdns'),
    dns = require('../native-dns.js'),
    domains = require('../domains.js'),
@@ -22,12 +21,12 @@ var assert = require('assert'),
    _ = require('underscore');

 function removePrivateFields(domainObject) {
-    domainObject.config.credentials.private_key = constants.SECRET_PLACEHOLDER;
+    domainObject.config.credentials.private_key = domains.SECRET_PLACEHOLDER;
    return domainObject;
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.credentials.private_key === constants.SECRET_PLACEHOLDER && currentConfig.credentials) newConfig.credentials.private_key = currentConfig.credentials.private_key;
+    if (newConfig.credentials.private_key === domains.SECRET_PLACEHOLDER && currentConfig.credentials) newConfig.credentials.private_key = currentConfig.credentials.private_key;
 }

 function getDnsCredentials(dnsConfig) {
@@ -12,7 +12,6 @@ exports = module.exports = {

 var assert = require('assert'),
    BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
    debug = require('debug')('box:dns/godaddy'),
    dns = require('../native-dns.js'),
    domains = require('../domains.js'),
@@ -33,12 +32,12 @@ function formatError(response) {
 }

 function removePrivateFields(domainObject) {
-    domainObject.config.apiSecret = constants.SECRET_PLACEHOLDER;
+    domainObject.config.apiSecret = domains.SECRET_PLACEHOLDER;
    return domainObject;
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.apiSecret === constants.SECRET_PLACEHOLDER) newConfig.apiSecret = currentConfig.apiSecret;
+    if (newConfig.apiSecret === domains.SECRET_PLACEHOLDER) newConfig.apiSecret = currentConfig.apiSecret;
 }

 function upsert(domainObject, location, type, values, callback) {
@@ -21,13 +21,13 @@ var assert = require('assert'),
    util = require('util');

 function removePrivateFields(domainObject) {
-    // in-place removal of tokens and api keys with constants.SECRET_PLACEHOLDER
+    // in-place removal of tokens and api keys with domains.SECRET_PLACEHOLDER
    return domainObject;
 }

 // eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
-    // in-place injection of tokens and api keys which came in with constants.SECRET_PLACEHOLDER
+    // in-place injection of tokens and api keys which came in with domains.SECRET_PLACEHOLDER
 }

 function upsert(domainObject, location, type, values, callback) {
@@ -12,7 +12,6 @@ exports = module.exports = {

 let async = require('async'),
    assert = require('assert'),
-    constants = require('../constants.js'),
    BoxError = require('../boxerror.js'),
    debug = require('debug')('box:dns/linode'),
    dns = require('../native-dns.js'),
@@ -28,12 +27,12 @@ function formatError(response) {
 }

 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
    return domainObject;
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }

 function getZoneId(dnsConfig, zoneName, callback) {
@@ -119,10 +118,9 @@ function get(domainObject, location, type, callback) {
        zoneName = domainObject.zoneName,
        name = domains.getName(domainObject, location, type) || '';

-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, { records }) {
        if (error) return callback(error);

-        const { records } = result;
        var tmp = records.map(function (record) { return record.target; });

        debug('get: %j', tmp);
@@ -144,10 +142,9 @@ function upsert(domainObject, location, type, values, callback) {

    debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);

-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, { zoneId, records }) {
        if (error) return callback(error);

-        const { zoneId, records } = result;
        let i = 0, recordIds = []; // used to track available records to update instead of create

        async.eachSeries(values, function (value, iteratorCallback) {
@@ -224,10 +221,9 @@ function del(domainObject, location, type, values, callback) {
        zoneName = domainObject.zoneName,
        name = domains.getName(domainObject, location, type) || '';

-    getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
+    getZoneRecords(dnsConfig, zoneName, name, type, function (error, { zoneId, records }) {
        if (error) return callback(error);

-        const { zoneId, records } = result;
        if (records.length === 0) return callback(null);

        var tmp = records.filter(function (record) { return values.some(function (value) { return value === record.target; }); });
@@ -290,7 +286,7 @@ function verifyDnsConfig(domainObject, callback) {
        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));

        if (nameservers.map(function (n) { return n.toLowerCase(); }).indexOf('ns1.linode.com') === -1) {
-            debug('verifyDnsConfig: %j does not contains linode NS', nameservers);
+            debug('verifyDnsConfig: %j does not contains DO NS', nameservers);
            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Linode', { field: 'nameservers' }));
        }

@@ -12,7 +12,6 @@ exports = module.exports = {

 var assert = require('assert'),
    BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
    debug = require('debug')('box:dns/namecheap'),
    dns = require('../native-dns.js'),
    domains = require('../domains.js'),
@@ -26,12 +25,12 @@ var assert = require('assert'),
 const ENDPOINT = 'https://api.namecheap.com/xml.response';

 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
    return domainObject;
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }

 function getQuery(dnsConfig, callback) {
@@ -78,11 +77,13 @@ function getInternal(dnsConfig, zoneName, subdomain, type, callback) {

                    return callback(new BoxError(BoxError.EXTERNAL_ERROR, errorMessage));
                }
-                const host = safe.query(tmp, 'CommandResponse[0].DomainDNSGetHostsResult[0].host');
-                if (!host) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response: ${JSON.stringify(tmp)}`));
-                if (!Array.isArray(host)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `host is not an array: ${JSON.stringify(tmp)}`));
+                if (!tmp.CommandResponse[0]) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
+                if (!tmp.CommandResponse[0].DomainDNSGetHostsResult[0]) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response'));
+
+                var hosts = result.ApiResponse.CommandResponse[0].DomainDNSGetHostsResult[0].host.map(function (h) {
+                    return h['$'];
+                });

-                const hosts = host.map(h => h['$']);
                callback(null, hosts);
            });
        });
@@ -12,7 +12,6 @@ exports = module.exports = {

 var assert = require('assert'),
    BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
    debug = require('debug')('box:dns/namecom'),
    dns = require('../native-dns.js'),
    domains = require('../domains.js'),
@@ -28,12 +27,12 @@ function formatError(response) {
 }

 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
    return domainObject;
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }

 function addRecord(dnsConfig, zoneName, name, type, values, callback) {
@@ -55,10 +54,6 @@ function addRecord(dnsConfig, zoneName, name, type, values, callback) {
    if (type === 'MX') {
        data.priority = parseInt(values[0].split(' ')[0], 10);
        data.answer = values[0].split(' ')[1];
-    } else if (type === 'TXT') {
-        // we have to strip the quoting for some odd reason for name.com! If you change that also change updateRecord
-        let tmp = values[0];
-        data.answer = tmp.indexOf('"') === 0 && tmp.lastIndexOf('"') === tmp.length-1 ? tmp.slice(1, tmp.length-1) : tmp;
    } else {
        data.answer = values[0];
    }
@@ -96,10 +91,6 @@ function updateRecord(dnsConfig, zoneName, recordId, name, type, values, callbac
    if (type === 'MX') {
        data.priority = parseInt(values[0].split(' ')[0], 10);
        data.answer = values[0].split(' ')[1];
-    } else if (type === 'TXT') {
-        // we have to strip the quoting for some odd reason for name.com! If you change that also change addRecord
-        let tmp = values[0];
-        data.answer = tmp.indexOf('"') === 0 && tmp.lastIndexOf('"') === tmp.length-1 ? tmp.slice(1, tmp.length-1) : tmp;
    } else {
        data.answer = values[0];
    }
@@ -1,303 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
-};
-
-var assert = require('assert'),
-    BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
-    debug = require('debug')('box:dns/netcup'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
-    superagent = require('superagent'),
-    util = require('util'),
-    waitForDns = require('./waitfordns.js');
-
-var API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON';
-
-function formatError(response) {
-    if (response.body) return util.format('Netcup DNS error [%s] %s', response.body.statuscode, response.body.longmessage);
-    else return util.format('Netcup DNS error [%s] %s', response.statusCode, response.text);
-}
-
-function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
-    return domainObject;
-}
-
-function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
-}
-
-// returns a api session id
-function login(dnsConfig, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const data = {
-        action: 'login',
-        param:{
-            apikey: dnsConfig.apiKey,
-            apipassword: dnsConfig.apiPassword,
-            customernumber: dnsConfig.customerNumber
-        }
-    };
-
-    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-        callback(null, result.body.responsedata.apisessionid);
-    });
-}
-
-function getAllRecords(dnsConfig, apiSessionId, zoneName, callback) {
-    assert.strictEqual(typeof dnsConfig, 'object');
-    assert.strictEqual(typeof apiSessionId, 'string');
-    assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`getAllRecords: getting dns records of ${zoneName}`);
-
-    const data = {
-        action: 'infoDnsRecords',
-        param:{
-            apikey: dnsConfig.apiKey,
-            apisessionid: apiSessionId,
-            customernumber: dnsConfig.customerNumber,
-            domainname: zoneName,
-        }
-    };
-
-    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-        debug('getAllRecords:', JSON.stringify(result.body.responsedata.dnsrecords || []));
-
-        callback(null, result.body.responsedata.dnsrecords || []);
-    });
-}
-
-function upsert(domainObject, location, type, values, callback) {
-    assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
-
-    const dnsConfig = domainObject.config,
-        zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
-
-    debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
-
-    login(dnsConfig, function (error, apiSessionId) {
-        if (error) return callback(error);
-
-        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
-            if (error) return callback(error);
-
-            let records = [];
-
-            values.forEach(function (value) {
-                // remove possible quotation
-                if (value.charAt(0) === '"') value = value.slice(1);
-                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
-
-                let priority = null;
-                if (type === 'MX') {
-                    priority = parseInt(value.split(' ')[0], 10);
-                    value = value.split(' ')[1];
-                }
-
-                let record = result.find(function (r) { return r.hostname === name && r.type === type; });
-                if (!record) record = { hostname: name, type: type, destination: value, deleterecord: false };
-                else record.destination = value;
-
-                if (priority !== null) record.priority = priority;
-
-                records.push(record);
-            });
-
-            const data = {
-                action: 'updateDnsRecords',
-                param:{
-                    apikey: dnsConfig.apiKey,
-                    apisessionid: apiSessionId,
-                    customernumber: dnsConfig.customerNumber,
-                    domainname: zoneName,
-                    dnsrecordset: {
-                        dnsrecords: records
-                    }
-                }
-            };
-
-            debug('upserting', JSON.stringify(data));
-
-            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                debug('upsert:', result.body);
-
-                callback(null);
-            });
-        });
-    });
-}
-
-function get(domainObject, location, type, callback) {
-    assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    const dnsConfig = domainObject.config,
-        zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
-
-    debug('get: %s for zone %s of type %s', name, zoneName, type);
-
-    login(dnsConfig, function (error, apiSessionId) {
-        if (error) return callback(error);
-
-        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
-            if (error) return callback(error);
-
-            // We only return the value string
-            callback(null, result.filter(function (r) { return r.hostname === name && r.type === type; }).map(function (r) { return r.destination; }));
-        });
-    });
-}
-
-function del(domainObject, location, type, values, callback) {
-    assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
-    assert.strictEqual(typeof callback, 'function');
-
-    const dnsConfig = domainObject.config,
-        zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
-
-    debug('del: %s for zone %s of type %s with values %j', name, zoneName, type, values);
-
-    login(dnsConfig, function (error, apiSessionId) {
-        if (error) return callback(error);
-
-        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
-            if (error) return callback(error);
-
-            let records = [];
-
-            values.forEach(function (value) {
-                // remove possible quotation
-                if (value.charAt(0) === '"') value = value.slice(1);
-                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
-
-                let record = result.find(function (r) { return r.hostname === name && r.type === type && r.destination === value; });
-                if (!record) return;
-
-                record.deleterecord = true;
-
-                records.push(record);
-            });
-
-            if (records.length === 0) return callback(null);
-
-            const data = {
-                action: 'updateDnsRecords',
-                param:{
-                    apikey: dnsConfig.apiKey,
-                    apisessionid: apiSessionId,
-                    customernumber: dnsConfig.customerNumber,
-                    domainname: zoneName,
-                    dnsrecordset: {
-                        dnsrecords: records
-                    }
-                }
-            };
-
-            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
-
-                debug('del:', result.body.responsedata);
-
-                callback(null);
-            });
-        });
-    });
-}
-
-function wait(domainObject, location, type, value, options, callback) {
-    assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof value, 'string');
-    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
-    assert.strictEqual(typeof callback, 'function');
-
-    const fqdn = domains.fqdn(location, domainObject);
-
-    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
-}
-
-function verifyDnsConfig(domainObject, callback) {
-    assert.strictEqual(typeof domainObject, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const dnsConfig = domainObject.config,
-        zoneName = domainObject.zoneName;
-
-    if (!dnsConfig.customerNumber || typeof dnsConfig.customerNumber !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'customerNumber must be a non-empty string', { field: 'customerNumber' }));
-    if (!dnsConfig.apiKey || typeof dnsConfig.apiKey !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string', { field: 'apiKey' }));
-    if (!dnsConfig.apiPassword || typeof dnsConfig.apiPassword !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiPassword must be a non-empty string', { field: 'apiPassword' }));
-
-    const ip = '127.0.0.1';
-
-    var credentials = {
-        customerNumber: dnsConfig.customerNumber,
-        apiKey: dnsConfig.apiKey,
-        apiPassword: dnsConfig.apiPassword,
-    };
-
-    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
-
-    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
-        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
-        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
-
-        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) {
-            debug('verifyDnsConfig: %j does not contains Netcup NS', nameservers);
-            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup', { field: 'nameservers' }));
-        }
-
-        const location = 'cloudrontestdns';
-
-        upsert(domainObject, location, 'A', [ ip ], function (error) {
-            if (error) return callback(error);
-
-            debug('verifyDnsConfig: Test A record added');
-
-            del(domainObject, location, 'A', [ ip ], function (error) {
-                if (error) return callback(error);
-
-                debug('verifyDnsConfig: Test A record removed again');
-
-                callback(null, credentials);
-            });
-        });
-    });
-}
@@ -13,7 +13,6 @@ exports = module.exports = {
 var assert = require('assert'),
    AWS = require('aws-sdk'),
    BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
    debug = require('debug')('box:dns/route53'),
    dns = require('../native-dns.js'),
    domains = require('../domains.js'),
@@ -22,12 +21,12 @@ var assert = require('assert'),
    _ = require('underscore');

 function removePrivateFields(domainObject) {
-    domainObject.config.secretAccessKey = constants.SECRET_PLACEHOLDER;
+    domainObject.config.secretAccessKey = domains.SECRET_PLACEHOLDER;
    return domainObject;
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.secretAccessKey === constants.SECRET_PLACEHOLDER) newConfig.secretAccessKey = currentConfig.secretAccessKey;
+    if (newConfig.secretAccessKey === domains.SECRET_PLACEHOLDER) newConfig.secretAccessKey = currentConfig.secretAccessKey;
 }

 function getDnsCredentials(dnsConfig) {
@@ -281,14 +280,13 @@ function verifyDnsConfig(domainObject, callback) {
            }

            const location = 'cloudrontestdns';
-            const newDomainObject = Object.assign({ }, domainObject, { config: credentials });

-            upsert(newDomainObject, location, 'A', [ ip ], function (error) {
+            upsert(domainObject, location, 'A', [ ip ], function (error) {
                if (error) return callback(error);

                debug('verifyDnsConfig: Test A record added');

-                del(newDomainObject, location, 'A', [ ip ], function (error) {
+                del(domainObject, location, 'A', [ ip ], function (error) {
                    if (error) return callback(error);

                    debug('verifyDnsConfig: Test A record removed again');
@@ -1,39 +1,40 @@
 'use strict';

 exports = module.exports = {
-    testRegistryConfig,
-    setRegistryConfig,
-    injectPrivateFields,
-    removePrivateFields,
+    testRegistryConfig: testRegistryConfig,
+    setRegistryConfig: setRegistryConfig,
+    injectPrivateFields: injectPrivateFields,
+    removePrivateFields: removePrivateFields,

-    ping,
+    SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8),

-    info,
-    downloadImage,
-    createContainer,
-    startContainer,
-    restartContainer,
-    stopContainer,
+    ping: ping,
+
+    info: info,
+    downloadImage: downloadImage,
+    createContainer: createContainer,
+    startContainer: startContainer,
+    restartContainer: restartContainer,
+    stopContainer: stopContainer,
    stopContainerByName: stopContainer,
-    stopContainers,
-    deleteContainer,
-    deleteImage,
-    deleteContainers,
-    createSubcontainer,
-    getContainerIdByIp,
-    inspect,
-    getContainerIp,
+    stopContainers: stopContainers,
+    deleteContainer: deleteContainer,
+    deleteContainerByName: deleteContainer,
+    deleteImage: deleteImage,
+    deleteContainers: deleteContainers,
+    createSubcontainer: createSubcontainer,
+    getContainerIdByIp: getContainerIdByIp,
+    inspect: inspect,
    inspectByName: inspect,
-    execContainer,
-    getEvents,
-    memoryUsage,
-    createVolume,
-    removeVolume,
-    clearVolume,
-    update
+    execContainer: execContainer,
+    getEvents: getEvents,
+    memoryUsage: memoryUsage,
+    createVolume: createVolume,
+    removeVolume: removeVolume,
+    clearVolume: clearVolume
 };

-const apps = require('./apps.js'),
+var addons = require('./addons.js'),
    async = require('async'),
    assert = require('assert'),
    BoxError = require('./boxerror.js'),
@@ -41,15 +42,11 @@ const apps = require('./apps.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:docker'),
    Docker = require('dockerode'),
-    os = require('os'),
    path = require('path'),
-    services = require('./services.js'),
    settings = require('./settings.js'),
    shell = require('./shell.js'),
    safe = require('safetydance'),
-    system = require('./system.js'),
    util = require('util'),
-    volumes = require('./volumes.js'),
    _ = require('underscore');

 const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
@@ -58,6 +55,12 @@ const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
 const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
 const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });

+function debugApp(app) {
+    assert(typeof app === 'object');
+
+    debug(app.fqdn + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
 function testRegistryConfig(auth, callback) {
    assert.strictEqual(typeof auth, 'object');
    assert.strictEqual(typeof callback, 'function');
@@ -70,13 +73,13 @@ function testRegistryConfig(auth, callback) {
 }

 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.password === constants.SECRET_PLACEHOLDER) newConfig.password = currentConfig.password;
+    if (newConfig.password === exports.SECRET_PLACEHOLDER) newConfig.password = currentConfig.password;
 }

 function removePrivateFields(registryConfig) {
    assert.strictEqual(typeof registryConfig, 'object');

-    if (registryConfig.password) registryConfig.password = constants.SECRET_PLACEHOLDER;
+    if (registryConfig.password) registryConfig.password = exports.SECRET_PLACEHOLDER;

    return registryConfig;
 }
@@ -176,53 +179,13 @@ function downloadImage(manifest, callback) {

    debug('downloadImage %s', manifest.dockerImage);

-    const image = gConnection.getImage(manifest.dockerImage);
+    var attempt = 1;

-    image.inspect(function (error, result) {
-        if (!error && result) return callback(null); // image is already present locally
+    async.retry({ times: 10, interval: 5000, errorFilter: e => e.reason !== BoxError.NOT_FOUND }, function (retryCallback) {
+        debug('Downloading image %s. attempt: %s', manifest.dockerImage, attempt++);

-        let attempt = 1;
-
-        async.retry({ times: 10, interval: 5000, errorFilter: e => e.reason !== BoxError.NOT_FOUND }, function (retryCallback) {
-            debug('Downloading image %s. attempt: %s', manifest.dockerImage, attempt++);
-
-            pullImage(manifest, retryCallback);
-        }, callback);
-    });
-}
-
-function getBinds(app, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (app.mounts.length === 0) return callback(null);
-
-    let binds = [];
-
-    volumes.list(function (error, result) {
-        if (error) return callback(error);
-        let volumesById = {};
-        result.forEach(r => volumesById[r.id] = r);
-
-        for (const mount of app.mounts) {
-            const volume = volumesById[mount.volumeId];
-            binds.push(`${volume.hostPath}:/media/${volume.name}:${mount.readOnly ? 'ro' : 'rw'}`);
-        }
-
-        callback(null, binds);
-    });
-}
-
-function getLowerUpIp() { // see getifaddrs and IFF_LOWER_UP and netdevice
-    const ni = os.networkInterfaces(); // { lo: [], eth0: [] }
-    for (const iname of Object.keys(ni)) {
-        if (iname === 'lo') continue;
-        for (const address of ni[iname]) {
-            if (!address.internal && address.family === 'IPv4') return address.address;
-        }
-    }
-
-    return null;
+        pullImage(manifest, retryCallback);
+    }, callback);
 }

 function createSubcontainer(app, name, cmd, options, callback) {
@@ -232,11 +195,12 @@ function createSubcontainer(app, name, cmd, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    let isAppContainer = !cmd; // non app-containers are like scheduler
+    let isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers

    var manifest = app.manifest;
    var exposedPorts = {}, dockerPortBindings = { };
    var domain = app.fqdn;
+    const hostname = isAppContainer ? app.id : name;

    const envPrefix = manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_';

@@ -250,6 +214,11 @@ function createSubcontainer(app, name, cmd, options, callback) {
        `${envPrefix}APP_DOMAIN=${domain}`
    ];

+    // docker portBindings requires ports to be exposed
+    exposedPorts[manifest.httpPort + '/tcp'] = {};
+
+    dockerPortBindings[manifest.httpPort + '/tcp'] = [ { HostIp: '127.0.0.1', HostPort: app.httpPort + '' } ];
+
    var portEnv = [];
    for (let portName in app.portBindings) {
        const hostPort = app.portBindings[portName];
@@ -258,121 +227,104 @@ function createSubcontainer(app, name, cmd, options, callback) {

        var containerPort = ports[portName].containerPort || hostPort;

-        // docker portBindings requires ports to be exposed
        exposedPorts[`${containerPort}/${portType}`] = {};
        portEnv.push(`${portName}=${hostPort}`);

-        const hostIp = hostPort === 53 ? getLowerUpIp() : '0.0.0.0'; // port 53 is special because it is possibly taken by systemd-resolved
-        dockerPortBindings[`${containerPort}/${portType}`] = [ { HostIp: hostIp, HostPort: hostPort + '' } ];
+        dockerPortBindings[`${containerPort}/${portType}`] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
    }

    let appEnv = [];
    Object.keys(app.env).forEach(function (name) { appEnv.push(`${name}=${app.env[name]}`); });

-    let memoryLimit = apps.getMemoryLimit(app);
+    // first check db record, then manifest
+    var memoryLimit = app.memoryLimit || manifest.memoryLimit || 0;
+
+    if (memoryLimit === -1) { // unrestricted
+        memoryLimit = 0;
+    } else if (memoryLimit === 0 || memoryLimit < constants.DEFAULT_MEMORY_LIMIT) { // ensure we never go below minimum (in case we change the default)
+        memoryLimit = constants.DEFAULT_MEMORY_LIMIT;
+    }

    // give scheduler tasks twice the memory limit since background jobs take more memory
    // if required, we can make this a manifest and runtime argument later
    if (!isAppContainer) memoryLimit *= 2;

-    services.getEnvironment(app, function (error, addonEnv) {
+    addons.getEnvironment(app, function (error, addonEnv) {
        if (error) return callback(error);

-        getBinds(app, function (error, binds) {
-            if (error) return callback(error);
-
-            let containerOptions = {
-                name: name, // for referencing containers
-                Tty: isAppContainer,
-                Image: app.manifest.dockerImage,
-                Cmd: (isAppContainer && app.debugMode && app.debugMode.cmd) ? app.debugMode.cmd : cmd,
-                Env: stdEnv.concat(addonEnv).concat(portEnv).concat(appEnv),
-                ExposedPorts: isAppContainer ? exposedPorts : { },
-                Volumes: { // see also ReadonlyRootfs
-                    '/tmp': {},
-                    '/run': {}
-                },
-                Labels: {
-                    'fqdn': app.fqdn,
-                    'appId': app.id,
-                    'isSubcontainer': String(!isAppContainer),
-                    'isCloudronManaged': String(true)
-                },
-                HostConfig: {
-                    Mounts: services.getMountsSync(app, app.manifest.addons),
-                    Binds: binds, // ideally, we have to use 'Mounts' but we have to create volumes then
-                    LogConfig: {
-                        Type: 'syslog',
-                        Config: {
-                            'tag': app.id,
-                            'syslog-address': 'udp://127.0.0.1:2514', // see apps.js:validatePortBindings()
-                            'syslog-format': 'rfc5424'
-                        }
-                    },
-                    Memory: system.getMemoryAllocation(memoryLimit),
-                    MemorySwap: memoryLimit, // Memory + Swap
-                    PortBindings: isAppContainer ? dockerPortBindings : { },
-                    PublishAllPorts: false,
-                    ReadonlyRootfs: app.debugMode ? !!app.debugMode.readonlyRootfs : true,
-                    RestartPolicy: {
-                        'Name': isAppContainer ? 'unless-stopped' : 'no',
-                        'MaximumRetryCount': 0
-                    },
-                    CpuShares: app.cpuShares,
-                    VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ],
-                    SecurityOpt: [ 'apparmor=docker-cloudron-app' ],
-                    CapAdd: [],
-                    CapDrop: []
-                }
-            };
-
-            // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail
-            // location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns
-            // name to look up the internal docker ip. this makes curl from within container fail
-            // Note that Hostname has no effect on DNS. We have to use the --net-alias for dns.
-            // Hostname cannot be set with container NetworkMode. Subcontainers run is the network space of the app container
-            // This is done to prevent lots of up/down events and iptables locking
-            if (isAppContainer) {
-                containerOptions.Hostname = app.id;
-                containerOptions.HostConfig.NetworkMode = 'cloudron'; // user defined bridge network
-                containerOptions.HostConfig.Dns = ['172.18.0.1']; // use internal dns
-                containerOptions.HostConfig.DnsSearch = ['.']; // use internal dns
-
-                containerOptions.NetworkingConfig = {
-                    EndpointsConfig: {
-                        cloudron: {
-                            IPAMConfig: {
-                                IPv4Address: app.containerIp
-                            },
-                            Aliases: [ name ] // adds hostname entry with container name
-                        }
+        // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail
+        // location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns
+        // name to look up the internal docker ip. this makes curl from within container fail
+        // Note that Hostname has no effect on DNS. We have to use the --net-alias for dns.
+        // Hostname cannot be set with container NetworkMode
+        var containerOptions = {
+            name: name, // for referencing containers
+            Tty: isAppContainer,
+            Hostname: hostname,
+            Image: app.manifest.dockerImage,
+            Cmd: (isAppContainer && app.debugMode && app.debugMode.cmd) ? app.debugMode.cmd : cmd,
+            Env: stdEnv.concat(addonEnv).concat(portEnv).concat(appEnv),
+            ExposedPorts: isAppContainer ? exposedPorts : { },
+            Volumes: { // see also ReadonlyRootfs
+                '/tmp': {},
+                '/run': {}
+            },
+            Labels: {
+                'fqdn': app.fqdn,
+                'appId': app.id,
+                'isSubcontainer': String(!isAppContainer),
+                'isCloudronManaged': String(true)
+            },
+            HostConfig: {
+                Mounts: addons.getMountsSync(app, app.manifest.addons),
+                LogConfig: {
+                    Type: 'syslog',
+                    Config: {
+                        'tag': app.id,
+                        'syslog-address': 'udp://127.0.0.1:2514', // see apps.js:validatePortBindings()
+                        'syslog-format': 'rfc5424'
                    }
-                };
-            } else {
-                containerOptions.HostConfig.NetworkMode = `container:${app.containerId}`; // scheduler containers must have same IP as app for various addon auth
+                },
+                Memory: memoryLimit / 2,
+                MemorySwap: memoryLimit, // Memory + Swap
+                PortBindings: isAppContainer ? dockerPortBindings : { },
+                PublishAllPorts: false,
+                ReadonlyRootfs: app.debugMode ? !!app.debugMode.readonlyRootfs : true,
+                RestartPolicy: {
+                    'Name': isAppContainer ? 'unless-stopped' : 'no',
+                    'MaximumRetryCount': 0
+                },
+                CpuShares: app.cpuShares,
+                VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ],
+                NetworkMode: 'cloudron', // user defined bridge network
+                Dns: ['172.18.0.1'], // use internal dns
+                DnsSearch: ['.'], // use internal dns
+                SecurityOpt: [ 'apparmor=docker-cloudron-app' ]
+            },
+            NetworkingConfig: {
+                EndpointsConfig: {
+                    cloudron: {
+                        Aliases: [ name ] // this allows sub-containers reach app containers by name
+                    }
+                }
            }
+        };

-            var capabilities = manifest.capabilities || [];
+        var capabilities = manifest.capabilities || [];
+        if (capabilities.includes('net_admin')) {
+            containerOptions.HostConfig.CapAdd = [
+                'NET_ADMIN'
+            ];
+        }

-            // https://docs-stage.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
-            if (capabilities.includes('net_admin')) containerOptions.HostConfig.CapAdd.push('NET_ADMIN', 'NET_RAW');
-            if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping
-            if (!capabilities.includes('ping')) containerOptions.HostConfig.CapDrop.push('NET_RAW'); // NET_RAW is included by default by Docker
+        containerOptions = _.extend(containerOptions, options);

-            if (capabilities.includes('vaapi') && safe.fs.existsSync('/dev/dri')) {
-                containerOptions.HostConfig.Devices = [
-                    { PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' }
-                ];
-            }
+        debugApp(app, 'Creating container for %s', app.manifest.dockerImage);

-            containerOptions = _.extend(containerOptions, options);
+        gConnection.createContainer(containerOptions, function (error, container) {
+            if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

-            gConnection.createContainer(containerOptions, function (error, container) {
-                if (error && error.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
-                if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
-
-                callback(null, container);
-            });
+            callback(null, container);
        });
    });
 }
@@ -386,6 +338,7 @@ function startContainer(containerId, callback) {
    assert.strictEqual(typeof callback, 'function');

    var container = gConnection.getContainer(containerId);
+    debug('Starting container %s', containerId);

    container.start(function (error) {
        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -401,6 +354,7 @@ function restartContainer(containerId, callback) {
    assert.strictEqual(typeof callback, 'function');

    var container = gConnection.getContainer(containerId);
+    debug('Restarting container %s', containerId);

    container.restart(function (error) {
        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -421,6 +375,7 @@ function stopContainer(containerId, callback) {
    }

    var container = gConnection.getContainer(containerId);
+    debug('Stopping container %s', containerId);

    var options = {
        t: 10 // wait for 10 seconds before killing it
@@ -429,18 +384,24 @@ function stopContainer(containerId, callback) {
    container.stop(options, function (error) {
        if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error stopping container:' + error.message));

-        container.wait(function (error/*, data */) {
+        debug('Waiting for container ' + containerId);
+
+        container.wait(function (error, data) {
            if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error waiting on container:' + error.message));

+            debug('Container %s stopped with status code [%s]', containerId, data ? String(data.StatusCode) : '');
+
            return callback(null);
        });
    });
 }

-function deleteContainer(containerId, callback) { // id can also be name
+function deleteContainer(containerId, callback) {
    assert(!containerId || typeof containerId === 'string');
    assert.strictEqual(typeof callback, 'function');

+    debug('deleting container %s', containerId);
+
    if (containerId === null) return callback(null);

    var container = gConnection.getContainer(containerId);
@@ -467,6 +428,8 @@ function deleteContainers(appId, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

+    debug('deleting containers of %s', appId);
+
    let labels = [ 'appId=' + appId ];
    if (options.managedOnly) labels.push('isCloudronManaged=true');

@@ -483,6 +446,8 @@ function stopContainers(appId, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof callback, 'function');

+    debug('Stopping containers of %s', appId);
+
    gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

@@ -549,29 +514,13 @@ function inspect(containerId, callback) {
    var container = gConnection.getContainer(containerId);

    container.inspect(function (error, result) {
-        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to find container ${containerId}`));
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
        if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));

        callback(null, result);
    });
 }

-function getContainerIp(containerId, callback) {
-    assert.strictEqual(typeof containerId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (constants.TEST) return callback(null, '127.0.5.5');
-
-    inspect(containerId, function (error, result) {
-        if (error) return callback(error);
-
-        const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
-        if (!ip) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error getting container IP'));
-
-        callback(null, ip);
-    });
-}
-
 function execContainer(containerId, options, callback) {
    assert.strictEqual(typeof containerId, 'string');
    assert.strictEqual(typeof options, 'object');
@@ -624,10 +573,10 @@ function memoryUsage(containerId, callback) {
    });
 }

-function createVolume(name, volumeDataDir, labels, callback) {
+function createVolume(app, name, volumeDataDir, callback) {
+    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof volumeDataDir, 'string');
-    assert.strictEqual(typeof labels, 'object');
    assert.strictEqual(typeof callback, 'function');

    const volumeOptions = {
@@ -638,7 +587,10 @@ function createVolume(name, volumeDataDir, labels, callback) {
            device: volumeDataDir,
            o: 'bind'
        },
-        Labels: labels
+        Labels: {
+            'fqdn': app.fqdn,
+            'appId': app.id
+        },
    };

    // requires sudo because the path can be outside appsdata
@@ -653,7 +605,8 @@ function createVolume(name, volumeDataDir, labels, callback) {
    });
 }

-function clearVolume(name, options, callback) {
+function clearVolume(app, name, options, callback) {
+    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');
@@ -673,13 +626,14 @@ function clearVolume(name, options, callback) {
 }

 // this only removes the volume and not the data
-function removeVolume(name, callback) {
+function removeVolume(app, name, callback) {
+    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof callback, 'function');

    let volume = gConnection.getVolume(name);
    volume.remove(function (error) {
-        if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume: ${error.message}`));
+        if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume of ${app.id} ${error.message}`));

        callback();
    });
@@ -694,17 +648,3 @@ function info(callback) {
        callback(null, result);
    });
 }
-
-function update(name, memory, memorySwap, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof memory, 'number');
-    assert.strictEqual(typeof memorySwap, 'number');
-    assert.strictEqual(typeof callback, 'function');
-
-    const args = `update --memory ${memory} --memory-swap ${memorySwap} ${name}`.split(' ');
-    // scale back db containers, if possible. this is retried because updating memory constraints can fail
-    // with failed to write to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/xx/memory.memsw.limit_in_bytes: device or resource busy
-    async.retry({ times: 10, interval: 60 * 1000 }, function (retryCallback) {
-        shell.spawn(`update(${name})`, '/usr/bin/docker', args, { }, retryCallback);
-    }, callback);
-}
@@ -23,6 +23,11 @@ var apps = require('./apps.js'),
 var gHttpServer = null;

 function authorizeApp(req, res, next) {
+    // TODO add here some authorization
+    // - block apps not using the docker addon
+    // - block calls regarding platform containers
+    // - only allow managing and inspection of containers belonging to the app
+
    // make the tests pass for now
    if (constants.TEST) {
        req.app = { id: 'testappid' };
@@ -55,12 +60,10 @@ function attachDockerRequest(req, res, next) {
        // Force node to send out the headers, this is required for the /container/wait api to make the docker cli proceed
        res.write(' ');

-        dockerResponse.on('error', function (error) { debug('dockerResponse error:', error); });
+        dockerResponse.on('error', function (error) { console.error('dockerResponse error:', error); });
        dockerResponse.pipe(res, { end: true });
    });

-    req.dockerRequest.on('error', () => {}); // abort() throws
-
    next();
 }

@@ -71,21 +74,22 @@ function containersCreate(req, res, next) {
    safe.set(req.body, 'Labels',  _.extend({ }, safe.query(req.body, 'Labels'), { appId: req.app.id, isCloudronManaged: String(false) }));    // overwrite the app id to track containers of an app
    safe.set(req.body, 'HostConfig.LogConfig', { Type: 'syslog', Config: { 'tag': req.app.id, 'syslog-address': 'udp://127.0.0.1:2514', 'syslog-format': 'rfc5424' }});

-    const appDataDir = path.join(paths.APPS_DATA_DIR, req.app.id, 'data');
+    const appDataDir = path.join(paths.APPS_DATA_DIR, req.app.id, 'data'),
+        dockerDataDir = path.join(paths.APPS_DATA_DIR, req.app.id, 'docker');

-    debug('Original bind mounts:', req.body.HostConfig.Binds);
+    debug('Original volume binds:', req.body.HostConfig.Binds);

    let binds = [];
    for (let bind of (req.body.HostConfig.Binds || [])) {
-        if (!bind.startsWith('/app/data/')) {
-            req.dockerRequest.abort();
-            return next(new HttpError(400, 'Binds must be under /app/data/'));
-        }
-
-        binds.push(bind.replace(new RegExp('^/app/data/'), appDataDir + '/'));
+        if (bind.startsWith(appDataDir)) binds.push(bind); // eclipse will inspect docker to find out the host folders and pass that to child containers
+        else if (bind.startsWith('/app/data')) binds.push(bind.replace(new RegExp('^/app/data'), appDataDir));
+        else binds.push(`${dockerDataDir}/${bind}`);
    }

-    debug('Rewritten bind mounts:', binds);
+    // cleanup the paths from potential double slashes
+    binds = binds.map(function (bind) { return bind.replace(/\/+/g, '/'); });
+
+    debug('Rewritten volume binds:', binds);
    safe.set(req.body, 'HostConfig.Binds', binds);

    let plainBody = JSON.stringify(req.body);
@@ -113,9 +117,6 @@ function start(callback) {
    assert(gHttpServer === null, 'Already started');

    let json = middleware.json({ strict: true });
-
-    // we protect container create as the app/admin can otherwise mount random paths (like the ghost file)
-    // protected other paths is done by preventing install/exec access of apps using docker addon
    let router = new express.Router();
    router.post('/:version/containers/create', containersCreate);

@@ -136,7 +137,7 @@ function start(callback) {
        .use(middleware.lastMile());

    gHttpServer = http.createServer(proxyServer);
-    gHttpServer.listen(constants.DOCKER_PROXY_PORT, '172.18.0.1', callback);
+    gHttpServer.listen(constants.DOCKER_PROXY_PORT, '0.0.0.0', callback);

    // Overwrite the default 2min request timeout. This is required for large builds for example
    gHttpServer.setTimeout(60 * 60 * 1000);
@@ -16,18 +16,14 @@ var assert = require('assert'),
    database = require('./database.js'),
    safe = require('safetydance');

-var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson' ].join(',');
+var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson' ].join(',');

 function postProcess(data) {
    data.config = safe.JSON.parse(data.configJson);
-    delete data.configJson;
-
    data.tlsConfig = safe.JSON.parse(data.tlsConfigJson);
+    delete data.configJson;
    delete data.tlsConfigJson;

-    data.wellKnown = safe.JSON.parse(data.wellKnownJson);
-    delete data.wellKnownJson;
-
    return data;
 }

@@ -55,22 +51,17 @@ function getAll(callback) {
    });
 }

-function add(name, data, callback) {
+function add(name, domain, callback) {
    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof data.zoneName, 'string');
-    assert.strictEqual(typeof data.provider, 'string');
-    assert.strictEqual(typeof data.config, 'object');
-    assert.strictEqual(typeof data.tlsConfig, 'object');
+    assert.strictEqual(typeof domain, 'object');
+    assert.strictEqual(typeof domain.zoneName, 'string');
+    assert.strictEqual(typeof domain.provider, 'string');
+    assert.strictEqual(typeof domain.config, 'object');
+    assert.strictEqual(typeof domain.tlsConfig, 'object');
    assert.strictEqual(typeof callback, 'function');

-    let queries = [
-        { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson) VALUES (?, ?, ?, ?, ?)', args: [ name, data.zoneName, data.provider, JSON.stringify(data.config), JSON.stringify(data.tlsConfig) ] },
-        { query: 'INSERT INTO mail (domain, dkimSelector) VALUES (?, ?)', args: [ name, data.dkimSelector || 'cloudron' ] },
-    ];
-
-    database.transaction(queries, function (error) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'Domain already exists'));
+    database.query('INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson) VALUES (?, ?, ?, ?, ?)', [ name, domain.zoneName, domain.provider, JSON.stringify(domain.config), JSON.stringify(domain.tlsConfig) ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));

        callback(null);
@@ -90,9 +81,6 @@ function update(name, domain, callback) {
        } else if (k === 'tlsConfig') {
            fields.push('tlsConfigJson = ?');
            args.push(JSON.stringify(domain[k]));
-        } else if (k === 'wellKnown') {
-            fields.push('wellKnownJson = ?');
-            args.push(JSON.stringify(domain[k]));
        } else {
            fields.push(k + ' = ?');
            args.push(domain[k]);
@@ -112,12 +100,7 @@ function del(domain, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof callback, 'function');

-    let queries = [
-        { query: 'DELETE FROM mail WHERE domain = ?', args: [ domain ] },
-        { query: 'DELETE FROM domains WHERE domain = ?', args: [ domain ] },
-    ];
-
-    database.transaction(queries, function (error, results) {
+    database.query('DELETE FROM domains WHERE domain=?', [ domain ], function (error, result) {
        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') {
            if (error.message.indexOf('apps_mailDomain_constraint') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by an app or the mailbox of an app. Check the domains of apps and the Email section of each app.'));
            if (error.message.indexOf('subdomains') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more app(s).'));
@@ -125,9 +108,8 @@ function del(domain, callback) {

            return callback(new BoxError(BoxError.CONFLICT, error.message));
        }
-
        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
+        if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));

        callback(null);
    });
@@ -1,41 +1,45 @@
 'use strict';

 module.exports = exports = {
-    add,
-    get,
-    getAll,
-    update,
-    del,
-    clear,
+    add: add,
+    get: get,
+    getAll: getAll,
+    update: update,
+    del: del,
+    clear: clear,

-    fqdn,
-    getName,
+    fqdn: fqdn,
+    getName: getName,

-    getDnsRecords,
-    upsertDnsRecords,
-    removeDnsRecords,
+    getDnsRecords: getDnsRecords,
+    upsertDnsRecords: upsertDnsRecords,
+    removeDnsRecords: removeDnsRecords,

-    waitForDnsRecord,
+    waitForDnsRecord: waitForDnsRecord,

-    removePrivateFields,
-    removeRestrictedFields,
+    removePrivateFields: removePrivateFields,
+    removeRestrictedFields: removeRestrictedFields,

-    validateHostname,
+    validateHostname: validateHostname,

-    makeWildcard,
+    makeWildcard: makeWildcard,

-    parentDomain,
+    parentDomain: parentDomain,

-    checkDnsRecords
+    checkDnsRecords: checkDnsRecords,
+
+    prepareDashboardDomain: prepareDashboardDomain,
+
+    SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8)
 };

 var assert = require('assert'),
+    async = require('async'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:domains'),
    domaindb = require('./domaindb.js'),
    eventlog = require('./eventlog.js'),
-    mail = require('./mail.js'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
    settings = require('./settings.js'),
@@ -44,13 +48,12 @@ var assert = require('assert'),
    util = require('util'),
    _ = require('underscore');

-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 // choose which subdomain backend we use for test purpose we use route53
 function api(provider) {
    assert.strictEqual(typeof provider, 'string');

    switch (provider) {
+    case 'caas': return require('./dns/caas.js');
    case 'cloudflare': return require('./dns/cloudflare.js');
    case 'route53': return require('./dns/route53.js');
    case 'gcdns': return require('./dns/gcdns.js');
@@ -60,7 +63,6 @@ function api(provider) {
    case 'linode': return require('./dns/linode.js');
    case 'namecom': return require('./dns/namecom.js');
    case 'namecheap': return require('./dns/namecheap.js');
-    case 'netcup': return require('./dns/netcup.js');
    case 'noop': return require('./dns/noop.js');
    case 'manual': return require('./dns/manual.js');
    case 'wildcard': return require('./dns/wildcard.js');
@@ -90,12 +92,14 @@ function verifyDnsConfig(dnsConfig, domain, zoneName, provider, callback) {
        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`));
        if (error) return callback(error);

+        result.hyphenatedSubdomains = !!dnsConfig.hyphenatedSubdomains;
+
        callback(null, result);
    });
 }

 function fqdn(location, domainObject) {
-    return location + (location ? '.' : '') + domainObject.domain;
+    return location + (location ? (domainObject.config.hyphenatedSubdomains ? '-' : '.') : '') + domainObject.domain;
 }

 // Hostname validation comes from RFC 1123 (section 2.1)
@@ -129,6 +133,10 @@ function validateHostname(location, domainObject) {
        if (/^[-.]/.test(location)) return new BoxError(BoxError.BAD_FIELD, 'Subdomain cannot start or end with hyphen or dot', { field: 'location' });
    }

+    if (domainObject.config.hyphenatedSubdomains) {
+        if (location.indexOf('.') !== -1) return new BoxError(BoxError.BAD_FIELD, 'Subdomain cannot contain a dot', { field: 'location' });
+    }
+
    return null;
 }

@@ -140,9 +148,10 @@ function validateTlsConfig(tlsConfig, dnsProvider) {
    case 'letsencrypt-prod':
    case 'letsencrypt-staging':
    case 'fallback':
+    case 'caas':
        break;
    default:
-        return new BoxError(BoxError.BAD_FIELD, 'tlsConfig.provider must be fallback, letsencrypt-prod/staging', { field: 'tlsProvider' });
+        return new BoxError(BoxError.BAD_FIELD, 'tlsConfig.provider must be caas, fallback, letsencrypt-prod/staging', { field: 'tlsProvider' });
    }

    if (tlsConfig.wildcard) {
@@ -153,12 +162,6 @@ function validateTlsConfig(tlsConfig, dnsProvider) {
    return null;
 }

-function validateWellKnown(wellKnown) {
-    assert.strictEqual(typeof wellKnown, 'object');
-
-    return null;
-}
-
 function add(domain, data, auditSource, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data.zoneName, 'string');
@@ -168,7 +171,7 @@ function add(domain, data, auditSource, callback) {
    assert.strictEqual(typeof data.tlsConfig, 'object');
    assert.strictEqual(typeof callback, 'function');

-    let { zoneName, provider, config, fallbackCertificate, tlsConfig, dkimSelector } = data;
+    let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;

    if (!tld.isValid(domain)) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
    if (domain.endsWith('.')) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
@@ -185,18 +188,16 @@ function add(domain, data, auditSource, callback) {
        if (error) return callback(error);
    } else {
        fallbackCertificate = reverseProxy.generateFallbackCertificateSync({ domain, config });
-        if (fallbackCertificate.error) return callback(fallbackCertificate.error);
+        if (fallbackCertificate.error) return callback(error);
    }

    let error = validateTlsConfig(tlsConfig, provider);
    if (error) return callback(error);

-    if (!dkimSelector) dkimSelector = 'cloudron-' + settings.adminDomain().replace(/\./g, '');
-
    verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
        if (error) return callback(error);

-        domaindb.add(domain, { zoneName, provider, config: sanitizedConfig, tlsConfig, dkimSelector }, function (error) {
+        domaindb.add(domain, { zoneName: zoneName, provider: provider, config: sanitizedConfig, tlsConfig: tlsConfig }, function (error) {
            if (error) return callback(error);

            reverseProxy.setFallbackCertificate(domain, fallbackCertificate, function (error) {
@@ -204,8 +205,6 @@ function add(domain, data, auditSource, callback) {

                eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });

-                mail.onDomainAdded(domain, NOOP_CALLBACK);
-
                callback();
            });
        });
@@ -253,7 +252,7 @@ function update(domain, data, auditSource, callback) {
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

-    let { zoneName, provider, config, fallbackCertificate, tlsConfig, wellKnown } = data;
+    let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;

    if (settings.isDemo() && (domain === settings.adminDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));

@@ -274,9 +273,6 @@ function update(domain, data, auditSource, callback) {
        error = validateTlsConfig(tlsConfig, provider);
        if (error) return callback(error);

-        error = validateWellKnown(wellKnown, provider);
-        if (error) return callback(error);
-
        if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config);

        verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
@@ -284,10 +280,9 @@ function update(domain, data, auditSource, callback) {

            let newData = {
                config: sanitizedConfig,
-                zoneName,
-                provider,
-                tlsConfig,
-                wellKnown
+                zoneName: zoneName,
+                provider: provider,
+                tlsConfig: tlsConfig
            };

            domaindb.update(domain, newData, function (error) {
@@ -313,15 +308,12 @@ function del(domain, auditSource, callback) {
    assert.strictEqual(typeof callback, 'function');

    if (domain === settings.adminDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain'));
-    if (domain === settings.mailDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove mail domain'));

    domaindb.del(domain, function (error) {
        if (error) return callback(error);

        eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });

-        mail.onDomainRemoved(domain, NOOP_CALLBACK);
-
        return callback(null);
    });
 }
@@ -342,7 +334,19 @@ function getName(domain, location, type) {

    if (location === '') return part;

-    return part ? `${location}.${part}` : location;
+    if (!domain.config.hyphenatedSubdomains) return part ? `${location}.${part}` : location;
+
+    // hyphenatedSubdomains
+    if (type !== 'TXT') return `${location}-${part}`;
+
+    if (location.startsWith('_acme-challenge.')) {
+        return `${location}-${part}`;
+    } else if (location === '_acme-challenge') {
+        const up = part.replace(/^[^.]*\.?/, ''); // this gets the domain one level up
+        return up ? `${location}.${up}` : location;
+    } else {
+        return `${location}.${part}`;
+    }
 }

 function getDnsRecords(location, domain, type, callback) {
@@ -442,7 +446,7 @@ function waitForDnsRecord(location, domain, type, value, options, callback) {

 // removes all fields that are strictly private and should never be returned by API calls
 function removePrivateFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate', 'wellKnown');
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate');
    return api(result.provider).removePrivateFields(result);
 }

@@ -450,16 +454,46 @@ function removePrivateFields(domain) {
 function removeRestrictedFields(domain) {
    var result = _.pick(domain, 'domain', 'zoneName', 'provider');

-    result.config = {}; // always ensure config object
+    // always ensure config object
+    result.config = { hyphenatedSubdomains: !!domain.config.hyphenatedSubdomains };

    return result;
 }

-function makeWildcard(vhost) {
-    assert.strictEqual(typeof vhost, 'string');
+function makeWildcard(hostname) {
+    assert.strictEqual(typeof hostname, 'string');

-    // if the vhost is like *.example.com, this function will do nothing
-    let parts = vhost.split('.');
+    let parts = hostname.split('.');
    parts[0] = '*';
    return parts.join('.');
 }
+
+function prepareDashboardDomain(domain, auditSource, progressCallback, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(domain, function (error, domainObject) {
+        if (error) return callback(error);
+
+        const adminFqdn = fqdn(constants.ADMIN_LOCATION, domainObject);
+
+        sysinfo.getServerIp(function (error, ip) {
+            if (error) return callback(error);
+
+            async.series([
+                (done) => { progressCallback({ percent: 10, message: `Updating DNS of ${adminFqdn}` }); done(); },
+                upsertDnsRecords.bind(null, constants.ADMIN_LOCATION, domain, 'A', [ ip ]),
+                (done) => { progressCallback({ percent: 40, message: `Waiting for DNS of ${adminFqdn}` }); done(); },
+                waitForDnsRecord.bind(null, constants.ADMIN_LOCATION, domain, 'A', ip, { interval: 30000, times: 50000 }),
+                (done) => { progressCallback({ percent: 70, message: `Getting certificate of ${adminFqdn}` }); done(); },
+                reverseProxy.ensureCertificate.bind(null, fqdn(constants.ADMIN_LOCATION, domainObject), domain, auditSource)
+            ], function (error) {
+                if (error) return callback(error);
+
+                callback(null);
+            });
+        });
+    });
+}
@@ -1,12 +1,11 @@
 'use strict';

 exports = module.exports = {
-    add,
-    upsert,
-    get,
-    getAllPaged,
-    getByCreationTime,
-    cleanup,
+    add: add,
+    get: get,
+    getAllPaged: getAllPaged,
+    getByCreationTime: getByCreationTime,
+    cleanup: cleanup,

    // keep in sync with webadmin index.js filter
    ACTION_ACTIVATE: 'cloudron.activate',
@@ -40,7 +39,6 @@ exports = module.exports = {
    ACTION_DOMAIN_UPDATE: 'domain.update',
    ACTION_DOMAIN_REMOVE: 'domain.remove',

-    ACTION_MAIL_LOCATION: 'mail.location',
    ACTION_MAIL_ENABLED: 'mail.enabled',
    ACTION_MAIL_DISABLED: 'mail.disabled',
    ACTION_MAIL_MAILBOX_ADD: 'mail.box.add',
@@ -58,15 +56,10 @@ exports = module.exports = {

    ACTION_USER_ADD: 'user.add',
    ACTION_USER_LOGIN: 'user.login',
-    ACTION_USER_LOGOUT: 'user.logout',
    ACTION_USER_REMOVE: 'user.remove',
    ACTION_USER_UPDATE: 'user.update',
    ACTION_USER_TRANSFER: 'user.transfer',

-    ACTION_VOLUME_ADD: 'volume.add',
-    ACTION_VOLUME_UPDATE: 'volume.update',
-    ACTION_VOLUME_REMOVE: 'volume.remove',
-
    ACTION_DYNDNS_UPDATE: 'dyndns.update',

    ACTION_SUPPORT_TICKET: 'support.ticket',
@@ -92,24 +85,9 @@ function add(action, source, data, callback) {

    callback = callback || NOOP_CALLBACK;

-    eventlogdb.add(uuid.v4(), action, source, data, function (error, id) {
-        if (error) return callback(error);
-
-        callback(null, { id: id });
-
-        notifications.onEvent(id, action, source, data, NOOP_CALLBACK);
-    });
-}
-
-function upsert(action, source, data, callback) {
-    assert.strictEqual(typeof action, 'string');
-    assert.strictEqual(typeof source, 'object');
-    assert.strictEqual(typeof data, 'object');
-    assert(!callback || typeof callback === 'function');
-
-    callback = callback || NOOP_CALLBACK;
-
-    eventlogdb.upsert(uuid.v4(), action, source, data, function (error, id) {
+    // we do only daily upserts for login actions, so they don't spam the db
+    var api = action === exports.ACTION_USER_LOGIN ? eventlogdb.upsert : eventlogdb.add;
+    api(uuid.v4(), action, source, data, function (error, id) {
        if (error) return callback(error);

        callback(null, { id: id });
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Girish Ramakrishnan	92f143163b	make app password work with sftp (cherry picked from commit `9faae96d61`) (cherry picked from commit `f3b979f112`)	2020-03-26 21:56:45 -07:00
Girish Ramakrishnan	3c1a1f1b81	5.0.6 changelog	2020-03-26 19:30:52 -07:00
Girish Ramakrishnan	4bc7c70e2e	make eventlog routes owner only (cherry picked from commit `007a8d248d`)	2020-03-26 19:08:43 -07:00
Girish Ramakrishnan	cc6ddf50b1	5.0.5 changes	2020-03-25 07:57:39 -07:00