graphite: restart collectd as well

(cherry picked from commit 0447dce0d6)

CHANGES

@@ -2155,3 +2155,86 @@
 * mail: fix crash because of write after timeout closure
 * scaleway: fix installation issue where THP is not enabled in kernel
 
+[6.1.0]
+* mail: update haraka to 2.8.27. this fixes zero-length queue file crash
+* update: set/unset appStoreId from the update route
+* proxyauth: Do not follow redirects
+* proxyauth: add 2FA
+* appstore: add category translations
+* appstore: add media category
+* prepend the version to assets when sourcing to avoid cache hits on update
+* filemanger: list volumes of the app
+* Display upload size and size progress
+* nfs: chown the backups for hardlinks to work
+* remove user add/remove/role change email notifications
+* persist update indicator across restarts
+* cloudron-setup: add --generate-setup-token
+* dashboard: pass accessToken query param to automatically login
+* wellknown: add a way to set well known docs
+* oom: notification mails have links to dashboard
+* collectd: do not install xorg* packages
+* apptask: backup/restore tasks now use the backup memory limit configuration
+* eventlog: add logout event
+* mailbox: include alias in mailbox search
+* proxyAuth: add path exclusion
+* turn: fix for CVE-2020-26262
+* app password: fix regression where apps are not listed anymore in the UI
+* Support for multiDomain apps (domain aliases)
+* netcup: add dns provider
+* Container swap size is now dynamically determined based on system RAM/swap ratio
+
+[6.1.1]
+* Fix bug where platform does not start if memory limits could not be applied
+
+[6.1.2]
+* App disk usage was not shown in graphs
+* Email autoconfig
+* Fix SOGo login
+
+[6.2.0]
+* ovh: object storage URL has changed from s3 to storage subdomain
+* ionos: add profit bricks object storage
+* update node to 14.15.4
+* update docker to 20.10.3
+* new base image 3.0.0
+* postgresql updated to 12.5
+* redis updated to 5.0.7
+* dovecot updated to 2.3.7
+* proxyAuth: fix docker UA detection
+* registry config: add UI to disable it
+* update solr to 8.8.1
+* firewall: fix issue where script errored when having more than 15 wl/bl ports
+* If groups are used, do not allow app installation without choosing the access settings
+* tls addon
+* Do not overwrite existing DMARC record
+* Sync dns records
+* Dry run restore
+* linode: show cloudron is installing when user SSHs
+* mysql: disable bin logs
+* Show cancel task button if task is still running after 2 minutes
+* filemanager: fix various bugs involving file names with spaces
+* Change Referrer-policy default to 'same-origin'
+* rsync: preserve and restore symlinks
+* Clean up backups function now removes missing backups
+
+[6.2.1]
+* Avoid updown notifications on full restore
+* Add retries to downloader logic in installer
+
+[6.2.2]
+* Fix ENOBUFS issue with backups when collecting fs metadata
+
+[6.2.3]
+* Fix addon crashes with missing databases
+* Update mail container for LMTP cert fix
+* Fix services view showing yellow icon
+
+[6.2.4]
+* Another addon crash fix
+
+[6.2.5]
+* update: set memory limit properly
+* Fix bug where renew certs button did not work
+* sftp: fix rebuild condition
+* Fix display of user management/dashboard visiblity for email apps
+* graphite: disable tagdb and reduce log noise

baseimage/createAMI

@@ -1,193 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-assertNotEmpty() {
-    : "${!1:? "$1 is not set."}"
-}
-
-readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-export JSON="${SOURCE_DIR}/node_modules/.bin/json"
-
-INSTANCE_TYPE="t2.micro"
-BLOCK_DEVICE="DeviceName=/dev/sda1,Ebs={VolumeSize=20,DeleteOnTermination=true,VolumeType=gp2}"
-SSH_KEY_NAME="id_rsa_yellowtent"
-
-revision=$(git rev-parse HEAD)
-ami_name=""
-server_id=""
-server_ip=""
-destroy_server="yes"
-deploy_env="prod"
-image_id=""
-
-args=$(getopt -o "" -l "revision:,name:,no-destroy,env:,region:" -n "$0" -- "$@")
-eval set -- "${args}"
-
-while true; do
-    case "$1" in
-    --env) deploy_env="$2"; shift 2;;
-    --revision) revision="$2"; shift 2;;
-    --name) ami_name="$2"; shift 2;;
-    --no-destroy) destroy_server="no"; shift 2;;
-    --region)
-        case "$2" in
-        "us-east-1")
-            image_id="ami-6edd3078"
-            security_group="sg-a5e17fd9"
-            subnet_id="subnet-b8fbc0f1"
-            ;;
-        "eu-central-1")
-            image_id="ami-5aee2235"
-            security_group="sg-19f5a770" # everything open on eu-central-1
-            subnet_id=""
-            ;;
-        *)
-            echo "Unknown aws region $2"
-            exit 1
-            ;;
-        esac
-        export AWS_DEFAULT_REGION="$2"    # used by the aws cli tool
-        shift 2
-        ;;
-    --) break;;
-    *) echo "Unknown option $1"; exit 1;;
-    esac
-done
-
-# TODO fix this
-export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY}"
-export AWS_SECRET_ACCESS_KEY="${AWS_ACCESS_SECRET}"
-
-readonly ssh_keys="${HOME}/.ssh/id_rsa_yellowtent"
-readonly SSH="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-
-if [[ ! -f "${ssh_keys}" ]]; then
-    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
-    exit 1
-fi
-
-if [[ -z "${image_id}" ]]; then
-    echo "--region is required (us-east-1 or eu-central-1)"
-    exit 1
-fi
-
-function get_pretty_revision() {
-    local git_rev="$1"
-    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
-
-    echo "${sha1}"
-}
-
-function wait_for_ssh() {
-    echo "=> Waiting for ssh connection"
-    while true; do
-        echo -n "."
-
-        if $SSH ubuntu@${server_ip} echo "hello"; then
-            echo ""
-            break
-        fi
-
-        sleep 5
-    done
-}
-
-now=$(date "+%Y-%m-%d-%H%M%S")
-pretty_revision=$(get_pretty_revision "${revision}")
-
-if [[ -z "${ami_name}" ]]; then
-    ami_name="box-${deploy_env}-${pretty_revision}-${now}"
-fi
-
-echo "=> Create EC2 instance"
-id=$(aws ec2 run-instances --image-id "${image_id}" --instance-type "${INSTANCE_TYPE}" --security-group-ids "${security_group}" --block-device-mappings "${BLOCK_DEVICE}" --key-name "${SSH_KEY_NAME}" --subnet-id "${subnet_id}" --associate-public-ip-address \
-    | $JSON Instances \
-    | $JSON 0.InstanceId)
-
-[[ -z "$id" ]] && exit 1
-echo "Instance created  ID $id"
-
-echo "=> Waiting for instance to get a public IP"
-while true; do
-    server_ip=$(aws ec2 describe-instances --instance-ids ${id} \
-        | $JSON Reservations.0.Instances \
-        | $JSON 0.PublicIpAddress)
-
-    if [[ ! -z "${server_ip}" ]]; then
-        echo ""
-        break
-    fi
-
-    echo -n "."
-    sleep 1
-done
-
-echo "Got public IP ${server_ip}"
-
-wait_for_ssh
-
-echo "=> Fetching cloudron-setup"
-while true; do
-
-    if $SSH ubuntu@${server_ip} wget "https://cloudron.io/cloudron-setup" -O "cloudron-setup"; then
-        echo ""
-        break
-    fi
-
-    echo -n "."
-    sleep 5
-done
-
-echo "=> Running cloudron-setup"
-$SSH ubuntu@${server_ip} sudo /bin/bash "cloudron-setup" --env "${deploy_env}" --provider "ami" --skip-reboot
-
-wait_for_ssh
-
-echo "=> Removing ssh key"
-$SSH ubuntu@${server_ip} sudo rm /home/ubuntu/.ssh/authorized_keys /root/.ssh/authorized_keys
-
-echo "=> Creating AMI"
-image_id=$(aws ec2 create-image --instance-id "${id}" --name "${ami_name}" | $JSON ImageId)
-[[ -z "$id" ]] && exit 1
-echo "Creating AMI with Id ${image_id}"
-
-echo "=> Waiting for AMI to be created"
-while true; do
-    state=$(aws ec2 describe-images --image-ids ${image_id} \
-        | $JSON Images \
-        | $JSON 0.State)
-
-    if [[ "${state}" == "available" ]]; then
-        echo ""
-        break
-    fi
-
-    echo -n "."
-    sleep 5
-done
-
-if [[ "${destroy_server}" == "yes" ]]; then
-    echo "=> Deleting EC2 instance"
-
-    while true; do
-        state=$(aws ec2 terminate-instances --instance-id "${id}" \
-            | $JSON TerminatingInstances \
-            | $JSON 0.CurrentState.Name)
-
-        if [[ "${state}" == "shutting-down" ]]; then
-            echo ""
-            break
-        fi
-
-        echo -n "."
-        sleep 5
-    done
-fi
-
-echo ""
-echo "Done."
-echo ""
-echo "New AMI is: ${image_id}"
-echo ""

baseimage/digitalocean.sh

@@ -1,261 +0,0 @@
-#!/bin/bash
-
-if [[ -z "${DIGITAL_OCEAN_TOKEN}" ]]; then
-    echo "Script requires DIGITAL_OCEAN_TOKEN env to be set"
-    exit 1
-fi
-
-if [[ -z "${JSON}" ]]; then
-    echo "Script requires JSON env to be set to path of JSON binary"
-    exit 1
-fi
-
-readonly CURL="curl --retry 5 -s -u ${DIGITAL_OCEAN_TOKEN}:"
-
-function debug() {
-    echo "$@" >&2
-}
-
-function get_ssh_key_id() {
-    id=$($CURL "https://api.digitalocean.com/v2/account/keys" \
-        | $JSON ssh_keys \
-        | $JSON -c "this.name === \"$1\"" \
-        | $JSON 0.id)
-    [[ -z "$id" ]] && exit 1
-    echo "$id"
-}
-
-function create_droplet() {
-    local ssh_key_id="$1"
-    local box_name="$2"
-
-    local image_region="sfo2"
-    local ubuntu_image_slug="ubuntu-16-04-x64"
-    local box_size="1gb"
-
-    local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"
-
-    id=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets" | $JSON droplet.id)
-    [[ -z "$id" ]] && exit 1
-    echo "$id"
-}
-
-function get_droplet_ip() {
-    local droplet_id="$1"
-    ip=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}" | $JSON "droplet.networks.v4[0].ip_address")
-    [[ -z "$ip" ]] && exit 1
-    echo "$ip"
-}
-
-function get_droplet_id() {
-    local droplet_name="$1"
-    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=200" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
-    [[ -z "$id" ]] && exit 1
-    echo "$id"
-}
-
-function power_off_droplet() {
-    local droplet_id="$1"
-    local data='{"type":"power_off"}'
-    local response=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions")
-    local event_id=`echo "${response}" | $JSON action.id`
-
-    if [[ -z "${event_id}" ]]; then
-        debug "Got no event id, assuming already powered off."
-        debug "Response: ${response}"
-        return
-    fi
-
-    debug "Powered off droplet. Event id: ${event_id}"
-    debug -n "Waiting for droplet to power off"
-
-    while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
-        if [[ "${event_status}" == "completed" ]]; then
-            break
-        fi
-        debug -n "."
-        sleep 10
-    done
-    debug ""
-}
-
-function power_on_droplet() {
-    local droplet_id="$1"
-    local data='{"type":"power_on"}'
-    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
-
-    debug "Powered on droplet. Event id: ${event_id}"
-
-    if [[ -z "${event_id}" ]]; then
-        debug "Got no event id, assuming already powered on"
-        return
-    fi
-
-    debug -n "Waiting for droplet to power on"
-
-    while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
-        if [[ "${event_status}" == "completed" ]]; then
-            break
-        fi
-        debug -n "."
-        sleep 10
-    done
-    debug ""
-}
-
-function get_image_id() {
-    local snapshot_name="$1"
-    local image_id=""
-
-    if ! response=$($CURL "https://api.digitalocean.com/v2/images?per_page=200"); then
-        echo "Failed to get image listing. ${response}"
-        return 1
-    fi
-
-    if ! image_id=$(echo "$response" \
-       | $JSON images \
-       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id); then
-        echo "Failed to parse curl response: ${response}"
-        return 1
-    fi
-
-    if [[ -z "${image_id}" ]]; then
-        echo "Failed to get image id of ${snapshot_name}. reponse: ${response}"
-        return 1
-    fi
-
-    echo "${image_id}"
-}
-
-function snapshot_droplet() {
-    local droplet_id="$1"
-    local snapshot_name="$2"
-    local data="{\"type\":\"snapshot\",\"name\":\"${snapshot_name}\"}"
-    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
-
-    debug "Droplet snapshotted as ${snapshot_name}. Event id: ${event_id}"
-    debug -n "Waiting for snapshot to complete"
-
-    while true; do
-        if ! response=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}"); then
-            echo "Could not get action status. ${response}"
-            continue
-        fi
-        if ! event_status=$(echo "${response}" | $JSON action.status); then
-            echo "Could not parse action.status from response. ${response}"
-            continue
-        fi
-        if [[ "${event_status}" == "completed" ]]; then
-            break
-        fi
-        debug -n "."
-        sleep 10
-    done
-    debug "! done"
-
-    if ! image_id=$(get_image_id "${snapshot_name}"); then
-        return 1
-    fi
-    echo "${image_id}"
-}
-
-function destroy_droplet() {
-    local droplet_id="$1"
-    # TODO: check for 204 status
-    $CURL -X DELETE "https://api.digitalocean.com/v2/droplets/${droplet_id}"
-    debug "Droplet destroyed"
-    debug ""
-}
-
-function transfer_image() {
-    local image_id="$1"
-    local region_slug="$2"
-    local data="{\"type\":\"transfer\",\"region\":\"${region_slug}\"}"
-    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/images/${image_id}/actions" | $JSON action.id`
-    echo "${event_id}"
-}
-
-function wait_for_image_event() {
-    local image_id="$1"
-    local event_id="$2"
-
-    debug -n "Waiting for ${event_id}"
-
-    while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/images/${image_id}/actions/${event_id}" | $JSON action.status`
-        if [[ "${event_status}" == "completed" ]]; then
-            break
-        fi
-        debug -n "."
-        sleep 10
-    done
-    debug ""
-}
-
-function transfer_image_to_all_regions() {
-    local image_id="$1"
-
-    xfer_events=()
-    image_regions=(ams2) ## sfo1 is where the image is created
-    for image_region in ${image_regions[@]}; do
-        xfer_event=$(transfer_image ${image_id} ${image_region})
-        echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}"
-        xfer_events+=("${xfer_event}")
-        sleep 1
-    done
-
-    echo "Image transfer initiated, but they will take some time to get transferred."
-
-    for xfer_event in ${xfer_events[@]}; do
-        $vps wait_for_image_event "${image_id}" "${xfer_event}"
-    done
-}
-
-if [[ $# -lt 1 ]]; then
-    debug "<command> <params...>"
-    exit 1
-fi
-
-case $1 in
-get_ssh_key_id)
-    get_ssh_key_id "${@:2}"
-    ;;
-
-create)
-    create_droplet "${@:2}"
-    ;;
-
-get_id)
-    get_droplet_id "${@:2}"
-    ;;
-
-get_ip)
-    get_droplet_ip "${@:2}"
-    ;;
-
-power_on)
-    power_on_droplet "${@:2}"
-    ;;
-
-power_off)
-    power_off_droplet "${@:2}"
-    ;;
-
-snapshot)
-    snapshot_droplet "${@:2}"
-    ;;
-
-destroy)
-    destroy_droplet "${@:2}"
-    ;;
-
-transfer_image_to_all_regions)
-    transfer_image_to_all_regions "${@:2}"
-    ;;
-
-*)
-    echo "Unknown command $1"
-    exit 1
-esac

baseimage/initializeBaseUbuntuImage.sh

@@ -32,8 +32,9 @@ debconf-set-selections <<< 'mysql-server mysql-server/root_password_again passwo
 
 gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
 mysql_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "mysql-server-8.0" || echo "mysql-server-5.7")
-apt-get -y install \
+apt-get -y install --no-install-recommends \
     acl \
+    apparmor \
     build-essential \
     cifs-utils \
     cron \
@@ -54,6 +55,7 @@ apt-get -y install \
     tzdata \
     unattended-upgrades \
     unbound \
+    unzip \
     xfsprogs
 
 echo "==> installing nginx for xenial for TLSv3 support"
@@ -63,18 +65,19 @@ apt install -y /tmp/nginx.deb
 rm /tmp/nginx.deb
 
 # on some providers like scaleway the sudo file is changed and we want to keep the old one
-apt-get -o Dpkg::Options::="--force-confold" install -y sudo
+apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends sudo
 
 # this ensures that unattended upgades are enabled, if it was disabled during ubuntu install time (see #346)
 # debconf-set-selection of unattended-upgrades/enable_auto_updates + dpkg-reconfigure does not work
 cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
 echo "==> Installing node.js"
-mkdir -p /usr/local/node-10.18.1
-curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-10.18.1
-ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
-ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
-apt-get install -y python   # Install python which is required for npm rebuild
+readonly node_version=14.15.4
+mkdir -p /usr/local/node-${node_version}
+curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-${node_version}
+ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
+ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
+apt-get install -y --no-install-recommends python   # Install python which is required for npm rebuild
 [[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
 # https://docs.docker.com/engine/installation/linux/ubuntulinux/
@@ -85,9 +88,10 @@ mkdir -p /etc/systemd/system/docker.service.d
 echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
 
 # there are 3 packages for docker - containerd, CLI and the daemon
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.13-2_amd64.deb" -o /tmp/containerd.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
+readonly docker_version=20.10.3
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 # apt install with install deps (as opposed to dpkg -i)
 apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
 rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
@@ -100,7 +104,7 @@ fi
 
 # do not upgrade grub because it might prompt user and break this script
 echo "==> Enable memory accounting"
-apt-get -y --no-upgrade install grub2-common
+apt-get -y --no-upgrade --no-install-recommends install grub2-common
 sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
 update-grub
 
@@ -119,7 +123,9 @@ for image in ${images}; do
 done
 
 echo "==> Install collectd"
-if ! apt-get install -y libcurl3-gnutls collectd collectd-utils; then
+# without this, libnotify4 will install gnome-shell
+apt-get install -y libnotify4 --no-install-recommends
+if ! apt-get install -y --no-install-recommends libcurl3-gnutls collectd collectd-utils; then
     # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
     echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
     sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
@@ -127,8 +133,13 @@ fi
 # https://bugs.launchpad.net/ubuntu/+source/collectd/+bug/1872281
 [[ "${ubuntu_version}" == "20.04" ]] && echo -e "\nLD_PRELOAD=/usr/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.so" >> /etc/default/collectd
 
+# some hosts like atlantic install ntp which conflicts with timedatectl. https://serverfault.com/questions/1024770/ubuntu-20-04-time-sync-problems-and-possibly-incorrect-status-information
 echo "==> Configuring host"
 sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
+if systemctl is-active ntp; then
+    systemctl stop ntp
+    apt purge -y ntp
+fi
 timedatectl set-ntp 1
 # mysql follows the system timezone
 timedatectl set-timezone UTC

migrations/20201222182945-groupMembers-unique-members.js

@@ -0,0 +1,18 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'CREATE TABLE groupMembers_copy(groupId VARCHAR(128) NOT NULL, userId VARCHAR(128) NOT NULL, FOREIGN KEY(groupId) REFERENCES userGroups(id), FOREIGN KEY(userId) REFERENCES users(id), UNIQUE (groupId, userId)) CHARACTER SET utf8 COLLATE utf8_bin'),    // In mysql CREATE TABLE.. LIKE does not copy indexes
+        db.runSql.bind(db, 'INSERT INTO groupMembers_copy SELECT * FROM groupMembers GROUP BY groupId, userId'),
+        db.runSql.bind(db, 'DROP TABLE groupMembers'),
+        db.runSql.bind(db, 'ALTER TABLE groupMembers_copy RENAME TO groupMembers')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE groupMembers DROP INDEX groupMembers_member'),
+    ], callback);
+};

migrations/20201223014453-domains-add-wellKnownJson.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE domains ADD COLUMN wellKnownJson TEXT', function (error) {
+        if (error) return callback(error);
+
+        // keep the paths around, so that we don't need to trigger a re-configure. the old nginx config will use the paths
+        // the new one will proxy calls to the box code
+        const WELLKNOWN_DIR = '/home/yellowtent/boxdata/well-known';
+        const output = safe.child_process.execSync('find . -type f -printf "%P\n"', { cwd: WELLKNOWN_DIR, encoding: 'utf8' });
+        if (!output) return callback();
+        const paths = output.trim().split('\n');
+        if (paths.length === 0) return callback(); // user didn't configure any well-known
+
+        let wellKnown = {};
+        for (let path of paths) {
+            const fqdn = path.split('/', 1)[0];
+            const loc = path.slice(fqdn.length+1);
+            const doc = safe.fs.readFileSync(`${WELLKNOWN_DIR}/${path}`, { encoding: 'utf8' });
+            if (!doc) continue;
+
+            wellKnown[fqdn] = {};
+            wellKnown[fqdn][loc] = doc;
+        }
+
+        console.log('Migrating well-known', JSON.stringify(wellKnown, null, 4));
+
+        async.eachSeries(Object.keys(wellKnown), function (fqdn, iteratorDone) {
+            db.runSql('UPDATE domains SET wellKnownJson=? WHERE domain=?', [ JSON.stringify(wellKnown[fqdn]), fqdn ], function (error, result) {
+                if (error) {
+                    console.error(error); // maybe the domain does not exist anymore
+                } else if (result.affectedRows === 0) {
+                    console.log(`Could not migrate wellknown as domain ${fqdn} is missing`);
+                }
+                iteratorDone();
+            });
+        }, function (error) {
+            callback(error);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE domains DROP COLUMN wellKnownJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210120025243-settings-platform-config-memoryLimit.js

@@ -0,0 +1,23 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM settings WHERE name=?', ['platform_config'], function (error, results) {
+        if (error || results.length === 0) return callback(null);
+
+        let value = JSON.parse(results[0].value);
+
+        for (const serviceName of Object.keys(value)) {
+            const service = value[serviceName];
+            if (!service.memorySwap) continue;
+            service.memoryLimit = service.memorySwap;
+            delete service.memorySwap;
+            delete service.memory;
+        }
+
+        db.runSql('UPDATE settings SET value=? WHERE name=?', [ JSON.stringify(value), 'platform_config' ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210120025750-apps-servicesConfig-memoryLimit.js

@@ -0,0 +1,28 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM apps', function (error, apps) {
+        if (error) return callback(error);
+
+        async.eachSeries(apps, function (app, iteratorDone) {
+            if (!app.servicesConfigJson) return iteratorDone();
+
+            let servicesConfig = JSON.parse(app.servicesConfigJson);
+            for (const serviceName of Object.keys(servicesConfig)) {
+                const service = servicesConfig[serviceName];
+                if (!service.memorySwap) continue;
+                service.memoryLimit = service.memorySwap;
+                delete service.memorySwap;
+                delete service.memory;
+            }
+
+            db.runSql('UPDATE apps SET servicesConfigJson=? WHERE id=?', [ JSON.stringify(servicesConfig), app.id ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210121201822-settings-rename-platform-config-to-services-config.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'services_config', 'platform_config' ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('UPDATE settings SET name=? WHERE name=?', [ 'platform_config', 'services_config' ], callback);
+};

migrations/20210204181904-settings-change-ovh-url.js

@@ -0,0 +1,10 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    /* this contained an invalid migration of OVH URLs from s3 subdomain to storage subdomain. See https://forum.cloudron.io/topic/4584/issue-with-backups-listings-and-saving-backup-config-in-6-2  */
+    callback();
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210209223121-settings-add-registry-config-provider.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT value FROM settings WHERE name="registry_config"', function (error, results) {
+        if (error || results.length === 0) return callback(error);
+
+        var registryConfig = JSON.parse(results[0].value);
+        if (!registryConfig.provider) registryConfig.provider = 'other';
+
+        db.runSql('UPDATE settings SET value=? WHERE name="registry_config"', [ JSON.stringify(registryConfig) ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/schema.sql

@@ -44,7 +44,8 @@ CREATE TABLE IF NOT EXISTS groupMembers(
     groupId VARCHAR(128) NOT NULL,
     userId VARCHAR(128) NOT NULL,
     FOREIGN KEY(groupId) REFERENCES userGroups(id),
-    FOREIGN KEY(userId) REFERENCES users(id));
+    FOREIGN KEY(userId) REFERENCES users(id),
+    UNIQUE (groupId, userId));
 
 CREATE TABLE IF NOT EXISTS tokens(
     id VARCHAR(128) NOT NULL UNIQUE,
@@ -147,6 +148,7 @@ CREATE TABLE IF NOT EXISTS domains(
     provider VARCHAR(16) NOT NULL,
     configJson TEXT, /* JSON containing the dns backend provider config */
     tlsConfigJson TEXT, /* JSON containing the tls provider config */
+    wellKnownJson TEXT, /* JSON containing well known docs for this domain */
 
     PRIMARY KEY (domain))
 

package.json

@@ -10,80 +10,77 @@
     "type": "git",
     "url": "https://git.cloudron.io/cloudron/box.git"
   },
-  "engines": {
-    "node": ">=4.0.0 <=4.1.1"
-  },
   "dependencies": {
-    "@google-cloud/dns": "^1.2.9",
-    "@google-cloud/storage": "^2.5.0",
+    "@google-cloud/dns": "^2.1.0",
+    "@google-cloud/storage": "^5.8.0",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^2.6.3",
-    "aws-sdk": "^2.759.0",
+    "async": "^3.2.0",
+    "aws-sdk": "^2.850.0",
     "basic-auth": "^2.0.1",
     "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.9.0",
+    "cloudron-manifestformat": "^5.10.1",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.0.0",
     "connect-timeout": "^1.9.0",
     "cookie-parser": "^1.4.5",
     "cookie-session": "^1.4.0",
     "cron": "^1.8.2",
-    "db-migrate": "^0.11.11",
-    "db-migrate-mysql": "^2.1.1",
-    "debug": "^4.2.0",
-    "dockerode": "^2.5.8",
-    "ejs": "^2.6.1",
+    "db-migrate": "^0.11.12",
+    "db-migrate-mysql": "^2.1.2",
+    "debug": "^4.3.1",
+    "dockerode": "^3.2.1",
+    "ejs": "^3.1.6",
     "ejs-cli": "^2.2.1",
     "express": "^4.17.1",
     "ipaddr.js": "^2.0.0",
-    "js-yaml": "^3.14.0",
-    "json": "^9.0.6",
+    "js-yaml": "^4.0.0",
+    "json": "^10.0.0",
     "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.2.0",
-    "lodash": "^4.17.20",
+    "ldapjs": "^2.2.4",
+    "lodash": "^4.17.21",
     "lodash.chunk": "^4.2.0",
-    "mime": "^2.4.6",
-    "moment": "^2.29.0",
-    "moment-timezone": "^0.5.31",
+    "mime": "^2.5.2",
+    "moment": "^2.29.1",
+    "moment-timezone": "^0.5.33",
     "morgan": "^1.10.0",
     "multiparty": "^4.2.2",
     "mustache-express": "^1.3.0",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.4.11",
+    "nodemailer": "^6.4.18",
     "nodemailer-smtp-transport": "^2.7.4",
     "once": "^1.4.0",
-    "pretty-bytes": "^5.4.1",
+    "pretty-bytes": "^5.6.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
     "qrcode": "^1.4.4",
-    "readdirp": "^3.4.0",
+    "readdirp": "^3.5.0",
     "request": "^2.88.2",
-    "rimraf": "^2.6.3",
+    "rimraf": "^3.0.2",
     "s3-block-read-stream": "^0.5.0",
     "safetydance": "^1.1.1",
-    "semver": "^6.1.1",
+    "semver": "^7.3.4",
     "showdown": "^1.9.1",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^5.3.1",
+    "superagent": "^6.1.0",
     "supererror": "^0.7.2",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
-    "tar-stream": "^2.1.4",
+    "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
-    "underscore": "^1.11.0",
-    "uuid": "^3.4.0",
-    "validator": "^11.0.0",
-    "ws": "^7.3.1",
+    "underscore": "^1.12.0",
+    "uuid": "^8.3.2",
+    "validator": "^13.5.2",
+    "ws": "^7.4.3",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.4.1",
     "js2xmlparser": "^4.0.1",
-    "mocha": "^6.2.3",
+    "mocha": "^8.3.0",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^10.0.6",
-    "node-sass": "^4.14.1",
+    "nock": "^13.0.7",
+    "node-sass": "^5.0.0",
     "recursive-readdir": "^2.2.2"
   },
   "scripts": {

runTests

@@ -24,6 +24,7 @@ cd ${DATA_DIR}
 mkdir -p appsdata
 mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com boxdata/sftp/ssh
 mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
+sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
 
 # translations
 mkdir -p box/dashboard/dist/translation

scripts/cloudron-setup

@@ -2,6 +2,12 @@
 
 set -eu -o pipefail
 
+function exitHandler() {
+    rm -f /etc/update-motd.d/91-cloudron-install-in-progress
+}
+
+trap exitHandler EXIT
+
 # change this to a hash when we make a upgrade release
 readonly LOG_FILE="/var/log/cloudron-setup.log"
 readonly MINIMUM_DISK_SIZE_GB="18" # this is the size of "/" and required to fit in docker images 18 is a safe bet for different reporting on 20GB min
@@ -43,12 +49,14 @@ fi
 initBaseImage="true"
 provider="generic"
 requestedVersion=""
+installServerOrigin="https://api.cloudron.io"
 apiServerOrigin="https://api.cloudron.io"
 webServerOrigin="https://cloudron.io"
 sourceTarballUrl=""
 rebootServer="true"
+setupToken=""
 
-args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot,generate-setup-token" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
@@ -60,13 +68,18 @@ while true; do
         if [[ "$2" == "dev" ]]; then
             apiServerOrigin="https://api.dev.cloudron.io"
             webServerOrigin="https://dev.cloudron.io"
+            installServerOrigin="https://api.dev.cloudron.io"
         elif [[ "$2" == "staging" ]]; then
             apiServerOrigin="https://api.staging.cloudron.io"
             webServerOrigin="https://staging.cloudron.io"
+            installServerOrigin="https://api.staging.cloudron.io"
+        elif [[ "$2" == "unstable" ]]; then
+            installServerOrigin="https://api.dev.cloudron.io"
         fi
         shift 2;;
     --skip-baseimage-init) initBaseImage="false"; shift;;
     --skip-reboot) rebootServer="false"; shift;;
+    --generate-setup-token) setupToken="$(openssl rand -hex 10)"; shift;;
     --) break;;
     *) echo "Unknown option $1"; exit 1;;
     esac
@@ -85,6 +98,26 @@ if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubu
     exit 1
 fi
 
+# Install MOTD file for stack script style installations. this is removed by the trap exit handler. Heredoc quotes prevents parameter expansion
+cat > /etc/update-motd.d/91-cloudron-install-in-progress <<'EOF'
+#!/bin/bash
+
+printf "**********************************************************************\n\n"
+
+printf "\t\t\tWELCOME TO CLOUDRON\n"
+printf "\t\t\t-------------------\n"
+
+printf '\n\e[1;32m%-6s\e[m\n\n' "Cloudron is installing. Run 'tail -f /var/log/cloudron-setup.log' to view progress."
+
+printf "Cloudron overview - https://docs.cloudron.io/ \n"
+printf "Cloudron setup - https://docs.cloudron.io/installation/#setup \n"
+
+printf "\nFor help and more information, visit https://forum.cloudron.io\n\n"
+
+printf "**********************************************************************\n"
+EOF
+chmod +x /etc/update-motd.d/91-cloudron-install-in-progress
+
 # Can only write after we have confirmed script has root access
 echo "Running cloudron-setup with args : $@" > "${LOG_FILE}"
 
@@ -100,26 +133,20 @@ echo " Join us at https://forum.cloudron.io for any questions."
 echo ""
 
 if [[ "${initBaseImage}" == "true" ]]; then
-    echo "=> Installing software-properties-common"
-    if ! apt-get install -y software-properties-common &>> "${LOG_FILE}"; then
-        echo "Could not install software-properties-common (for add-apt-repository below). See ${LOG_FILE}"
-        exit 1
-    fi
-
     echo "=> Updating apt and installing script dependencies"
     if ! apt-get update &>> "${LOG_FILE}"; then
         echo "Could not update package repositories. See ${LOG_FILE}"
         exit 1
     fi
 
-    if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install curl python3 ubuntu-standard -y &>> "${LOG_FILE}"; then
+    if ! DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y install --no-install-recommends curl python3 ubuntu-standard software-properties-common -y &>> "${LOG_FILE}"; then
         echo "Could not install setup dependencies (curl). See ${LOG_FILE}"
         exit 1
     fi
 fi
 
 echo "=> Checking version"
-if ! releaseJson=$($curl -s "${apiServerOrigin}/api/v1/releases?boxVersion=${requestedVersion}"); then
+if ! releaseJson=$($curl -s "${installServerOrigin}/api/v1/releases?boxVersion=${requestedVersion}"); then
     echo "Failed to get release information"
     exit 1
 fi
@@ -157,6 +184,7 @@ fi
 echo "=> Installing version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
 echo "${provider}" > /etc/cloudron/PROVIDER
+[[ ! -z "${setupToken}" ]] && echo "${setupToken}" > /etc/cloudron/SETUP_TOKEN
 
 if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
     echo "Failed to install cloudron. See ${LOG_FILE} for details"
@@ -178,7 +206,12 @@ done
 if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
     ip='<IP>'
 fi
-echo -e "\n\n${GREEN}After reboot, visit https://${ip} and accept the self-signed certificate to finish setup.${DONE}\n"
+if [[ -z "${setupToken}" ]]; then
+    url="https://${ip}"
+else
+    url="https://${ip}/?setupToken=${setupToken}"
+fi
+echo -e "\n\n${GREEN}After reboot, visit ${url} and accept the self-signed certificate to finish setup.${DONE}\n"
 
 if [[ "${rebootServer}" == "true" ]]; then
     systemctl stop box mysql # sometimes mysql ends up having corrupt privilege tables
@@ -186,7 +219,7 @@ if [[ "${rebootServer}" == "true" ]]; then
     read -p "The server has to be rebooted to apply all the settings. Reboot now ? [Y/n] " yn
     yn=${yn:-y}
     case $yn in
-        [Yy]* ) systemctl reboot;;
+        [Yy]* ) exitHandler; systemctl reboot;;
         * ) exit;;
     esac
 fi

scripts/cloudron-support

@@ -73,6 +73,9 @@ echo -n "Generating Cloudron Support stats..."
 # clear file
 rm -rf $OUT
 
+echo -e $LINE"DASHBOARD DOMAIN"$LINE >> $OUT
+mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" &>> $OUT 2>/dev/null || true
+
 echo -e $LINE"PROVIDER"$LINE >> $OUT
 cat /etc/cloudron/PROVIDER &>> $OUT || true
 
@@ -99,7 +102,7 @@ systemctl status --lines=100 box mysql unbound cloudron-syslog nginx collectd do
 echo -e $LINE"Box logs"$LINE >> $OUT
 tail -n 100 /home/yellowtent/platformdata/logs/box.log &>> $OUT
 
-echo -e $LINE"Firewall chains"$LINE >> $OUT
+echo -e $LINE"Interface Info"$LINE >> $OUT
 ip addr &>> $OUT
 
 echo -e $LINE"Firewall chains"$LINE >> $OUT

scripts/createReleaseTarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v10.18.1" ]]; then
-    echo "This script requires node 10.18.1"
+if [[ "$(node --version)" != "v14.15.4" ]]; then
+    echo "This script requires node 14.15.4"
     exit 1
 fi
 

scripts/installer.sh

@@ -11,6 +11,10 @@ if [[ ${EUID} -ne 0 ]]; then
     exit 1
 fi
 
+function log() {
+  echo -e "$(date +'%Y-%m-%dT%H:%M:%S')" "==> installer: $1"
+}
+
 readonly user=yellowtent
 readonly box_src_dir=/home/${user}/box
 
@@ -21,36 +25,37 @@ readonly box_src_tmp_dir="$(realpath ${script_dir}/..)"
 readonly ubuntu_version=$(lsb_release -rs)
 readonly ubuntu_codename=$(lsb_release -cs)
 
-readonly is_update=$(systemctl is-active box && echo "yes" || echo "no")
+readonly is_update=$(systemctl is-active -q box && echo "yes" || echo "no")
 
-echo "==> installer: Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION) <=="
+log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION)"
 
-echo "==> installer: updating docker"
+log "updating docker"
 
-if [[ $(docker version --format {{.Client.Version}}) != "19.03.12" ]]; then
+readonly docker_version=20.10.3
+if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
     # there are 3 packages for docker - containerd, CLI and the daemon
-    curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.13-2_amd64.deb" -o /tmp/containerd.deb
-    curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
-    curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.3-1_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
-    echo "==> installer: Waiting for all dpkg tasks to finish..."
+    log "Waiting for all dpkg tasks to finish..."
     while fuser /var/lib/dpkg/lock; do
         sleep 1
     done
 
     while ! dpkg --force-confold --configure -a; do
-        echo "==> installer: Failed to fix packages. Retry"
+        log "Failed to fix packages. Retry"
         sleep 1
     done
 
     # the latest docker might need newer packages
     while ! apt update -y; do
-        echo "==> installer: Failed to update packages. Retry"
+        log "Failed to update packages. Retry"
         sleep 1
     done
 
     while ! apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb; do
-        echo "==> installer: Failed to install docker. Retry"
+        log "Failed to install docker. Retry"
         sleep 1
     done
 
@@ -59,31 +64,21 @@ fi
 
 readonly nginx_version=$(nginx -v 2>&1)
 if [[ "${nginx_version}" != *"1.18."* ]]; then
-    echo "==> installer: installing nginx 1.18"
-    curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
+    log "installing nginx 1.18"
+    $curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
     # apt install with install deps (as opposed to dpkg -i)
     apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
     rm /tmp/nginx.deb
 fi
 
-if ! which ipset; then
-    echo "==> installer: installing ipset"
-    apt install -y ipset
-fi
-
-# Only used for the cloudron-translation-update script
-if ! which unzip; then
-    echo "==> installer: installing unzip"
-    apt install -y unzip
-fi
-
-echo "==> installer: updating node"
-if [[ "$(node --version)" != "v10.18.1" ]]; then
-    mkdir -p /usr/local/node-10.18.1
-    $curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
-    ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
-    ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-10.15.1
+log "updating node"
+readonly node_version=14.15.4
+if [[ "$(node --version)" != "v${node_version}" ]]; then
+    mkdir -p /usr/local/node-${node_version}
+    $curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-${node_version}
+    ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
+    ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
+    rm -rf /usr/local/node-10.18.1
 fi
 
 # this is here (and not in updater.js) because rebuild requires the above node
@@ -94,31 +89,31 @@ for try in `seq 1 10`; do
     # however by default npm drops privileges for npm rebuild
     # https://docs.npmjs.com/misc/config#unsafe-perm
     if cd "${box_src_tmp_dir}" && npm rebuild --unsafe-perm; then break; fi
-    echo "==> installer: Failed to rebuild, trying again"
+    log "Failed to rebuild, trying again"
     sleep 5
 done
 
 if [[ ${try} -eq 10 ]]; then
-    echo "==> installer: npm rebuild failed, giving up"
+    log "npm rebuild failed, giving up"
     exit 4
 fi
 
-echo "==> installer: downloading new addon images"
+log "downloading new addon images"
 images=$(node -e "var i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
-echo -e "\tPulling docker images: ${images}"
+log "\tPulling docker images: ${images}"
 for image in ${images}; do
-    if ! docker pull "${image}"; then           # this pulls the image using the sha256
-        echo "==> installer: Could not pull ${image}"
-        exit 5
-    fi
-    if ! docker pull "${image%@sha256:*}"; then  # this will tag the image for readability
-        echo "==> installer: Could not pull ${image%@sha256:*}"
-        exit 6
-   fi
+    while ! docker pull "${image}"; do           # this pulls the image using the sha256
+        log "Could not pull ${image}"
+        sleep 5
+    done
+    while ! docker pull "${image%@sha256:*}"; do  # this will tag the image for readability
+        log "Could not pull ${image%@sha256:*}"
+        sleep 5
+   done
 done
 
-echo "==> installer: update cloudron-syslog"
+log "update cloudron-syslog"
 CLOUDRON_SYSLOG_DIR=/usr/local/cloudron-syslog
 CLOUDRON_SYSLOG="${CLOUDRON_SYSLOG_DIR}/bin/cloudron-syslog"
 CLOUDRON_SYSLOG_VERSION="1.0.3"
@@ -126,7 +121,7 @@ while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLO
     rm -rf "${CLOUDRON_SYSLOG_DIR}"
     mkdir -p "${CLOUDRON_SYSLOG_DIR}"
     if npm install --unsafe-perm -g --prefix "${CLOUDRON_SYSLOG_DIR}" cloudron-syslog@${CLOUDRON_SYSLOG_VERSION}; then break; fi
-    echo "===> installer: Failed to install cloudron-syslog, trying again"
+    log "Failed to install cloudron-syslog, trying again"
     sleep 5
 done
 
@@ -135,17 +130,17 @@ if ! id "${user}" 2>/dev/null; then
 fi
 
 if [[ "${is_update}" == "yes" ]]; then
-    echo "==> installer: stop box service for update"
+    log "stop box service for update"
     ${box_src_dir}/setup/stop.sh
 fi
 
 # ensure we are not inside the source directory, which we will remove now
 cd /root
 
-echo "==> installer: switching the box code"
+log "switching the box code"
 rm -rf "${box_src_dir}"
 mv "${box_src_tmp_dir}" "${box_src_dir}"
 chown -R "${user}:${user}" "${box_src_dir}"
 
-echo "==> installer: calling box setup script"
+log "calling box setup script"
 "${box_src_dir}/setup/start.sh"

setup/start.sh

@@ -5,7 +5,11 @@ set -eu -o pipefail
 # This script is run after the box code is switched. This means that this script
 # should pretty much always succeed. No network logic/download code here.
 
-echo "==> Cloudron Start"
+function log() {
+  echo -e "$(date +'%Y-%m-%dT%H:%M:%S')" "==> start: $1"
+}
+
+log "Cloudron Start"
 
 readonly USER="yellowtent"
 readonly HOME_DIR="/home/${USER}"
@@ -26,7 +30,7 @@ if ! getent group media; then
     addgroup --gid 500 --system media
 fi
 
-echo "==> Configuring docker"
+log "Configuring docker"
 cp "${script_dir}/start/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
 systemctl enable apparmor
 systemctl restart apparmor
@@ -39,7 +43,7 @@ mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
 
 # keep these in sync with paths.js
-echo "==> Ensuring directories"
+log "Ensuring directories"
 
 mkdir -p "${PLATFORM_DATA_DIR}/graphite"
 mkdir -p "${PLATFORM_DATA_DIR}/mysql"
@@ -71,7 +75,7 @@ mkdir -p "${BOX_DATA_DIR}/sftp/ssh" # sftp keys
 mkdir -p /var/backups
 chmod 777 /var/backups
 
-echo "==> Configuring journald"
+log "Configuring journald"
 sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
     -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
     -i /etc/systemd/journald.conf
@@ -92,7 +96,7 @@ setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
 # Give user access to nginx logs (uses adm group)
 usermod -a -G adm ${USER}
 
-echo "==> Setting up unbound"
+log "Setting up unbound"
 # DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org)
 # We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
 # We listen on 0.0.0.0 because there is no way control ordering of docker (which creates the 172.18.0.0/16) and unbound
@@ -102,7 +106,7 @@ cp -f "${script_dir}/start/unbound.conf" /etc/unbound/unbound.conf.d/cloudron-ne
 # update the root anchor after a out-of-disk-space situation (see #269)
 unbound-anchor -a /var/lib/unbound/root.key
 
-echo "==> Adding systemd services"
+log "Adding systemd services"
 cp -r "${script_dir}/start/systemd/." /etc/systemd/system/
 [[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/MemoryMax/MemoryLimit/g' -i /etc/systemd/system/box.service
 systemctl daemon-reload
@@ -124,11 +128,11 @@ systemctl restart unbound
 # ensure cloudron-syslog runs
 systemctl restart cloudron-syslog
 
-echo "==> Configuring sudoers"
+log "Configuring sudoers"
 rm -f /etc/sudoers.d/${USER}
 cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER}
 
-echo "==> Configuring collectd"
+log "Configuring collectd"
 rm -rf /etc/collectd /var/log/collectd.log
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
@@ -140,7 +144,7 @@ if [[ "${ubuntu_version}" == "20.04" ]]; then
 fi
 systemctl restart collectd
 
-echo "==> Configuring logrotate"
+log "Configuring logrotate"
 if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then
     echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf
 fi
@@ -150,10 +154,10 @@ cp "${script_dir}/start/logrotate/"* "${PLATFORM_DATA_DIR}/logrotate.d/"
 # logrotate files have to be owned by root, this is here to fixup existing installations where we were resetting the owner to yellowtent
 chown root:root "${PLATFORM_DATA_DIR}/logrotate.d/"
 
-echo "==> Adding motd message for admins"
+log "Adding motd message for admins"
 cp "${script_dir}/start/cloudron-motd" /etc/update-motd.d/92-cloudron
 
-echo "==> Configuring nginx"
+log "Configuring nginx"
 # link nginx config to system config
 unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
 ln -s "${PLATFORM_DATA_DIR}/nginx" /etc/nginx
@@ -181,18 +185,26 @@ if [[ ! -f /etc/mysql/mysql.cnf ]] || ! diff -q "${script_dir}/start/mysql.cnf"
     cp "${script_dir}/start/mysql.cnf" /etc/mysql/mysql.cnf
     while true; do
         if ! systemctl list-jobs | grep mysql; then break; fi
-        echo "Waiting for mysql jobs..."
+        log "Waiting for mysql jobs..."
         sleep 1
     done
-    while true; do
-        if systemctl restart mysql; then break; fi
-        echo "Restarting MySql again after sometime since this fails randomly"
+    log "Stopping mysql"
+    systemctl stop mysql
+    while mysqladmin ping 2>/dev/null; do
+        log "Waiting for mysql to stop..."
         sleep 1
     done
-else
-    systemctl start mysql
 fi
 
+# the start/stop of mysql is separate to make sure it got reloaded with latest config and it's up and running before we start the new box code
+# when using 'system restart mysql', it seems to restart much later and the box code loses connection during platform startup (dangerous!)
+log "Starting mysql"
+systemctl start mysql
+while ! mysqladmin ping 2>/dev/null; do
+    log "Waiting for mysql to start..."
+    sleep 1
+done
+
 readonly mysql_root_password="password"
 mysqladmin -u root -ppassword password password # reset default root password
 if [[ "${ubuntu_version}" == "20.04" ]]; then
@@ -203,17 +215,17 @@ mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'
 
 # set HOME explicity, because it's not set when the installer calls it. this is done because
 # paths.js uses this env var and some of the migrate code requires box code
-echo "==> Migrating data"
+log "Migrating data"
 cd "${BOX_SRC_DIR}"
 if ! HOME=${HOME_DIR} BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@127.0.0.1/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up; then
-    echo "DB migration failed"
+    log "DB migration failed"
     exit 1
 fi
 
 rm -f /etc/cloudron/cloudron.conf
 
 if [[ ! -f "${BOX_DATA_DIR}/dhparams.pem" ]]; then
-    echo "==> Generating dhparams (takes forever)"
+    log "Generating dhparams (takes forever)"
     openssl dhparam -out "${BOX_DATA_DIR}/dhparams.pem" 2048
     cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem"
 else
@@ -230,11 +242,7 @@ if [[ ! -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" ]]; then
     fi
 fi
 
-# old installations used to create appdata/<app>/redis which is now part of old backups and prevents restore
-echo "==> Cleaning up stale redis directories"
-find "${APPS_DATA_DIR}" -maxdepth 2 -type d -name redis -exec rm -rf {} +
-
-echo "==> Changing ownership"
+log "Changing ownership"
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
 chown -R "${USER}" /etc/cloudron
 chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update"
@@ -248,9 +256,9 @@ find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${BOX_DATA_DIR}/mail"
 chown "${USER}:${USER}" "${BOX_DATA_DIR}/mail"
 chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys
 
-echo "==> Starting Cloudron"
+log "Starting Cloudron"
 systemctl start box
 
 sleep 2 # give systemd sometime to start the processes
 
-echo "==> Almost done"
+log "Almost done"

setup/start/cloudron-firewall.sh

@@ -20,14 +20,20 @@ fi
 iptables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
 iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,25,80,202,443 -j ACCEPT # 202 is the alternate ssh port
 
-# whitelist any user ports
+# whitelist any user ports. we used to use --dports but it has a 15 port limit (XT_MULTI_PORTS)
 ports_json="/home/yellowtent/boxdata/firewall/ports.json"
 if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(','))" 2>/dev/null); then
-    [[ -n "${allowed_tcp_ports}" ]] && iptables -A CLOUDRON -p tcp -m tcp -m multiport --dports "${allowed_tcp_ports}" -j ACCEPT
+    IFS=',' arr=(${allowed_tcp_ports})
+    for p in "${arr[@]}"; do
+        iptables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
+    done
 fi
 
 if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(','))" 2>/dev/null); then
-    [[ -n "${allowed_tcp_ports}" ]] && iptables -A CLOUDRON -p udp -m udp -m multiport --dports "${allowed_tcp_ports}" -j ACCEPT
+    IFS=',' arr=(${allowed_udp_ports})
+    for p in "${arr[@]}"; do
+        iptables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
+    done
 fi
 
 # turn and stun service
@@ -92,3 +98,5 @@ fi
 # Workaround issue where Docker insists on adding itself first in FORWARD table
 iptables -D FORWARD -j CLOUDRON_RATELIMIT || true
 iptables -I FORWARD 1 -j CLOUDRON_RATELIMIT
+
+echo "==> Setting up firewall done"

setup/start/cloudron-motd

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+[[ -f /etc/update-motd.d/91-cloudron-install-in-progress ]] && exit
+
 printf "**********************************************************************\n\n"
 
 if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
@@ -10,10 +12,17 @@ if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
     fi
     echo "${ip}" > /tmp/.cloudron-motd-cache
 
+    if [[ ! -f /etc/cloudron/SETUP_TOKEN ]]; then
+        url="https://${ip}"
+    else
+        setupToken="$(cat /etc/cloudron/SETUP_TOKEN)"
+        url="https://${ip}/?setupToken=${setupToken}"
+    fi
+
     printf "\t\t\tWELCOME TO CLOUDRON\n"
     printf "\t\t\t-------------------\n"
 
-    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit https://${ip} on your browser and accept the self-signed certificate to finish setup."
+    printf '\n\e[1;32m%-6s\e[m\n\n' "Visit ${url} on your browser and accept the self-signed certificate to finish setup."
     printf "Cloudron overview - https://docs.cloudron.io/ \n"
     printf "Cloudron setup - https://docs.cloudron.io/installation/#setup \n"
 else

setup/start/mysql.cnf

@@ -15,6 +15,9 @@ collation-server = utf8mb4_unicode_ci
 # set timezone to UTC
 default_time_zone='+00:00'
 
+# disable bin logs. they are only useful in replication mode
+skip-log-bin
+
 [mysqldump]
 quick
 quote-names

setup/start/sudoers

@@ -13,9 +13,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/mkdirvolume.sh
 Defaults!/home/yellowtent/box/src/scripts/rmaddondir.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmaddondir.sh
 
-Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadnginx.sh
-
 Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 
@@ -25,9 +22,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/configurecollec
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 
-Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh
-
 Defaults!/home/yellowtent/box/src/scripts/update.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/update.sh
 
@@ -44,11 +38,8 @@ yellowtent ALL=(root) NOPASSWD:SETENV: /home/yellowtent/box/src/scripts/backupup
 Defaults!/home/yellowtent/box/src/scripts/restart.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restart.sh
 
-Defaults!/home/yellowtent/box/src/scripts/restartdocker.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartdocker.sh
-
-Defaults!/home/yellowtent/box/src/scripts/restartunbound.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartunbound.sh
+Defaults!/home/yellowtent/box/src/scripts/restartservice.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartservice.sh
 
 Defaults!/home/yellowtent/box/src/scripts/rmmailbox.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmailbox.sh

setup/start/systemd/box.service

@@ -1,8 +1,6 @@
 [Unit]
 Description=Cloudron Admin
 OnFailure=crashnotifier@%n.service
-; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
-BindsTo=systemd-journald.service
 After=mysql.service nginx.service
 ; As cloudron-resize-fs is a one-shot, the Wants= automatically ensures that the service *finishes*
 Wants=cloudron-resize-fs.service

src/appdb.js

@@ -26,6 +26,7 @@ exports = module.exports = {
     // subdomain table types
     SUBDOMAIN_TYPE_PRIMARY: 'primary',
     SUBDOMAIN_TYPE_REDIRECT: 'redirect',
+    SUBDOMAIN_TYPE_ALIAS: 'alias',
 
     _clear: clear
 };
@@ -37,16 +38,14 @@ var assert = require('assert'),
     safe = require('safetydance'),
     util = require('util');
 
-var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
+const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
     'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp',
     'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
     'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(',');
 
-var PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
-
-const SUBDOMAIN_FIELDS = [ 'appId', 'domain', 'subdomain', 'type' ].join(',');
+const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
 
 function postProcess(result) {
     assert.strictEqual(typeof result, 'object');
@@ -97,11 +96,23 @@ function postProcess(result) {
     result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
     delete result.servicesConfigJson;
 
-    result.alternateDomains = result.alternateDomains || [];
-    result.alternateDomains.forEach(function (d) {
-        delete d.appId;
-        delete d.type;
-    });
+    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
+    delete result.subdomains;
+    delete result.domains;
+    delete result.subdomainTypes;
+
+    result.alternateDomains = [];
+    result.aliasDomains = [];
+    for (let i = 0; i < subdomainTypes.length; i++) {
+        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
+            result.location = subdomains[i];
+            result.domain = domains[i];
+        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
+            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
+        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
+            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
+        }
+    }
 
     let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
     delete result.envNames;
@@ -127,9 +138,9 @@ function postProcess(result) {
 // each query simply join apps table with another table by id. we then join the full result together
 const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
 const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
-const SUBDOMAIN_QUERY = `SELECT id, subdomains.subdomain AS location, subdomains.domain AS domain FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = '${exports.SUBDOMAIN_TYPE_PRIMARY}' GROUP BY apps.id`;
+const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
 const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
-const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, location, domain, volumeIds, volumeReadOnlys FROM apps`
+const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
     + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
     + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
     + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
@@ -143,15 +154,9 @@ function get(id, callback) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
         if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
 
-        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE appId = ? AND type = ?', [ id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        postProcess(result[0]);
 
-            result[0].alternateDomains = alternateDomains;
-
-            postProcess(result[0]);
-
-            callback(null, result[0]);
-        });
+        callback(null, result[0]);
     });
 }
 
@@ -163,15 +168,9 @@ function getByIpAddress(ip, callback) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
         if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
 
-        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE appId = ? AND type = ?', [ result[0].id, exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        postProcess(result[0]);
 
-            result[0].alternateDomains = alternateDomains;
-
-            postProcess(result[0]);
-
-            callback(null, result[0]);
-        });
+        callback(null, result[0]);
     });
 }
 
@@ -181,21 +180,9 @@ function getAll(callback) {
     database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ], function (error, results) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
-        database.query('SELECT ' + SUBDOMAIN_FIELDS + ' FROM subdomains WHERE type = ?', [ exports.SUBDOMAIN_TYPE_REDIRECT ], function (error, alternateDomains) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        results.forEach(postProcess);
 
-            alternateDomains.forEach(function (d) {
-                var domain = results.find(function (a) { return d.appId === a.id; });
-                if (!domain) return;
-
-                domain.alternateDomains = domain.alternateDomains || [];
-                domain.alternateDomains.push(d);
-            });
-
-            results.forEach(postProcess);
-
-            callback(null, results);
-        });
+        callback(null, results);
     });
 }
 
@@ -267,6 +254,15 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
         });
     }
 
+    if (data.aliasDomains) {
+        data.aliasDomains.forEach(function (d) {
+            queries.push({
+                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
+                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
+            });
+        });
+    }
+
     database.transaction(queries, function (error) {
         if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
         if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'no such domain'));
@@ -364,6 +360,7 @@ function updateWithConstraints(id, app, constraints, callback) {
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
     assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
     assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
+    assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
     assert(!('tags' in app) || Array.isArray(app.tags));
     assert(!('env' in app) || typeof app.env === 'object');
 
@@ -399,6 +396,12 @@ function updateWithConstraints(id, app, constraints, callback) {
                 queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
             });
         }
+
+        if ('aliasDomains' in app) {
+            app.aliasDomains.forEach(function (d) {
+                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
+            });
+        }
     }
 
     if ('mounts' in app) {
@@ -413,7 +416,7 @@ function updateWithConstraints(id, app, constraints, callback) {
         if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig') {
             fields.push(`${p}Json = ?`);
             values.push(JSON.stringify(app[p]));
-        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'env' && p !== 'mounts') {
+        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
             fields.push(p + ' = ?');
             values.push(app[p]);
         }

src/apphealthmonitor.js

@@ -10,49 +10,48 @@ var appdb = require('./appdb.js'),
     docker = require('./docker.js'),
     eventlog = require('./eventlog.js'),
     safe = require('safetydance'),
-    superagent = require('superagent'),
-    util = require('util');
+    superagent = require('superagent');
 
 exports = module.exports = {
     run
 };
 
 const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
-const UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes
+const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes
 
 const OOM_EVENT_LIMIT = 60 * 60 * 1000; // 60 minutes
+let gStartTime = null; // time when apphealthmonitor was started
 let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago
 
-function debugApp(app) {
-    assert(typeof app === 'object');
-
-    debug(app.fqdn + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)) + ' - ' + app.id);
-}
-
 function setHealth(app, health, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof health, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let now = new Date(), healthTime = app.healthTime, curHealth = app.health;
+    // app starts out with null health
+    // if it became healthy, we update immediately. this is required for ui to say "running" etc
+    // if it became unhealthy/error/dead, wait for a threshold before updating db
+
+    const now = new Date(), lastHealth = app.health;
+    let healthTime = gStartTime > app.healthTime ? gStartTime : app.healthTime; // on box restart, clamp value to start time
 
     if (health === apps.HEALTH_HEALTHY) {
         healthTime = now;
-        if (curHealth && curHealth !== apps.HEALTH_HEALTHY) { // app starts out with null health
-            debugApp(app, 'app switched from %s to healthy', curHealth);
+        if (lastHealth && lastHealth !== apps.HEALTH_HEALTHY) { // app starts out with null health
+            debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
 
             // do not send mails for dev apps
             if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, auditSource.HEALTH_MONITOR, { app: app });
         }
     } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
-        if (curHealth === apps.HEALTH_HEALTHY) {
-            debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000));
+        if (lastHealth === apps.HEALTH_HEALTHY) {
+            debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
 
             // do not send mails for dev apps
             if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, auditSource.HEALTH_MONITOR, { app: app });
         }
     } else {
-        debugApp(app, 'waiting for %s seconds to update the app health', (UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000);
+        debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);
         return callback(null);
     }
 
@@ -61,6 +60,7 @@ function setHealth(app, health, callback) {
         if (error) return callback(error);
 
         app.health = health;
+        app.healthTime = healthTime;
 
         callback(null);
     });
@@ -118,7 +118,7 @@ function getContainerInfo(containerId, callback) {
 
 /*
     OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:0.10.0 /bin/bash
+        docker run -ti -m 100M cloudron/base:2.0.0 /bin/bash
         apt-get update && apt-get install stress
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
@@ -187,6 +187,8 @@ function run(intervalSecs, callback) {
     assert.strictEqual(typeof intervalSecs, 'number');
     assert.strictEqual(typeof callback, 'function');
 
+    if (!gStartTime) gStartTime = new Date();
+
     async.series([
         processApp, // this is first because docker.getEvents seems to get 'stuck' sometimes
         processDockerEvents.bind(null, intervalSecs)

src/apps.js

@@ -63,6 +63,7 @@ exports = module.exports = {
 
     getDataDir,
     getIconPath,
+    getMemoryLimit,
 
     downloadFile,
     uploadFile,
@@ -135,7 +136,6 @@ var appdb = require('./appdb.js'),
     superagent = require('superagent'),
     tasks = require('./tasks.js'),
     TransformStream = require('stream').Transform,
-    updateChecker = require('./updatechecker.js'),
     users = require('./users.js'),
     util = require('util'),
     uuid = require('uuid'),
@@ -182,6 +182,11 @@ function validatePortBindings(portBindings, manifest) {
         [50000, 51000] /* turn udp ports */
     ];
 
+    const ALLOWED_PORTS = [
+        53, // dns 53 is special and adblocker apps can use them
+        853 // dns over tls
+    ];
+
     if (!portBindings) return null;
 
     for (let portName in portBindings) {
@@ -191,7 +196,7 @@ function validatePortBindings(portBindings, manifest) {
         if (!Number.isInteger(hostPort)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not an integer`, { field: 'portBindings', portName: portName });
         if (RESERVED_PORTS.indexOf(hostPort) !== -1) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
         if (RESERVED_PORT_RANGES.find(range => (hostPort >= range[0] && hostPort <= range[1]))) return new BoxError(BoxError.BAD_FIELD, `Port ${hostPort} is reserved.`, { field: 'portBindings', portName: portName });
-        if (hostPort !== 53 && (hostPort <= 1023 || hostPort > 65535)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not in permitted range`, { field: 'portBindings', portName: portName }); // dns 53 is special and adblocker apps can use them
+        if (ALLOWED_PORTS.indexOf(hostPort) === -1 && (hostPort <= 1023 || hostPort > 65535)) return new BoxError(BoxError.BAD_FIELD, `${hostPort} is not in permitted range`, { field: 'portBindings', portName: portName });
     }
 
     // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
@@ -406,13 +411,13 @@ function removeInternalFields(app) {
         'location', 'domain', 'fqdn', 'mailboxName', 'mailboxDomain',
         'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares',
         'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags',
-        'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts');
+        'label', 'alternateDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts');
 }
 
 // non-admins can only see these
 function removeRestrictedFields(app) {
     return _.pick(app,
-        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'alternateDomains', 'sso',
+        'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'accessRestriction', 'alternateDomains', 'aliasDomains', 'sso',
         'location', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup');
 }
 
@@ -446,6 +451,20 @@ function getIconPath(app, options, callback) {
     callback(new BoxError(BoxError.NOT_FOUND, 'No icon'));
 }
 
+function getMemoryLimit(app) {
+    assert.strictEqual(typeof app, 'object');
+
+    let memoryLimit = app.memoryLimit || app.manifest.memoryLimit || 0;
+
+    if (memoryLimit === -1) { // unrestricted
+        memoryLimit = 0;
+    } else if (memoryLimit === 0 || memoryLimit < constants.DEFAULT_MEMORY_LIMIT) { // ensure we never go below minimum (in case we change the default)
+        memoryLimit = constants.DEFAULT_MEMORY_LIMIT;
+    }
+
+    return memoryLimit;
+}
+
 function postProcess(app, domainObjectMap) {
     let result = {};
     for (let portName in app.portBindings) {
@@ -456,6 +475,7 @@ function postProcess(app, domainObjectMap) {
     app.iconUrl = getIconUrlSync(app);
     app.fqdn = domains.fqdn(app.location, domainObjectMap[app.domain]);
     app.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+    app.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 }
 
 function hasAccessTo(app, user, callback) {
@@ -605,20 +625,31 @@ function scheduleTask(appId, installationState, taskId, callback) {
     assert.strictEqual(typeof taskId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    appTaskManager.scheduleTask(appId, taskId, function (error) {
-        debug(`scheduleTask: task ${taskId} of ${appId} completed`);
-        if (error && (error.code === tasks.ECRASHED || error.code === tasks.ESTOPPED)) { // if task crashed, update the error
-            debug(`Apptask crashed/stopped: ${error.message}`);
-            let boxError = new BoxError(BoxError.TASK_ERROR, error.message);
-            boxError.details.crashed = error.code === tasks.ECRASHED;
-            boxError.details.stopped = error.code === tasks.ESTOPPED;
-            // see also apptask makeTaskError
-            boxError.details.taskId = taskId;
-            boxError.details.installationState = installationState;
-            appdb.update(appId, { installationState: exports.ISTATE_ERROR, error: boxError.toPlainObject(), taskId: null }, callback);
-        } else if (!(installationState === exports.ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall
-            appdb.update(appId, { taskId: null }, callback);
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(error);
+
+        let memoryLimit = 400;
+        if (installationState === exports.ISTATE_PENDING_BACKUP || installationState === exports.ISTATE_PENDING_CLONE || installationState === exports.ISTATE_PENDING_RESTORE || installationState === exports.ISTATE_PENDING_UPDATE) {
+            memoryLimit = 'memoryLimit' in backupConfig ? Math.max(backupConfig.memoryLimit/1024/1024, 400) : 400;
         }
+
+        const options = { timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15, memoryLimit };
+
+        appTaskManager.scheduleTask(appId, taskId, options, function (error) {
+            debug(`scheduleTask: task ${taskId} of ${appId} completed`);
+            if (error && (error.code === tasks.ECRASHED || error.code === tasks.ESTOPPED)) { // if task crashed, update the error
+                debug(`Apptask crashed/stopped: ${error.message}`);
+                let boxError = new BoxError(BoxError.TASK_ERROR, error.message);
+                boxError.details.crashed = error.code === tasks.ECRASHED;
+                boxError.details.stopped = error.code === tasks.ESTOPPED;
+                // see also apptask makeTaskError
+                boxError.details.taskId = taskId;
+                boxError.details.installationState = installationState;
+                appdb.update(appId, { installationState: exports.ISTATE_ERROR, error: boxError.toPlainObject(), taskId: null }, callback);
+            } else if (!(installationState === exports.ISTATE_PENDING_UNINSTALL && !error)) { // clear out taskId except for successful uninstall
+                appdb.update(appId, { taskId: null }, callback);
+            }
+        });
     });
 }
 
@@ -680,7 +711,13 @@ function validateLocations(locations, callback) {
         for (let location of locations) {
             if (!(location.domain in domainObjectMap)) return callback(new BoxError(BoxError.BAD_FIELD, 'No such domain', { field: 'location', domain: location.domain, subdomain: location.subdomain }));
 
-            error = domains.validateHostname(location.subdomain, domainObjectMap[location.domain]);
+            let subdomain = location.subdomain;
+            if (location.type === 'alias' && subdomain.startsWith('*')) {
+                if (subdomain === '*') continue;
+                subdomain = subdomain.replace(/^\*\./, ''); // remove *.
+            }
+
+            error = domains.validateHostname(subdomain, domainObjectMap[location.domain]);
             if (error) return callback(new BoxError(BoxError.BAD_FIELD, 'Bad location: ' + error.message, { field: 'location', domain: location.domain, subdomain: location.subdomain }));
         }
 
@@ -712,10 +749,12 @@ function install(data, auditSource, callback) {
         enableBackup = 'enableBackup' in data ? data.enableBackup : true,
         enableAutomaticUpdate = 'enableAutomaticUpdate' in data ? data.enableAutomaticUpdate : true,
         alternateDomains = data.alternateDomains || [],
+        aliasDomains = data.aliasDomains || [],
         env = data.env || {},
         label = data.label || null,
         tags = data.tags || [],
         overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false,
+        skipDnsSetup = 'skipDnsSetup' in data ? data.skipDnsSetup : false,
         appStoreId = data.appStoreId,
         manifest = data.manifest;
 
@@ -764,7 +803,10 @@ function install(data, auditSource, callback) {
         }
     }
 
-    const locations = [{subdomain: location, domain}].concat(alternateDomains);
+    const locations = [{ subdomain: location, domain, type: 'primary' }]
+        .concat(alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
+        .concat(aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));
+
     validateLocations(locations, function (error, domainObjectMap) {
         if (error) return callback(error);
 
@@ -776,18 +818,19 @@ function install(data, auditSource, callback) {
         debug('Will install app with id : ' + appId);
 
         var data = {
-            accessRestriction: accessRestriction,
-            memoryLimit: memoryLimit,
-            sso: sso,
-            debugMode: debugMode,
-            mailboxName: mailboxName,
-            mailboxDomain: mailboxDomain,
-            enableBackup: enableBackup,
-            enableAutomaticUpdate: enableAutomaticUpdate,
-            alternateDomains: alternateDomains,
-            env: env,
-            label: label,
-            tags: tags,
+            accessRestriction,
+            memoryLimit,
+            sso,
+            debugMode,
+            mailboxName,
+            mailboxDomain,
+            enableBackup,
+            enableAutomaticUpdate,
+            alternateDomains,
+            aliasDomains,
+            env,
+            label,
+            tags,
             runState: exports.RSTATE_RUNNING,
             installationState: exports.ISTATE_PENDING_INSTALL
         };
@@ -806,7 +849,7 @@ function install(data, auditSource, callback) {
                 }
 
                 const task = {
-                    args: { restoreConfig: null, overwriteDns },
+                    args: { restoreConfig: null, skipDnsSetup, overwriteDns },
                     values: { },
                     requiredState: data.installationState
                 };
@@ -817,6 +860,7 @@ function install(data, auditSource, callback) {
                     const newApp = _.extend({}, data, { appStoreId, manifest, location, domain, portBindings });
                     newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
                     newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+                    newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
                     eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId, app: newApp, taskId: result.taskId });
 
@@ -1172,7 +1216,8 @@ function setLocation(app, data, auditSource, callback) {
         domain: data.domain.toLowerCase(),
         // these are intentionally reset, if not set
         portBindings: null,
-        alternateDomains: []
+        alternateDomains: [],
+        aliasDomains: []
     };
 
     if ('portBindings' in data) {
@@ -1192,14 +1237,21 @@ function setLocation(app, data, auditSource, callback) {
         values.alternateDomains = data.alternateDomains;
     }
 
-    const locations = [{subdomain: values.location, domain: values.domain}].concat(values.alternateDomains);
+    if ('aliasDomains' in data) {
+        values.aliasDomains = data.aliasDomains;
+    }
+
+    const locations = [{ subdomain: values.location, domain: values.domain, type: 'primary' }]
+        .concat(values.alternateDomains.map(ad => _.extend(ad, { type: 'redirect' })))
+        .concat(values.aliasDomains.map(ad => _.extend(ad, { type: 'alias' })));
 
     validateLocations(locations, function (error, domainObjectMap) {
         if (error) return callback(error);
 
         const task = {
             args: {
-                oldConfig: _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'portBindings'),
+                oldConfig: _.pick(app, 'location', 'domain', 'fqdn', 'alternateDomains', 'aliasDomains', 'portBindings'),
+                skipDnsSetup: !!data.skipDnsSetup,
                 overwriteDns: !!data.overwriteDns
             },
             values
@@ -1210,6 +1262,7 @@ function setLocation(app, data, auditSource, callback) {
 
             values.fqdn = domains.fqdn(values.location, domainObjectMap[values.domain]);
             values.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+            values.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
 
             eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, _.extend({ appId, app, taskId: result.taskId }, values));
 
@@ -1253,7 +1306,8 @@ function update(app, data, auditSource, callback) {
 
     const skipBackup = !!data.skipBackup,
         appId = app.id,
-        manifest = data.manifest;
+        manifest = data.manifest,
+        appStoreId = data.appStoreId;
 
     let values = {};
 
@@ -1268,14 +1322,12 @@ function update(app, data, auditSource, callback) {
     error = checkManifestConstraints(manifest);
     if (error) return callback(error);
 
-    var updateConfig = { skipBackup, manifest };
+    var updateConfig = { skipBackup, manifest, appStoreId }; // this will clear appStoreId when updating from a repo and set it if passed in for update route
 
     // prevent user from installing a app with different manifest id over an existing app
     // this allows cloudron install -f --app <appid> for an app installed from the appStore
     if (app.manifest.id !== updateConfig.manifest.id) {
         if (!data.force) return callback(new BoxError(BoxError.BAD_FIELD, 'manifest id does not match. force to override'));
-        // clear appStoreId so that this app does not get updates anymore
-        updateConfig.appStoreId = '';
     }
 
     // suffix '0' if prerelease is missing for semver.lte to work as expected
@@ -1322,9 +1374,6 @@ function update(app, data, auditSource, callback) {
 
         eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId, app, skipBackup, toManifest: manifest, fromManifest: app.manifest, force: data.force, taskId: result.taskId });
 
-        // clear update indicator, if update fails, it will come back through the update checker
-        updateChecker.resetAppUpdateInfo(appId);
-
         callback(null, { taskId: result.taskId });
     });
 }
@@ -1408,7 +1457,7 @@ function repair(app, data, auditSource, callback) {
 
     // maybe split this into a separate route like reinstall?
     if (errorState === exports.ISTATE_PENDING_INSTALL || errorState === exports.ISTATE_PENDING_CLONE) {
-        task.args = { overwriteDns: true };
+        task.args = { skipDnsSetup: false, overwriteDns: true };
         if (data.manifest) {
             let error = manifestFormat.parse(data.manifest);
             if (error) return callback(new BoxError(BoxError.BAD_FIELD, `manifest error: ${error.message}`));
@@ -1481,6 +1530,7 @@ function restore(app, backupId, auditSource, callback) {
             args: {
                 restoreConfig,
                 oldManifest: app.manifest,
+                skipDnsSetup: !!backupId, // if this is a restore, just skip dns setup. only re-installs should setup dns
                 overwriteDns: true
             },
             values
@@ -1536,6 +1586,7 @@ function importApp(app, data, auditSource, callback) {
             args: {
                 restoreConfig,
                 oldManifest: app.manifest,
+                skipDnsSetup: false,
                 overwriteDns: true
             },
             values: {}
@@ -1598,6 +1649,7 @@ function clone(app, data, user, auditSource, callback) {
         portBindings = data.portBindings || null,
         backupId = data.backupId,
         overwriteDns = 'overwriteDns' in data ? data.overwriteDns : false,
+        skipDnsSetup = 'skipDnsSetup' in data ? data.skipDnsSetup : false,
         appId = app.id;
 
     assert.strictEqual(typeof backupId, 'string');
@@ -1624,7 +1676,7 @@ function clone(app, data, user, auditSource, callback) {
         let mailboxName = hasMailAddon(manifest) ? mailboxNameForLocation(location, manifest) : null;
         let mailboxDomain = hasMailAddon(manifest) ? domain : null;
 
-        const locations = [{subdomain: location, domain}];
+        const locations = [{ subdomain: location, domain, type: 'primary' }];
         validateLocations(locations, function (error, domainObjectMap) {
             if (error) return callback(error);
 
@@ -1641,7 +1693,8 @@ function clone(app, data, user, auditSource, callback) {
                 enableBackup: app.enableBackup,
                 reverseProxyConfig: app.reverseProxyConfig,
                 env: app.env,
-                alternateDomains: []
+                alternateDomains: [],
+                aliasDomains: []
             };
 
             appdb.add(newAppId, appStoreId, manifest, location, domain, translatePortBindings(portBindings, manifest), data, function (error) {
@@ -1653,7 +1706,7 @@ function clone(app, data, user, auditSource, callback) {
 
                     const restoreConfig = { backupId: backupId, backupFormat: backupInfo.format };
                     const task = {
-                        args: { restoreConfig, overwriteDns, oldManifest: null },
+                        args: { restoreConfig, overwriteDns, skipDnsSetup, oldManifest: null },
                         values: {},
                         requiredState: exports.ISTATE_PENDING_CLONE
                     };
@@ -1663,6 +1716,8 @@ function clone(app, data, user, auditSource, callback) {
                         const newApp = _.extend({}, data, { appStoreId, manifest, location, domain, portBindings });
                         newApp.fqdn = domains.fqdn(newApp.location, domainObjectMap[newApp.domain]);
                         newApp.alternateDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+                        newApp.aliasDomains.forEach(function (ad) { ad.fqdn = domains.fqdn(ad.subdomain, domainObjectMap[ad.domain]); });
+
                         eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, oldApp: app, newApp: newApp, taskId: result.taskId });
 
                         callback(null, { id: newAppId, taskId: result.taskId });
@@ -1865,8 +1920,6 @@ function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is {
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!updateInfo) return callback(null);
-
     async.eachSeries(Object.keys(updateInfo), function iterator(appId, iteratorDone) {
         get(appId, function (error, app) {
             if (error) {
@@ -1926,7 +1979,8 @@ function listBackups(app, page, perPage, callback) {
     });
 }
 
-function restoreInstalledApps(callback) {
+function restoreInstalledApps(options, callback) {
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     getAll(function (error, apps) {
@@ -1949,7 +2003,7 @@ function restoreInstalledApps(callback) {
                 }
 
                 const task = {
-                    args: { restoreConfig, overwriteDns: true, oldManifest },
+                    args: { restoreConfig, skipDnsSetup: options.skipDnsSetup, overwriteDns: true, oldManifest },
                     values: {},
                     scheduleNow: false, // task will be scheduled by autoRestartTasks when platform is ready
                     requireNullTaskId: false // ignore existing stale taskId

src/appstore.js

@@ -1,28 +1,28 @@
 'use strict';
 
 exports = module.exports = {
-    getFeatures: getFeatures,
+    getFeatures,
 
-    getApps: getApps,
-    getApp: getApp,
-    getAppVersion: getAppVersion,
+    getApps,
+    getApp,
+    getAppVersion,
 
-    trackBeginSetup: trackBeginSetup,
-    trackFinishedSetup: trackFinishedSetup,
+    trackBeginSetup,
+    trackFinishedSetup,
 
-    registerWithLoginCredentials: registerWithLoginCredentials,
+    registerWithLoginCredentials,
 
-    purchaseApp: purchaseApp,
-    unpurchaseApp: unpurchaseApp,
+    purchaseApp,
+    unpurchaseApp,
 
-    getUserToken: getUserToken,
-    getSubscription: getSubscription,
-    isFreePlan: isFreePlan,
+    getUserToken,
+    getSubscription,
+    isFreePlan,
 
-    getAppUpdate: getAppUpdate,
-    getBoxUpdate: getBoxUpdate,
+    getAppUpdate,
+    getBoxUpdate,
 
-    createTicket: createTicket
+    createTicket
 };
 
 var apps = require('./apps.js'),
@@ -45,6 +45,8 @@ var apps = require('./apps.js'),
 // Keep in sync with appstore/routes/cloudrons.js
 let gFeatures = {
     userMaxCount: 5,
+    userGroups: false,
+    userRoles: false,
     domainMaxCount: 1,
     externalLdap: false,
     privateDockerRegistry: false,
@@ -247,13 +249,13 @@ function getBoxUpdate(options, callback) {
             if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
             if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
             if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 204) return callback(null); // no update
+            if (result.statusCode === 204) return callback(null, null); // no update
             if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
 
             var updateInfo = result.body;
 
             if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Invalid update version: %s %s', result.statusCode, result.text)));
+                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', result.statusCode, result.text)));
             }
 
             // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }

src/apptask.js

@@ -3,7 +3,7 @@
 'use strict';
 
 exports = module.exports = {
-    run: run,
+    run,
 
     // exported for testing
     _configureReverseProxy: configureReverseProxy,
@@ -11,13 +11,10 @@ exports = module.exports = {
     _createAppDir: createAppDir,
     _deleteAppDir: deleteAppDir,
     _verifyManifest: verifyManifest,
-    _registerSubdomains: registerSubdomains,
-    _unregisterSubdomains: unregisterSubdomains,
     _waitForDnsPropagation: waitForDnsPropagation
 };
 
-var addons = require('./addons.js'),
-    appdb = require('./appdb.js'),
+const appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
@@ -41,6 +38,7 @@ var addons = require('./addons.js'),
     reverseProxy = require('./reverseproxy.js'),
     rimraf = require('rimraf'),
     safe = require('safetydance'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     superagent = require('superagent'),
@@ -333,82 +331,6 @@ function removeIcon(app, callback) {
     callback(null);
 }
 
-function registerSubdomains(app, overwrite, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof overwrite, 'boolean');
-    assert.strictEqual(typeof callback, 'function');
-
-    sysinfo.getServerIp(function (error, ip) {
-        if (error) return callback(error);
-
-        const allDomains = [ { subdomain: app.location, domain: app.domain }].concat(app.alternateDomains);
-
-        debugApp(app, `registerSubdomain: Will register ${JSON.stringify(allDomains)}`);
-
-        async.eachSeries(allDomains, function (domain, iteratorDone) {
-            async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
-                debugApp(app, 'Registering subdomain: %s%s', domain.subdomain ? (domain.subdomain + '.') : '', domain.domain);
-
-                // get the current record before updating it
-                domains.getDnsRecords(domain.subdomain, domain.domain, 'A', function (error, values) {
-                    if (error && error.reason === BoxError.EXTERNAL_ERROR) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain })); // try again
-                    if (error && error.reason === BoxError.ACCESS_DENIED) return retryCallback(null, new BoxError(BoxError.ACCESS_DENIED, error.message, { domain }));
-                    if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, new BoxError(BoxError.NOT_FOUND, error.message, { domain }));
-                    if (error) return retryCallback(null, new BoxError(BoxError.EXTERNAL_ERROR, error.message, domain)); // give up for other errors
-
-                    if (values.length !== 0 && values[0] === ip) return retryCallback(null); // up-to-date
-
-                    // refuse to update any existing DNS record for custom domains that we did not create
-                    if (values.length !== 0 && !overwrite) return retryCallback(null, new BoxError(BoxError.ALREADY_EXISTS, 'DNS Record already exists', { domain }));
-
-                    domains.upsertDnsRecords(domain.subdomain, domain.domain, 'A', [ ip ], function (error) {
-                        if (error && (error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                            debugApp(app, 'registerSubdomains: Upsert error. Will retry.', error.message);
-                            return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain })); // try again
-                        }
-
-                        retryCallback(null, error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, domain) : null);
-                    });
-                });
-            }, function (error, result) {
-                if (error || result) return iteratorDone(error || result);
-
-                iteratorDone(null);
-            });
-        }, callback);
-    });
-}
-
-function unregisterSubdomains(app, allDomains, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert(Array.isArray(allDomains));
-    assert.strictEqual(typeof callback, 'function');
-
-    sysinfo.getServerIp(function (error, ip) {
-        if (error) return callback(error);
-
-        async.eachSeries(allDomains, function (domain, iteratorDone) {
-            async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
-                debugApp(app, 'Unregistering subdomain: %s%s', domain.subdomain ? (domain.subdomain + '.') : '', domain.domain);
-
-                domains.removeDnsRecords(domain.subdomain, domain.domain, 'A', [ ip ], function (error) {
-                    if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, null);
-                    if (error && (error.reason === BoxError.SBUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
-                        debugApp(app, 'registerSubdomains: Remove error. Will retry.', error.message);
-                        return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain })); // try again
-                    }
-
-                    retryCallback(null, error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain }) : null);
-                });
-            }, function (error, result) {
-                if (error || result) return iteratorDone(error || result);
-
-                iteratorDone();
-            });
-        }, callback);
-    });
-}
-
 function waitForDnsPropagation(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -424,8 +346,8 @@ function waitForDnsPropagation(app, callback) {
         domains.waitForDnsRecord(app.location, app.domain, 'A', ip, { times: 240 }, function (error) {
             if (error) return callback(new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: app.location, domain: app.domain }));
 
-            // now wait for alternateDomains, if any
-            async.eachSeries(app.alternateDomains, function (domain, iteratorCallback) {
+            // now wait for alternateDomains and aliasDomains, if any
+            async.eachSeries(app.alternateDomains.concat(app.aliasDomains), function (domain, iteratorCallback) {
                 domains.waitForDnsRecord(domain.subdomain, domain.domain, 'A', ip, { times: 240 }, function (error) {
                     if (error) return callback(new BoxError(BoxError.DNS_ERROR, `DNS Record is not synced yet: ${error.message}`, { ip: ip, subdomain: domain.subdomain, domain: domain.domain }));
 
@@ -492,6 +414,7 @@ function install(app, args, progressCallback, callback) {
 
     const restoreConfig = args.restoreConfig; // has to be set when restoring
     const overwriteDns = args.overwriteDns;
+    const skipDnsSetup = args.skipDnsSetup;
     const oldManifest = args.oldManifest;
 
     async.series([
@@ -512,7 +435,7 @@ function install(app, args, progressCallback, callback) {
                 addonsToRemove = app.manifest.addons;
             }
 
-            addons.teardownAddons(app, addonsToRemove, next);
+            services.teardownAddons(app, addonsToRemove, next);
         },
 
         function deleteAppDirIfNeeded(done) {
@@ -533,8 +456,15 @@ function install(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 20, message: 'Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
-        registerSubdomains.bind(null, app, overwriteDns),
+        function setupDnsIfNeeded(done) {
+            if (skipDnsSetup) return done();
+
+            async.series([
+                progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
+
+                domains.registerLocations.bind(null, [ { subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback)
+            ], done);
+        },
 
         progressCallback.bind(null, { percent: 40, message: 'Downloading image' }),
         downloadImage.bind(null, app.manifest),
@@ -546,24 +476,24 @@ function install(app, args, progressCallback, callback) {
             if (!restoreConfig) {
                 async.series([
                     progressCallback.bind(null, { percent: 60, message: 'Setting up addons' }),
-                    addons.setupAddons.bind(null, app, app.manifest.addons),
+                    services.setupAddons.bind(null, app, app.manifest.addons),
                 ], next);
             } else if (!restoreConfig.backupId) { // in-place import
                 async.series([
                     progressCallback.bind(null, { percent: 60, message: 'Importing addons in-place' }),
-                    addons.setupAddons.bind(null, app, app.manifest.addons),
-                    addons.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
-                    addons.restoreAddons.bind(null, app, app.manifest.addons),
+                    services.setupAddons.bind(null, app, app.manifest.addons),
+                    services.clearAddons.bind(null, app, _.omit(app.manifest.addons, 'localstorage')),
+                    services.restoreAddons.bind(null, app, app.manifest.addons),
                 ], next);
             } else {
                 async.series([
                     progressCallback.bind(null, { percent: 65, message: 'Download backup and restoring addons' }),
-                    addons.setupAddons.bind(null, app, app.manifest.addons),
-                    addons.clearAddons.bind(null, app, app.manifest.addons),
+                    services.setupAddons.bind(null, app, app.manifest.addons),
+                    services.clearAddons.bind(null, app, app.manifest.addons),
                     backups.downloadApp.bind(null, app, restoreConfig, (progress) => {
                         progressCallback({ percent: 65, message: progress.message });
                     }),
-                    addons.restoreAddons.bind(null, app, app.manifest.addons)
+                    services.restoreAddons.bind(null, app, app.manifest.addons)
                 ], next);
             }
         },
@@ -573,8 +503,14 @@ function install(app, args, progressCallback, callback) {
 
         startApp.bind(null, app),
 
-        progressCallback.bind(null, { percent: 85, message: 'Waiting for DNS propagation' }),
-        exports._waitForDnsPropagation.bind(null, app),
+        function waitForDns(done) {
+            if (skipDnsSetup) return done();
+
+            async.series([
+                progressCallback.bind(null, { percent: 85, message: 'Waiting for DNS propagation' }),
+                exports._waitForDnsPropagation.bind(null, app),
+            ], done);
+        },
 
         progressCallback.bind(null, { percent: 95, message: 'Configuring reverse proxy' }),
         configureReverseProxy.bind(null, app),
@@ -626,7 +562,7 @@ function create(app, args, progressCallback, callback) {
 
         // FIXME: re-setup addons only because sendmail addon to re-inject env vars on mailboxName change
         progressCallback.bind(null, { percent: 30, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        services.setupAddons.bind(null, app, app.manifest.addons),
 
         progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
         createContainer.bind(null, app),
@@ -652,6 +588,7 @@ function changeLocation(app, args, progressCallback, callback) {
 
     const oldConfig = args.oldConfig;
     const locationChanged = oldConfig.fqdn !== app.fqdn;
+    const skipDnsSetup = args.skipDnsSetup;
     const overwriteDns = args.overwriteDns;
 
     async.series([
@@ -663,27 +600,45 @@ function changeLocation(app, args, progressCallback, callback) {
                 return !app.alternateDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
             });
 
+            if (oldConfig.aliasDomains) {
+                obsoleteDomains = obsoleteDomains.concat(oldConfig.aliasDomains.filter(function (o) {
+                    return !app.aliasDomains.some(function (n) { return n.subdomain === o.subdomain && n.domain === o.domain; });
+                }));
+            }
+
             if (locationChanged) obsoleteDomains.push({ subdomain: oldConfig.location, domain: oldConfig.domain });
 
             if (obsoleteDomains.length === 0) return next();
 
-            unregisterSubdomains(app, obsoleteDomains, next);
+            domains.unregisterLocations(obsoleteDomains, progressCallback, next);
         },
 
-        progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
-        registerSubdomains.bind(null, app, overwriteDns),
+        function setupDnsIfNeeded(done) {
+            if (skipDnsSetup) return done();
+
+            async.series([
+                progressCallback.bind(null, { percent: 30, message: 'Registering subdomains' }),
+                domains.registerLocations.bind(null, [ { subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains), { overwriteDns }, progressCallback)
+            ], done);
+        },
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        services.setupAddons.bind(null, app, app.manifest.addons),
 
         progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
         createContainer.bind(null, app),
 
         startApp.bind(null, app),
 
-        progressCallback.bind(null, { percent: 80, message: 'Waiting for DNS propagation' }),
-        exports._waitForDnsPropagation.bind(null, app),
+        function waitForDns(done) {
+            if (skipDnsSetup) return done();
+
+            async.series([
+                progressCallback.bind(null, { percent: 80, message: 'Waiting for DNS propagation' }),
+                exports._waitForDnsPropagation.bind(null, app),
+            ], done);
+        },
 
         progressCallback.bind(null, { percent: 90, message: 'Configuring reverse proxy' }),
         configureReverseProxy.bind(null, app),
@@ -717,7 +672,7 @@ function migrateDataDir(app, args, progressCallback, callback) {
 
         // re-setup addons since this creates the localStorage volume
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),
+        services.setupAddons.bind(null, _.extend({}, app, { dataDir: newDataDir }), app.manifest.addons),
 
         progressCallback.bind(null, { percent: 60, message: 'Moving data dir' }),
         moveDataDir.bind(null, app, newDataDir),
@@ -762,7 +717,7 @@ function configure(app, args, progressCallback, callback) {
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         progressCallback.bind(null, { percent: 50, message: 'Setting up addons' }),
-        addons.setupAddons.bind(null, app, app.manifest.addons),
+        services.setupAddons.bind(null, app, app.manifest.addons),
 
         progressCallback.bind(null, { percent: 60, message: 'Creating container' }),
         createContainer.bind(null, app),
@@ -797,6 +752,8 @@ function update(app, args, progressCallback, callback) {
     // FIXME: this does not handle option changes (like multipleDatabases)
     const unusedAddons = _.omit(app.manifest.addons, Object.keys(updateConfig.manifest.addons));
     const httpPathsChanged = app.manifest.httpPaths !== updateConfig.manifest.httpPaths;
+    const httpPortChanged = app.manifest.httpPort !== updateConfig.manifest.httpPort;
+    const proxyAuthChanged = !_.isEqual(safe.query(app.manifest, 'addons.proxyAuth'), safe.query(updateConfig.manifest, 'addons.proxyAuth'));
 
     async.series([
         // this protects against the theoretical possibility of an app being marked for update from
@@ -835,7 +792,7 @@ function update(app, args, progressCallback, callback) {
         },
 
         // only delete unused addons after backup
-        addons.teardownAddons.bind(null, app, unusedAddons),
+        services.teardownAddons.bind(null, app, unusedAddons),
 
         // free unused ports
         function (next) {
@@ -864,17 +821,16 @@ function update(app, args, progressCallback, callback) {
         downloadIcon.bind(null, app),
 
         progressCallback.bind(null, { percent: 60, message: 'Updating addons' }),
-        addons.setupAddons.bind(null, app, updateConfig.manifest.addons),
+        services.setupAddons.bind(null, app, updateConfig.manifest.addons),
 
         progressCallback.bind(null, { percent: 70, message: 'Creating container' }),
         createContainer.bind(null, app),
 
         startApp.bind(null, app),
 
-        // needed for httpPaths changes
         progressCallback.bind(null, { percent: 90, message: 'Configuring reverse proxy' }),
         function (next) {
-            if (!httpPathsChanged) return next();
+            if (!httpPathsChanged && !proxyAuthChanged && !httpPortChanged) return next();
 
             configureReverseProxy(app, next);
         },
@@ -902,7 +858,7 @@ function start(app, args, progressCallback, callback) {
 
     async.series([
         progressCallback.bind(null, { percent: 10, message: 'Starting app services' }),
-        addons.startAppServices.bind(null, app),
+        services.startAppServices.bind(null, app),
 
         progressCallback.bind(null, { percent: 35, message: 'Starting container' }),
         docker.startContainer.bind(null, app.id),
@@ -936,7 +892,7 @@ function stop(app, args, progressCallback, callback) {
         docker.stopContainers.bind(null, app.id),
 
         progressCallback.bind(null, { percent: 50, message: 'Stopping app services' }),
-        addons.stopAppServices.bind(null, app),
+        services.stopAppServices.bind(null, app),
 
         progressCallback.bind(null, { percent: 80, message: 'Removing collectd profile' }),
         removeCollectdProfile.bind(null, app),
@@ -985,7 +941,7 @@ function uninstall(app, args, progressCallback, callback) {
         deleteContainers.bind(null, app, {}),
 
         progressCallback.bind(null, { percent: 30, message: 'Teardown addons' }),
-        addons.teardownAddons.bind(null, app, app.manifest.addons),
+        services.teardownAddons.bind(null, app, app.manifest.addons),
 
         progressCallback.bind(null, { percent: 40, message: 'Cleanup file manager' }),
 
@@ -996,7 +952,7 @@ function uninstall(app, args, progressCallback, callback) {
         docker.deleteImage.bind(null, app.manifest),
 
         progressCallback.bind(null, { percent: 70, message: 'Unregistering domains' }),
-        unregisterSubdomains.bind(null, app, [ { subdomain: app.location, domain: app.domain } ].concat(app.alternateDomains)),
+        domains.unregisterLocations.bind(null, [ { subdomain: app.location, domain: app.domain } ].concat(app.alternateDomains).concat(app.aliasDomains), progressCallback),
 
         progressCallback.bind(null, { percent: 80, message: 'Cleanup icon' }),
         removeIcon.bind(null, app),

src/apptaskmanager.js

@@ -13,7 +13,7 @@ let assert = require('assert'),
     path = require('path'),
     paths = require('./paths.js'),
     scheduler = require('./scheduler.js'),
-    sftp = require('./sftp.js'),
+    services = require('./services.js'),
     tasks = require('./tasks.js');
 
 let gActiveTasks = { }; // indexed by app id
@@ -37,9 +37,10 @@ function initializeSync() {
 }
 
 // callback is called when task is finished
-function scheduleTask(appId, taskId, callback) {
+function scheduleTask(appId, taskId, options, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof taskId, 'string');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     if (!gInitialized) initializeSync();
@@ -51,7 +52,7 @@ function scheduleTask(appId, taskId, callback) {
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug(`Reached concurrency limit, queueing task id ${taskId}`);
         tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, callback });
+        gPendingTasks.push({ appId, taskId, options, callback });
         return;
     }
 
@@ -60,7 +61,7 @@ function scheduleTask(appId, taskId, callback) {
     if (lockError) {
         debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
         tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, callback });
+        gPendingTasks.push({ appId, taskId, options, callback });
         return;
     }
 
@@ -72,25 +73,26 @@ function scheduleTask(appId, taskId, callback) {
 
     scheduler.suspendJobs(appId);
 
-    // TODO: set memory limit for app backup task
-    tasks.startTask(taskId, { logFile, timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15 }, function (error, result) {
+    tasks.startTask(taskId, Object.assign(options, { logFile }), function (error, result) {
         callback(error, result);
 
         delete gActiveTasks[appId];
         locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
 
-        // post app task hooks
-        sftp.rebuild(error => { if (error) debug('Unable to rebuild sftp:', error); });
         scheduler.resumeJobs(appId);
     });
 }
 
 function startNextTask() {
-    if (gPendingTasks.length === 0) return;
+    if (gPendingTasks.length === 0) {
+        // rebuild sftp when task queue is empty. this minimizes risk of sftp rebuild overlapping with other app tasks
+        services.rebuildService('sftp', error => { if (error) debug('Unable to rebuild sftp:', error); });
+        return;
+    }
 
     assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
     const t = gPendingTasks.shift();
-    scheduleTask(t.appId, t.taskId, t.callback);
+    scheduleTask(t.appId, t.taskId, t.options, t.callback);
 }
 

src/autoconfig.xml.ejs

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<clientConfig version="1.1">
+  <emailProvider id="<%= domain %>">
+    <domain><%= domain %></domain>
+    <displayName>Cloudron Mail</displayName>
+    <displayShortName>Cloudron</displayShortName>
+    <incomingServer type="imap">
+      <hostname><%= mailFqdn %></hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <outgoingServer type="smtp">
+      <hostname><%= mailFqdn %></hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+      <addThisServer>true</addThisServer>
+    </outgoingServer>
+
+    <documentation url="http://cloudron.io/email/#autodiscover">
+       <descr lang="en">Cloudron Email</descr>
+    </documentation>
+
+  </emailProvider>
+</clientConfig>
+

src/backupdb.js

@@ -18,6 +18,7 @@ exports = module.exports = {
     get,
     del,
     update,
+    list,
 
     _clear: clear
 };
@@ -80,6 +81,21 @@ function getByIdentifierPaged(identifier, page, perPage, callback) {
         });
 }
 
+function list(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups ORDER BY creationTime DESC LIMIT ?,?',
+        [ (page-1)*perPage, perPage ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
+}
+
 function get(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');

src/backups.js

@@ -1,36 +1,36 @@
 'use strict';
 
 exports = module.exports = {
-    testConfig: testConfig,
-    testProviderConfig: testProviderConfig,
+    testConfig,
+    testProviderConfig,
 
     getByIdentifierAndStatePaged,
 
-    get: get,
+    get,
 
-    startBackupTask: startBackupTask,
+    startBackupTask,
 
-    restore: restore,
+    restore,
 
-    backupApp: backupApp,
-    downloadApp: downloadApp,
+    backupApp,
+    downloadApp,
 
-    backupBoxAndApps: backupBoxAndApps,
+    backupBoxAndApps,
 
-    upload: upload,
+    upload,
 
-    startCleanupTask: startCleanupTask,
-    cleanup: cleanup,
-    cleanupCacheFilesSync: cleanupCacheFilesSync,
+    startCleanupTask,
+    cleanup,
+    cleanupCacheFilesSync,
 
-    injectPrivateFields: injectPrivateFields,
-    removePrivateFields: removePrivateFields,
+    injectPrivateFields,
+    removePrivateFields,
 
-    checkConfiguration: checkConfiguration,
+    checkConfiguration,
 
-    configureCollectd: configureCollectd,
+    configureCollectd,
 
-    generateEncryptionKeysSync: generateEncryptionKeysSync,
+    generateEncryptionKeysSync,
 
     BACKUP_IDENTIFIER_BOX: 'box',
 
@@ -48,8 +48,7 @@ exports = module.exports = {
     _applyBackupRetentionPolicy: applyBackupRetentionPolicy
 };
 
-var addons = require('./addons.js'),
-    apps = require('./apps.js'),
+const apps = require('./apps.js'),
     async = require('async'),
     assert = require('assert'),
     backupdb = require('./backupdb.js'),
@@ -72,6 +71,7 @@ var addons = require('./addons.js'),
     progressStream = require('progress-stream'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     syncer = require('./syncer.js'),
     tar = require('tar-fs'),
@@ -108,6 +108,7 @@ function api(provider) {
     case 'backblaze-b2': return require('./storage/s3.js');
     case 'linode-objectstorage': return require('./storage/s3.js');
     case 'ovh-objectstorage': return require('./storage/s3.js');
+    case 'ionos-objectstorage': return require('./storage/s3.js');
     case 'noop': return require('./storage/noop.js');
     default: return null;
     }
@@ -549,21 +550,29 @@ function saveFsMetadata(dataLayout, metadataFile, callback) {
     // contains paths prefixed with './'
     let metadata = {
         emptyDirs: [],
-        execFiles: []
+        execFiles: [],
+        symlinks: []
     };
 
+    // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer
     for (let lp of dataLayout.localPaths()) {
-        var emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty\n`, { encoding: 'utf8' });
-        if (emptyDirs === null) return callback(safe.error);
+        const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (emptyDirs === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`));
         if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed)));
 
-        var execFiles = safe.child_process.execSync(`find ${lp} -type f -executable\n`, { encoding: 'utf8' });
-        if (execFiles === null) return callback(safe.error);
-
+        const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (execFiles === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`));
         if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef)));
+
+        const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 });
+        if (symlinks === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`));
+        if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => {
+            const target = safe.fs.readlinkSync(sl);
+            return { path: dataLayout.toRemotePath(sl), target };
+        }));
     }
 
-    if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) return callback(safe.error);
+    if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) return callback(new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`));
 
     callback();
 }
@@ -691,7 +700,19 @@ function restoreFsMetadata(dataLayout, metadataFile, callback) {
         }, function (error) {
             if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to chmod: ${error.message}`));
 
-            callback();
+            async.eachSeries(metadata.symlinks || [], function createSymlink(symlink, iteratorDone) {
+                if (!symlink.target) return iteratorDone();
+                // the path may not exist if we had a directory full of symlinks
+                fs.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }, function (error) {
+                    if (error) return iteratorDone(error);
+
+                    fs.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file', iteratorDone);
+                });
+            }, function (error) {
+                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to symlink: ${error.message}`));
+
+                callback();
+            });
         });
     });
 }
@@ -805,7 +826,9 @@ function restore(backupConfig, backupId, progressCallback, callback) {
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    const dataLayout = new DataLayout(paths.BOX_DATA_DIR, []);
+    const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
+    if (!boxDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`));
+    const dataLayout = new DataLayout(boxDataDir, []);
 
     download(backupConfig, backupId, backupConfig.format, dataLayout, progressCallback, function (error) {
         if (error) return callback(error);
@@ -829,7 +852,7 @@ function downloadApp(app, restoreConfig, progressCallback, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
-    if (!appDataDir) return callback(safe.error);
+    if (!appDataDir) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
     const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
 
     const startTime = new Date();
@@ -932,7 +955,7 @@ function uploadBoxSnapshot(backupConfig, progressCallback, callback) {
         if (error) return callback(error);
 
         const boxDataDir = safe.fs.realpathSync(paths.BOX_DATA_DIR);
-        if (!boxDataDir) return callback(safe.error);
+        if (!boxDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`));
 
         const uploadConfig = {
             backupId: 'snapshot/box',
@@ -1045,7 +1068,7 @@ function snapshotApp(app, progressCallback, callback) {
         return callback(new BoxError(BoxError.FS_ERROR, 'Error creating config.json: ' + safe.error.message));
     }
 
-    addons.backupAddons(app, app.manifest.addons, function (error) {
+    services.backupAddons(app, app.manifest.addons, function (error) {
         if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
 
         debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`);
@@ -1115,7 +1138,7 @@ function uploadAppSnapshot(backupConfig, app, progressCallback, callback) {
 
         const backupId = util.format('snapshot/app_%s', app.id);
         const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
-        if (!appDataDir) return callback(safe.error);
+        if (!appDataDir) return callback(new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`));
 
         const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []);
 
@@ -1323,7 +1346,7 @@ function cleanupBackup(backupConfig, backup, progressCallback, callback) {
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
 
-    var backupFilePath = getBackupFilePath(backupConfig, backup.id, backup.format);
+    const backupFilePath = getBackupFilePath(backupConfig, backup.id, backup.format);
 
     function done(error) {
         if (error) {
@@ -1427,6 +1450,46 @@ function cleanupBoxBackups(backupConfig, progressCallback, callback) {
     });
 }
 
+function cleanupMissingBackups(backupConfig, progressCallback, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    let page = 1, perPage = 1000, more = false, missingBackupIds = [];
+
+    async.doWhilst(function (whilstCallback) {
+        backupdb.list(page, perPage, function (error, result) {
+            if (error) return whilstCallback(error);
+
+            async.eachSeries(result, function (backup, next) {
+                let backupFilePath = getBackupFilePath(backupConfig, backup.id, backup.format);
+                if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
+
+                api(backupConfig.provider).exists(backupConfig, backupFilePath, function (error, exists) {
+                    if (error || exists) return next();
+
+                    progressCallback({ message: `Removing missing backup ${backup.id}`});
+
+                    backupdb.del(backup.id, function (error) {
+                        if (error) debug(`cleanupBackup: error removing ${backup.id} from database`, error);
+
+                        missingBackupIds.push(backup.id);
+
+                        next();
+                    });
+                });
+            }, function () {
+                more = result.length === perPage;
+                whilstCallback();
+            });
+        });
+    }, function (testDone) { return testDone(null, more); }, function (error) {
+        if (error) return callback(error);
+
+        return callback(null, missingBackupIds);
+    });
+}
+
 function cleanupCacheFilesSync() {
     var files = safe.fs.readdirSync(path.join(paths.BACKUP_INFO_DIR));
     if (!files) return;
@@ -1498,12 +1561,18 @@ function cleanup(progressCallback, callback) {
             cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback, function (error, removedAppBackupIds) {
                 if (error) return callback(error);
 
-                progressCallback({ percent: 90, message: 'Cleaning snapshots' });
+                progressCallback({ percent: 70, message: 'Cleaning missing backups' });
 
-                cleanupSnapshots(backupConfig, function (error) {
+                cleanupMissingBackups(backupConfig, progressCallback, function (error, missingBackupIds) {
                     if (error) return callback(error);
 
-                    callback(null, { removedBoxBackupIds, removedAppBackupIds });
+                    progressCallback({ percent: 90, message: 'Cleaning snapshots' });
+
+                    cleanupSnapshots(backupConfig, function (error) {
+                        if (error) return callback(error);
+
+                        callback(null, { removedBoxBackupIds, removedAppBackupIds, missingBackupIds });
+                    });
                 });
             });
         });
@@ -1515,12 +1584,13 @@ function startCleanupTask(auditSource, callback) {
     tasks.add(tasks.TASK_CLEAN_BACKUPS, [], function (error, taskId) {
         if (error) return callback(error);
 
-        tasks.startTask(taskId, {}, (error, result) => { // result is { removedBoxBackups, removedAppBackups }
+        tasks.startTask(taskId, {}, (error, result) => { // result is { removedBoxBackupIds, removedAppBackupIds, missingBackupIds }
             eventlog.add(eventlog.ACTION_BACKUP_CLEANUP_FINISH, auditSource, {
                 taskId,
                 errorMessage: error ? error.message : null,
-                removedBoxBackups: result ? result.removedBoxBackups : [],
-                removedAppBackups: result ? result.removedAppBackups : []
+                removedBoxBackupIds: result ? result.removedBoxBackupIds : [],
+                removedAppBackupIds: result ? result.removedAppBackupIds : [],
+                missingBackupIds: result ? result.missingBackupIds : []
             });
         });
 

src/cert/acme2.js

@@ -139,7 +139,7 @@ Acme2.prototype.updateContact = function (registrationUri, callback) {
     const that = this;
     this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 200, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         debug(`updateContact: contact of user updated to ${that.email}`);
 
@@ -160,7 +160,7 @@ Acme2.prototype.registerUser = function (callback) {
     this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
         // 200 if already exists. 201 for new accounts
-        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to register new account. Expecting 200 or 201, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         debug(`registerUser: user registered keyid: ${result.headers.location}`);
 
@@ -186,7 +186,7 @@ Acme2.prototype.newOrder = function (domain, callback) {
     this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
         if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to send new order. Expecting 201, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         debug('newOrder: created order %s %j', domain, result.body);
 
@@ -259,7 +259,7 @@ Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
 
     this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 200, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         callback();
     });
@@ -313,7 +313,7 @@ Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, cal
     this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
         if (error) return callback(error);
         // 429 means we reached the cert limit for this domain
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 200, got %s %s', result.statusCode, result.text)));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
         return callback(null);
     });
@@ -362,7 +362,7 @@ Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
         that.postAsGet(certUrl, function (error, result) {
             if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
             if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
-            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
 
             const fullChainPem = result.body; // buffer
 
@@ -572,48 +572,50 @@ Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
 Acme2.prototype.getDirectory = function (callback) {
     const that = this;
 
-    request.get(this.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
-        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
+    async.retry({ times: 3, interval: 20000 }, function (retryCallback) {
+        request.get(that.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
+            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
+            if (response.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
 
-        if (typeof response.body.newNonce !== 'string' ||
-            typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));
+            if (typeof response.body.newNonce !== 'string' ||
+                typeof response.body.newOrder !== 'string' ||
+                typeof response.body.newAccount !== 'string') return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));
 
-        that.directory = response.body;
+            that.directory = response.body;
 
-        callback(null);
-    });
+            retryCallback(null);
+        });
+    }, callback);
 };
 
-Acme2.prototype.getCertificate = function (hostname, domain, callback) {
-    assert.strictEqual(typeof hostname, 'string');
+Acme2.prototype.getCertificate = function (vhost, domain, callback) {
+    assert.strictEqual(typeof vhost, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`getCertificate: start acme flow for ${hostname} from ${this.caDirectory}`);
+    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
 
-    if (hostname !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
-        hostname = domains.makeWildcard(hostname);
-        debug(`getCertificate: will get wildcard cert for ${hostname}`);
+    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
+        vhost = domains.makeWildcard(vhost);
+        debug(`getCertificate: will get wildcard cert for ${vhost}`);
     }
 
     const that = this;
     this.getDirectory(function (error) {
         if (error) return callback(error);
 
-        that.acmeFlow(hostname, domain, function (error) {
+        that.acmeFlow(vhost, domain, function (error) {
             if (error) return callback(error);
 
             var outdir = paths.APP_CERTS_DIR;
-            const certName = hostname.replace('*.', '_.');
+            const certName = vhost.replace('*.', '_.');
             callback(null, path.join(outdir, `${certName}.cert`), path.join(outdir, `${certName}.key`));
         });
     });
 };
 
-function getCertificate(hostname, domain, options, callback) {
-    assert.strictEqual(typeof hostname, 'string');
+function getCertificate(vhost, domain, options, callback) {
+    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -623,6 +625,6 @@ function getCertificate(hostname, domain, options, callback) {
         debug(`getCertificate: attempt ${attempt++}`);
 
         let acme = new Acme2(options || { });
-        acme.getCertificate(hostname, domain, retryCallback);
+        acme.getCertificate(vhost, domain, retryCallback);
     }, callback);
 }

src/cloudron.js

@@ -17,12 +17,12 @@ exports = module.exports = {
     setDashboardDomain,
     updateDashboardDomain,
     renewCerts,
+    syncDnsRecords,
 
     runSystemChecks
 };
 
-var addons = require('./addons.js'),
-    apps = require('./apps.js'),
+const apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
     async = require('async'),
@@ -43,6 +43,7 @@ var addons = require('./addons.js'),
     platform = require('./platform.js'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     spawn = require('child_process').spawn,
@@ -72,14 +73,15 @@ function uninitialize(callback) {
     ], callback);
 }
 
-function onActivated(callback) {
+function onActivated(options, callback) {
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     // Starting the platform after a user is available means:
     // 1. mail bounces can now be sent to the cloudron owner
     // 2. the restore code path can run without sudo (since mail/ is non-root)
     async.series([
-        platform.start,
+        platform.start.bind(null, options),
         cron.startJobs,
         function checkBackupConfiguration(done) {
             backups.checkConfiguration(function (error, message) {
@@ -147,7 +149,7 @@ function runStartupTasks() {
                     return reverseProxy.writeDefaultConfig({ activated: false }, callback);
                 }
 
-                onActivated(callback);
+                onActivated({}, callback);
             });
         }
     ];
@@ -350,7 +352,7 @@ function updateDashboardDomain(domain, auditSource, callback) {
     setDashboardDomain(domain, auditSource, function (error) {
         if (error) return callback(error);
 
-        addons.rebuildService('turn', NOOP_CALLBACK); // to update the realm variable
+        services.rebuildService('turn', NOOP_CALLBACK); // to update the realm variable
 
         callback(null);
     });
@@ -400,3 +402,16 @@ function setupDnsAndCert(subdomain, domain, auditSource, progressCallback, callb
         });
     });
 }
+
+function syncDnsRecords(options, callback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ], function (error, taskId) {
+        if (error) return callback(error);
+
+        tasks.startTask(taskId, {}, NOOP_CALLBACK);
+
+        callback(null, taskId);
+    });
+}

src/collectd/volume.ejs

@@ -0,0 +1,9 @@
+<Plugin python>
+  <Module du>
+    <Path>
+        Instance "<%= volumeId %>"
+        Dir "<%= hostPath %>"
+    </Path>
+  </Module>
+</Plugin>
+

src/constants.js

@@ -37,7 +37,14 @@ exports = module.exports = {
     DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024), // see also client.js
 
     DEMO_USERNAME: 'cloudron',
-    DEMO_BLACKLISTED_APPS: [ 'com.github.cloudtorrent', 'net.alltubedownload.cloudronapp', 'com.adguard.home.cloudronapp' ],
+    DEMO_BLACKLISTED_APPS: [
+        'com.github.cloudtorrent',
+        'net.alltubedownload.cloudronapp',
+        'com.adguard.home.cloudronapp',
+        'com.transmissionbt.cloudronapp',
+        'io.github.sickchill.cloudronapp',
+        'to.couchpota.cloudronapp'
+    ],
 
     AUTOUPDATE_PATTERN_NEVER: 'never',
 

src/cron.js

@@ -33,9 +33,10 @@ var appHealthMonitor = require('./apphealthmonitor.js'),
     settings = require('./settings.js'),
     system = require('./system.js'),
     updater = require('./updater.js'),
-    updateChecker = require('./updatechecker.js');
+    updateChecker = require('./updatechecker.js'),
+    _ = require('underscore');
 
-var gJobs = {
+const gJobs = {
     autoUpdater: null,
     backup: null,
     updateChecker: null,
@@ -51,7 +52,7 @@ var gJobs = {
     appHealthMonitor: null
 };
 
-var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 // cron format
 // Seconds: 0-59
@@ -198,9 +199,10 @@ function autoupdatePatternChanged(pattern, tz) {
                 return;
             }
 
-            if (updateInfo.apps && Object.keys(updateInfo.apps).length > 0) {
-                debug('Starting app update to %j', updateInfo.apps);
-                apps.autoupdateApps(updateInfo.apps, auditSource.CRON, NOOP_CALLBACK);
+            const appUpdateInfo = _.omit(updateInfo, 'box');
+            if (Object.keys(appUpdateInfo).length > 0) {
+                debug('Starting app update to %j', appUpdateInfo);
+                apps.autoupdateApps(appUpdateInfo, auditSource.CRON, NOOP_CALLBACK);
             } else {
                 debug('No app auto updates available');
             }

src/dns/digitalocean.js

@@ -1,13 +1,13 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
 var assert = require('assert'),
@@ -69,7 +69,7 @@ function getInternal(dnsConfig, zoneName, name, type, callback) {
 
                 iteratorDone();
             });
-    }, function () { return !!nextPage; }, function (error) {
+    }, function (testDone) { return testDone(null, !!nextPage); }, function (error) {
         debug('getInternal:', error, JSON.stringify(matchingRecords));
 
         if (error) return callback(error);

src/dns/linode.js

@@ -1,13 +1,13 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
 let async = require('async'),
@@ -99,7 +99,7 @@ function getZoneRecords(dnsConfig, zoneName, name, type, callback) {
 
                     iteratorDone();
                 });
-        }, function () { return more; }, function (error) {
+        }, function (testDone) { return testDone(null, more); }, function (error) {
             debug('getZoneRecords:', error, JSON.stringify(records));
 
             if (error) return callback(error);

src/dns/netcup.js

@@ -0,0 +1,303 @@
+'use strict';
+
+exports = module.exports = {
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    constants = require('../constants.js'),
+    debug = require('debug')('box:dns/netcup'),
+    dns = require('../native-dns.js'),
+    domains = require('../domains.js'),
+    superagent = require('superagent'),
+    util = require('util'),
+    waitForDns = require('./waitfordns.js');
+
+var API_ENDPOINT = 'https://ccp.netcup.net/run/webservice/servers/endpoint.php?JSON';
+
+function formatError(response) {
+    if (response.body) return util.format('Netcup DNS error [%s] %s', response.body.statuscode, response.body.longmessage);
+    else return util.format('Netcup DNS error [%s] %s', response.statusCode, response.text);
+}
+
+function removePrivateFields(domainObject) {
+    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    return domainObject;
+}
+
+function injectPrivateFields(newConfig, currentConfig) {
+    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+}
+
+// returns a api session id
+function login(dnsConfig, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const data = {
+        action: 'login',
+        param:{
+            apikey: dnsConfig.apiKey,
+            apipassword: dnsConfig.apiPassword,
+            customernumber: dnsConfig.customerNumber
+        }
+    };
+
+    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+        callback(null, result.body.responsedata.apisessionid);
+    });
+}
+
+function getAllRecords(dnsConfig, apiSessionId, zoneName, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof apiSessionId, 'string');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`getAllRecords: getting dns records of ${zoneName}`);
+
+    const data = {
+        action: 'infoDnsRecords',
+        param:{
+            apikey: dnsConfig.apiKey,
+            apisessionid: apiSessionId,
+            customernumber: dnsConfig.customerNumber,
+            domainname: zoneName,
+        }
+    };
+
+    superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
+        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+        debug('getAllRecords:', JSON.stringify(result.body.responsedata.dnsrecords || []));
+
+        callback(null, result.body.responsedata.dnsrecords || []);
+    });
+}
+
+function upsert(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '@';
+
+    debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
+
+    login(dnsConfig, function (error, apiSessionId) {
+        if (error) return callback(error);
+
+        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
+            if (error) return callback(error);
+
+            let records = [];
+
+            values.forEach(function (value) {
+                // remove possible quotation
+                if (value.charAt(0) === '"') value = value.slice(1);
+                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
+
+                let priority = null;
+                if (type === 'MX') {
+                    priority = parseInt(value.split(' ')[0], 10);
+                    value = value.split(' ')[1];
+                }
+
+                let record = result.find(function (r) { return r.hostname === name && r.type === type; });
+                if (!record) record = { hostname: name, type: type, destination: value, deleterecord: false };
+                else record.destination = value;
+
+                if (priority !== null) record.priority = priority;
+
+                records.push(record);
+            });
+
+            const data = {
+                action: 'updateDnsRecords',
+                param:{
+                    apikey: dnsConfig.apiKey,
+                    apisessionid: apiSessionId,
+                    customernumber: dnsConfig.customerNumber,
+                    domainname: zoneName,
+                    dnsrecordset: {
+                        dnsrecords: records
+                    }
+                }
+            };
+
+            debug('upserting', JSON.stringify(data));
+
+            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                debug('upsert:', result.body);
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function get(domainObject, location, type, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '@';
+
+    debug('get: %s for zone %s of type %s', name, zoneName, type);
+
+    login(dnsConfig, function (error, apiSessionId) {
+        if (error) return callback(error);
+
+        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
+            if (error) return callback(error);
+
+            // We only return the value string
+            callback(null, result.filter(function (r) { return r.hostname === name && r.type === type; }).map(function (r) { return r.destination; }));
+        });
+    });
+}
+
+function del(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName,
+        name = domains.getName(domainObject, location, type) || '@';
+
+    debug('del: %s for zone %s of type %s with values %j', name, zoneName, type, values);
+
+    login(dnsConfig, function (error, apiSessionId) {
+        if (error) return callback(error);
+
+        getAllRecords(dnsConfig, apiSessionId, zoneName, function (error, result) {
+            if (error) return callback(error);
+
+            let records = [];
+
+            values.forEach(function (value) {
+                // remove possible quotation
+                if (value.charAt(0) === '"') value = value.slice(1);
+                if (value.charAt(value.length -1) === '"') value = value.slice(0, -1);
+
+                let record = result.find(function (r) { return r.hostname === name && r.type === type && r.destination === value; });
+                if (!record) return;
+
+                record.deleterecord = true;
+
+                records.push(record);
+            });
+
+            if (records.length === 0) return callback(null);
+
+            const data = {
+                action: 'updateDnsRecords',
+                param:{
+                    apikey: dnsConfig.apiKey,
+                    apisessionid: apiSessionId,
+                    customernumber: dnsConfig.customerNumber,
+                    domainname: zoneName,
+                    dnsrecordset: {
+                        dnsrecords: records
+                    }
+                }
+            };
+
+            superagent.post(API_ENDPOINT).send(data).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+                if (result.body.statuscode !== 2000) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+                debug('del:', result.body.responsedata);
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function wait(domainObject, location, type, value, options, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+    assert.strictEqual(typeof callback, 'function');
+
+    const fqdn = domains.fqdn(location, domainObject);
+
+    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+}
+
+function verifyDnsConfig(domainObject, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config,
+        zoneName = domainObject.zoneName;
+
+    if (!dnsConfig.customerNumber || typeof dnsConfig.customerNumber !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'customerNumber must be a non-empty string', { field: 'customerNumber' }));
+    if (!dnsConfig.apiKey || typeof dnsConfig.apiKey !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiKey must be a non-empty string', { field: 'apiKey' }));
+    if (!dnsConfig.apiPassword || typeof dnsConfig.apiPassword !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'apiPassword must be a non-empty string', { field: 'apiPassword' }));
+
+    const ip = '127.0.0.1';
+
+    var credentials = {
+        customerNumber: dnsConfig.customerNumber,
+        apiKey: dnsConfig.apiKey,
+        apiPassword: dnsConfig.apiPassword,
+    };
+
+    if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here
+
+    dns.resolve(zoneName, 'NS', { timeout: 5000 }, function (error, nameservers) {
+        if (error && error.code === 'ENOTFOUND') return callback(new BoxError(BoxError.BAD_FIELD, 'Unable to resolve nameservers for this domain', { field: 'nameservers' }));
+        if (error || !nameservers) return callback(new BoxError(BoxError.BAD_FIELD, error ? error.message : 'Unable to get nameservers', { field: 'nameservers' }));
+
+        if (!nameservers.every(function (n) { return n.toLowerCase().indexOf('dns.netcup.net') !== -1; })) {
+            debug('verifyDnsConfig: %j does not contains Netcup NS', nameservers);
+            return callback(new BoxError(BoxError.BAD_FIELD, 'Domain nameservers are not set to Netcup', { field: 'nameservers' }));
+        }
+
+        const location = 'cloudrontestdns';
+
+        upsert(domainObject, location, 'A', [ ip ], function (error) {
+            if (error) return callback(error);
+
+            debug('verifyDnsConfig: Test A record added');
+
+            del(domainObject, location, 'A', [ ip ], function (error) {
+                if (error) return callback(error);
+
+                debug('verifyDnsConfig: Test A record removed again');
+
+                callback(null, credentials);
+            });
+        });
+    });
+}

src/docker.js

@@ -29,10 +29,11 @@ exports = module.exports = {
     memoryUsage,
     createVolume,
     removeVolume,
-    clearVolume
+    clearVolume,
+    update
 };
 
-var addons = require('./addons.js'),
+const apps = require('./apps.js'),
     async = require('async'),
     assert = require('assert'),
     BoxError = require('./boxerror.js'),
@@ -42,9 +43,12 @@ var addons = require('./addons.js'),
     Docker = require('dockerode'),
     os = require('os'),
     path = require('path'),
+    reverseProxy = require('./reverseproxy.js'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     safe = require('safetydance'),
+    system = require('./system.js'),
     util = require('util'),
     volumes = require('./volumes.js'),
     _ = require('underscore');
@@ -55,11 +59,13 @@ const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
 const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
 const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
 
-function testRegistryConfig(auth, callback) {
-    assert.strictEqual(typeof auth, 'object');
+function testRegistryConfig(config, callback) {
+    assert.strictEqual(typeof config, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    gConnection.checkAuth(auth, function (error /*, data */) { // this returns a 500 even for auth errors
+    if (config.provider === 'noop') return callback();
+
+    gConnection.checkAuth(config, function (error /*, data */) { // this returns a 500 even for auth errors
         if (error) return callback(new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' }));
 
         callback();
@@ -78,14 +84,14 @@ function removePrivateFields(registryConfig) {
     return registryConfig;
 }
 
-function setRegistryConfig(auth, callback) {
-    assert.strictEqual(typeof auth, 'object');
+function setRegistryConfig(config, callback) {
+    assert.strictEqual(typeof config, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    const isLogin = !!auth.password;
+    const isLogin = !!config.password;
 
     // currently, auth info is not stashed in the db but maybe it should for restore to work?
-    const cmd = isLogin ? `docker login ${auth.serverAddress} --username ${auth.username} --password ${auth.password}` : `docker logout ${auth.serverAddress}`;
+    const cmd = isLogin ? `docker login ${config.serverAddress} --username ${config.username} --password ${config.password}` : `docker logout ${config.serverAddress}`;
 
     child_process.exec(cmd, { }, function (error /*, stdout, stderr */) {
         if (error) return callback(new BoxError(BoxError.ACCESS_DENIED, error.message));
@@ -131,12 +137,12 @@ function getRegistryConfig(image, callback) {
 }
 
 function pullImage(manifest, callback) {
-    getRegistryConfig(manifest.dockerImage, function (error, authConfig) {
+    getRegistryConfig(manifest.dockerImage, function (error, config) {
         if (error) return callback(error);
 
-        debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${authConfig ? 'yes' : 'no'}`);
+        debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${config ? 'yes' : 'no'}`);
 
-        gConnection.pull(manifest.dockerImage, { authconfig: authConfig }, function (error, stream) {
+        gConnection.pull(manifest.dockerImage, { authconfig: config }, function (error, stream) {
             if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${manifest.dockerImage}. message: ${error.message} statusCode: ${error.statusCode}`));
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`));
 
@@ -188,25 +194,97 @@ function downloadImage(manifest, callback) {
     });
 }
 
-function getBinds(app, callback) {
+function getVolumeMounts(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (app.mounts.length === 0) return callback(null);
+    let mounts = [];
 
-    let binds = [];
+    if (app.mounts.length === 0) return callback(null, []);
 
     volumes.list(function (error, result) {
         if (error) return callback(error);
+
         let volumesById = {};
         result.forEach(r => volumesById[r.id] = r);
 
         for (const mount of app.mounts) {
             const volume = volumesById[mount.volumeId];
-            binds.push(`${volume.hostPath}:/media/${volume.name}:${mount.readOnly ? 'ro' : 'rw'}`);
+
+            mounts.push({
+                Source: volume.hostPath,
+                Target: `/media/${volume.name}`,
+                Type: 'bind',
+                ReadOnly: mount.readOnly
+            });
         }
 
-        callback(null, binds);
+        callback(null, mounts);
+    });
+}
+
+function getAddonMounts(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    let mounts = [];
+
+    const addons = app.manifest.addons;
+    if (!addons) return callback(null, mounts);
+
+    async.eachSeries(Object.keys(addons), function (addon, iteratorDone) {
+        switch (addon) {
+        case 'localstorage':
+            mounts.push({
+                Target: '/app/data',
+                Source: `${app.id}-localstorage`,
+                Type: 'volume',
+                ReadOnly: false
+            });
+
+            return iteratorDone();
+        case 'tls':
+            reverseProxy.getCertificate(app.fqdn, app.domain, function (error, bundle) {
+                if (error) return iteratorDone(error);
+
+                mounts.push({
+                    Target: '/etc/certs/tls_cert.pem',
+                    Source: bundle.certFilePath,
+                    Type: 'bind',
+                    ReadOnly: true
+                });
+
+                mounts.push({
+                    Target: '/etc/certs/tls_key.pem',
+                    Source: bundle.keyFilePath,
+                    Type: 'bind',
+                    ReadOnly: true
+                });
+
+                iteratorDone();
+            });
+
+            return;
+        default:
+            iteratorDone();
+        }
+    }, function (error) {
+        callback(error, mounts);
+    });
+}
+
+function getMounts(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    getVolumeMounts(app, function (error, volumeMounts) {
+        if (error) return callback(error);
+
+        getAddonMounts(app, function (error, addonMounts) {
+            if (error) return callback(error);
+
+            callback(null, volumeMounts.concat(addonMounts));
+        });
     });
 }
 
@@ -266,23 +344,16 @@ function createSubcontainer(app, name, cmd, options, callback) {
     let appEnv = [];
     Object.keys(app.env).forEach(function (name) { appEnv.push(`${name}=${app.env[name]}`); });
 
-    // first check db record, then manifest
-    var memoryLimit = app.memoryLimit || manifest.memoryLimit || 0;
-
-    if (memoryLimit === -1) { // unrestricted
-        memoryLimit = 0;
-    } else if (memoryLimit === 0 || memoryLimit < constants.DEFAULT_MEMORY_LIMIT) { // ensure we never go below minimum (in case we change the default)
-        memoryLimit = constants.DEFAULT_MEMORY_LIMIT;
-    }
+    let memoryLimit = apps.getMemoryLimit(app);
 
     // give scheduler tasks twice the memory limit since background jobs take more memory
     // if required, we can make this a manifest and runtime argument later
     if (!isAppContainer) memoryLimit *= 2;
 
-    addons.getEnvironment(app, function (error, addonEnv) {
+    services.getEnvironment(app, function (error, addonEnv) {
         if (error) return callback(error);
 
-        getBinds(app, function (error, binds) {
+        getMounts(app, function (error, mounts) {
             if (error) return callback(error);
 
             let containerOptions = {
@@ -303,8 +374,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                     'isCloudronManaged': String(true)
                 },
                 HostConfig: {
-                    Mounts: addons.getMountsSync(app, app.manifest.addons),
-                    Binds: binds, // ideally, we have to use 'Mounts' but we have to create volumes then
+                    Mounts: mounts,
                     LogConfig: {
                         Type: 'syslog',
                         Config: {
@@ -313,7 +383,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                             'syslog-format': 'rfc5424'
                         }
                     },
-                    Memory: memoryLimit / 2,
+                    Memory: system.getMemoryAllocation(memoryLimit),
                     MemorySwap: memoryLimit, // Memory + Swap
                     PortBindings: isAppContainer ? dockerPortBindings : { },
                     PublishAllPorts: false,
@@ -698,3 +768,17 @@ function info(callback) {
         callback(null, result);
     });
 }
+
+function update(name, memory, memorySwap, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof memory, 'number');
+    assert.strictEqual(typeof memorySwap, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    const args = `update --memory ${memory} --memory-swap ${memorySwap} ${name}`.split(' ');
+    // scale back db containers, if possible. this is retried because updating memory constraints can fail
+    // with failed to write to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/xx/memory.memsw.limit_in_bytes: device or resource busy
+    async.retry({ times: 10, interval: 60 * 1000 }, function (retryCallback) {
+        shell.spawn(`update(${name})`, '/usr/bin/docker', args, { }, retryCallback);
+    }, callback);
+}

src/domaindb.js

@@ -16,14 +16,18 @@ var assert = require('assert'),
     database = require('./database.js'),
     safe = require('safetydance');
 
-var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson' ].join(',');
+var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson' ].join(',');
 
 function postProcess(data) {
     data.config = safe.JSON.parse(data.configJson);
-    data.tlsConfig = safe.JSON.parse(data.tlsConfigJson);
     delete data.configJson;
+
+    data.tlsConfig = safe.JSON.parse(data.tlsConfigJson);
     delete data.tlsConfigJson;
 
+    data.wellKnown = safe.JSON.parse(data.wellKnownJson);
+    delete data.wellKnownJson;
+
     return data;
 }
 
@@ -86,6 +90,9 @@ function update(name, domain, callback) {
         } else if (k === 'tlsConfig') {
             fields.push('tlsConfigJson = ?');
             args.push(JSON.stringify(domain[k]));
+        } else if (k === 'wellKnown') {
+            fields.push('wellKnownJson = ?');
+            args.push(JSON.stringify(domain[k]));
         } else {
             fields.push(k + ' = ?');
             args.push(domain[k]);

src/domains.js

@@ -1,37 +1,44 @@
 'use strict';
 
 module.exports = exports = {
-    add: add,
-    get: get,
-    getAll: getAll,
-    update: update,
-    del: del,
-    clear: clear,
+    add,
+    get,
+    getAll,
+    update,
+    del,
+    clear,
 
-    fqdn: fqdn,
-    getName: getName,
+    fqdn,
+    getName,
 
-    getDnsRecords: getDnsRecords,
-    upsertDnsRecords: upsertDnsRecords,
-    removeDnsRecords: removeDnsRecords,
+    getDnsRecords,
+    upsertDnsRecords,
+    removeDnsRecords,
 
-    waitForDnsRecord: waitForDnsRecord,
+    waitForDnsRecord,
 
-    removePrivateFields: removePrivateFields,
-    removeRestrictedFields: removeRestrictedFields,
+    removePrivateFields,
+    removeRestrictedFields,
 
-    validateHostname: validateHostname,
+    validateHostname,
 
-    makeWildcard: makeWildcard,
+    makeWildcard,
 
-    parentDomain: parentDomain,
+    parentDomain,
 
-    checkDnsRecords: checkDnsRecords
+    registerLocations,
+    unregisterLocations,
+
+    checkDnsRecords,
+    syncDnsRecords
 };
 
-var assert = require('assert'),
+const apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
+    crypto = require('crypto'),
     debug = require('debug')('box:domains'),
     domaindb = require('./domaindb.js'),
     eventlog = require('./eventlog.js'),
@@ -60,6 +67,7 @@ function api(provider) {
     case 'linode': return require('./dns/linode.js');
     case 'namecom': return require('./dns/namecom.js');
     case 'namecheap': return require('./dns/namecheap.js');
+    case 'netcup': return require('./dns/netcup.js');
     case 'noop': return require('./dns/noop.js');
     case 'manual': return require('./dns/manual.js');
     case 'wildcard': return require('./dns/wildcard.js');
@@ -152,6 +160,12 @@ function validateTlsConfig(tlsConfig, dnsProvider) {
     return null;
 }
 
+function validateWellKnown(wellKnown) {
+    assert.strictEqual(typeof wellKnown, 'object');
+
+    return null;
+}
+
 function add(domain, data, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof data.zoneName, 'string');
@@ -178,13 +192,17 @@ function add(domain, data, auditSource, callback) {
         if (error) return callback(error);
     } else {
         fallbackCertificate = reverseProxy.generateFallbackCertificateSync({ domain, config });
-        if (fallbackCertificate.error) return callback(error);
+        if (fallbackCertificate.error) return callback(fallbackCertificate.error);
     }
 
     let error = validateTlsConfig(tlsConfig, provider);
     if (error) return callback(error);
 
-    if (!dkimSelector) dkimSelector = 'cloudron-' + settings.adminDomain().replace(/\./g, '');
+    if (!dkimSelector) {
+        // create a unique suffix. this lets one add this domain can be added in another cloudron instance and not have their dkim selector conflict
+        const suffix = crypto.createHash('sha256').update(settings.adminDomain()).digest('hex').substr(0, 6);
+        dkimSelector = `cloudron-${suffix}`;
+    }
 
     verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
         if (error) return callback(error);
@@ -246,7 +264,7 @@ function update(domain, data, auditSource, callback) {
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
+    let { zoneName, provider, config, fallbackCertificate, tlsConfig, wellKnown } = data;
 
     if (settings.isDemo() && (domain === settings.adminDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
 
@@ -267,6 +285,9 @@ function update(domain, data, auditSource, callback) {
         error = validateTlsConfig(tlsConfig, provider);
         if (error) return callback(error);
 
+        error = validateWellKnown(wellKnown, provider);
+        if (error) return callback(error);
+
         if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config);
 
         verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
@@ -274,9 +295,10 @@ function update(domain, data, auditSource, callback) {
 
             let newData = {
                 config: sanitizedConfig,
-                zoneName: zoneName,
-                provider: provider,
-                tlsConfig: tlsConfig
+                zoneName,
+                provider,
+                tlsConfig,
+                wellKnown
             };
 
             domaindb.update(domain, newData, function (error) {
@@ -302,7 +324,7 @@ function del(domain, auditSource, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     if (domain === settings.adminDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove admin domain'));
-    if (domain === settings.mailDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove mail domain'));
+    if (domain === settings.mailDomain()) return callback(new BoxError(BoxError.CONFLICT, 'Cannot remove mail domain. Change the mail server location first'));
 
     domaindb.del(domain, function (error) {
         if (error) return callback(error);
@@ -326,6 +348,7 @@ function clear(callback) {
 }
 
 // returns the 'name' that needs to be inserted into zone
+// eslint-disable-next-line no-unused-vars
 function getName(domain, location, type) {
     const part = domain.domain.slice(0, -domain.zoneName.length - 1);
 
@@ -431,7 +454,7 @@ function waitForDnsRecord(location, domain, type, value, options, callback) {
 
 // removes all fields that are strictly private and should never be returned by API calls
 function removePrivateFields(domain) {
-    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate');
+    var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate', 'wellKnown');
     return api(result.provider).removePrivateFields(result);
 }
 
@@ -444,10 +467,134 @@ function removeRestrictedFields(domain) {
     return result;
 }
 
-function makeWildcard(hostname) {
-    assert.strictEqual(typeof hostname, 'string');
+function makeWildcard(vhost) {
+    assert.strictEqual(typeof vhost, 'string');
 
-    let parts = hostname.split('.');
+    // if the vhost is like *.example.com, this function will do nothing
+    let parts = vhost.split('.');
     parts[0] = '*';
     return parts.join('.');
 }
+
+function registerLocations(locations, options, progressCallback, callback) {
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
+
+    const overwriteDns = options.overwriteDns || false;
+
+    sysinfo.getServerIp(function (error, ip) {
+        if (error) return callback(error);
+
+        async.eachSeries(locations, function (location, iteratorDone) {
+            async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
+                progressCallback({ message: `Registering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+
+                // get the current record before updating it
+                getDnsRecords(location.subdomain, location.domain, 'A', function (error, values) {
+                    if (error && error.reason === BoxError.EXTERNAL_ERROR) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location })); // try again
+                    if (error && error.reason === BoxError.ACCESS_DENIED) return retryCallback(null, new BoxError(BoxError.ACCESS_DENIED, error.message, { domain: location }));
+                    if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, new BoxError(BoxError.NOT_FOUND, error.message, { domain: location }));
+                    if (error) return retryCallback(null, new BoxError(BoxError.EXTERNAL_ERROR, error.message, location)); // give up for other errors
+
+                    if (values.length !== 0 && values[0] === ip) return retryCallback(null); // up-to-date
+
+                    // refuse to update any existing DNS record for custom domains that we did not create
+                    if (values.length !== 0 && !overwriteDns) return retryCallback(null, new BoxError(BoxError.ALREADY_EXISTS, 'DNS Record already exists', { domain: location }));
+
+                    upsertDnsRecords(location.subdomain, location.domain, 'A', [ ip ], function (error) {
+                        if (error && (error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
+                            progressCallback({ message: `registerSubdomains: Upsert error. Will retry. ${error.message}` });
+                            return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location })); // try again
+                        }
+
+                        retryCallback(null, error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, location) : null);
+                    });
+                });
+            }, function (error, result) {
+                if (error || result) return iteratorDone(error || result);
+
+                iteratorDone(null);
+            });
+        }, callback);
+    });
+}
+
+function unregisterLocations(locations, progressCallback, callback) {
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    sysinfo.getServerIp(function (error, ip) {
+        if (error) return callback(error);
+
+        async.eachSeries(locations, function (location, iteratorDone) {
+            async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
+                progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+
+                removeDnsRecords(location.subdomain, location.domain, 'A', [ ip ], function (error) {
+                    if (error && error.reason === BoxError.NOT_FOUND) return retryCallback(null, null);
+                    if (error && (error.reason === BoxError.SBUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
+                        progressCallback({ message: `Error unregistering location. Will retry. ${error.message}`});
+                        return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location })); // try again
+                    }
+
+                    retryCallback(null, error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }) : null);
+                });
+            }, function (error, result) {
+                if (error || result) return iteratorDone(error || result);
+
+                iteratorDone();
+            });
+        }, callback);
+    });
+}
+
+function syncDnsRecords(options, progressCallback, callback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (options.domain && options.type === 'mail') return mail.setDnsRecords(options.domain, callback);
+
+    getAll(function (error, domains) {
+        if (error) return callback(error);
+
+        if (options.domain) domains = domains.filter(d => d.domain === options.domain);
+
+        const mailSubdomain = settings.mailFqdn().substr(0, settings.mailFqdn().length - settings.mailDomain().length - 1);
+
+        apps.getAll(function (error, allApps) {
+            if (error) return callback(error);
+
+            let progress = 1, errors = [];
+
+            // we sync by domain only to get some nice progress
+            async.eachSeries(domains, function (domain, iteratorDone) {
+                progressCallback({ percent: progress, message: `Updating DNS of ${domain.domain}`});
+                progress += Math.round(100/(1+domains.length));
+
+                let locations = [];
+                if (domain.domain === settings.adminDomain()) locations.push({ subdomain: constants.ADMIN_LOCATION, domain: settings.adminDomain() });
+                if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.adminFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
+
+                allApps.forEach(function (app) {
+                    const appLocations = [{ subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains);
+                    locations = locations.concat(appLocations.filter(al => al.domain === domain.domain));
+                });
+
+                async.series([
+                    registerLocations.bind(null, locations, { overwriteDns: true }, progressCallback),
+                    progressCallback.bind(null, { message: `Updating mail DNS of ${domain.domain}`}),
+                    mail.setDnsRecords.bind(null, domain.domain)
+                ], function (error) {
+                    if (error) errors.push({ domain: domain.domain, message: error.message });
+                    iteratorDone();
+                });
+            }, () => callback(null, { errors }));
+        });
+    });
+}

src/eventlog.js

@@ -1,11 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    add: add,
-    get: get,
-    getAllPaged: getAllPaged,
-    getByCreationTime: getByCreationTime,
-    cleanup: cleanup,
+    add,
+    upsert,
+    get,
+    getAllPaged,
+    getByCreationTime,
+    cleanup,
 
     // keep in sync with webadmin index.js filter
     ACTION_ACTIVATE: 'cloudron.activate',
@@ -57,6 +58,7 @@ exports = module.exports = {
 
     ACTION_USER_ADD: 'user.add',
     ACTION_USER_LOGIN: 'user.login',
+    ACTION_USER_LOGOUT: 'user.logout',
     ACTION_USER_REMOVE: 'user.remove',
     ACTION_USER_UPDATE: 'user.update',
     ACTION_USER_TRANSFER: 'user.transfer',
@@ -90,9 +92,24 @@ function add(action, source, data, callback) {
 
     callback = callback || NOOP_CALLBACK;
 
-    // we do only daily upserts for login actions, so they don't spam the db
-    var api = action === exports.ACTION_USER_LOGIN ? eventlogdb.upsert : eventlogdb.add;
-    api(uuid.v4(), action, source, data, function (error, id) {
+    eventlogdb.add(uuid.v4(), action, source, data, function (error, id) {
+        if (error) return callback(error);
+
+        callback(null, { id: id });
+
+        notifications.onEvent(id, action, source, data, NOOP_CALLBACK);
+    });
+}
+
+function upsert(action, source, data, callback) {
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert(!callback || typeof callback === 'function');
+
+    callback = callback || NOOP_CALLBACK;
+
+    eventlogdb.upsert(uuid.v4(), action, source, data, function (error, id) {
         if (error) return callback(error);
 
         callback(null, { id: id });

src/graphs.js

@@ -1,47 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    startGraphite: startGraphite
-};
-
-var assert = require('assert'),
-    async = require('async'),
-    infra = require('./infra_version.js'),
-    paths = require('./paths.js'),
-    shell = require('./shell.js');
-
-function startGraphite(existingInfra, callback) {
-    assert.strictEqual(typeof existingInfra, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const tag = infra.images.graphite.tag;
-    const dataDir = paths.PLATFORM_DATA_DIR;
-    const memoryLimit = 256;
-
-    if (existingInfra.version === infra.version && infra.images.graphite.tag === existingInfra.images.graphite.tag) return callback();
-
-    const cmd = `docker run --restart=always -d --name="graphite" \
-                --hostname graphite \
-                --net cloudron \
-                --net-alias graphite \
-                --log-driver syslog \
-                --log-opt syslog-address=udp://127.0.0.1:2514 \
-                --log-opt syslog-format=rfc5424 \
-                --log-opt tag=graphite \
-                -m ${memoryLimit}m \
-                --memory-swap ${memoryLimit * 2}m \
-                --dns 172.18.0.1 \
-                --dns-search=. \
-                -p 127.0.0.1:2003:2003 \
-                -p 127.0.0.1:2004:2004 \
-                -p 127.0.0.1:8417:8000 \
-                -v "${dataDir}/graphite:/var/lib/graphite" \
-                --label isCloudronManaged=true \
-                --read-only -v /tmp -v /run "${tag}"`;
-
-    async.series([
-        shell.exec.bind(null, 'stopGraphite', 'docker stop graphite || true'),
-        shell.exec.bind(null, 'removeGraphite', 'docker rm -f graphite || true'),
-        shell.exec.bind(null, 'startGraphite', cmd)
-    ], callback);
-}

src/groupdb.js

@@ -196,6 +196,7 @@ function setMembers(groupId, userIds, callback) {
 
     database.transaction(queries, function (error) {
         if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.CONFLICT, 'Duplicate member in list'));
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
         callback(error);
@@ -227,6 +228,7 @@ function setMembership(userId, groupIds, callback) {
 
     database.transaction(queries, function (error) {
         if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, error.message));
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.CONFLICT, 'Already member'));
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
         callback(null);

src/groups.js

@@ -1,25 +1,25 @@
 'use strict';
 
 exports = module.exports = {
-    create: create,
-    remove: remove,
-    get: get,
-    getByName: getByName,
-    update: update,
-    getWithMembers: getWithMembers,
-    getAll: getAll,
-    getAllWithMembers: getAllWithMembers,
+    create,
+    remove,
+    get,
+    getByName,
+    update,
+    getWithMembers,
+    getAll,
+    getAllWithMembers,
 
-    getMembers: getMembers,
-    addMember: addMember,
-    setMembers: setMembers,
-    removeMember: removeMember,
-    isMember: isMember,
+    getMembers,
+    addMember,
+    setMembers,
+    removeMember,
+    isMember,
 
-    setMembership: setMembership,
-    getMembership: getMembership,
+    setMembership,
+    getMembership,
 
-    count: count
+    count
 };
 
 var assert = require('assert'),

src/infra_version.js

@@ -9,19 +9,19 @@ exports = module.exports = {
     'version': '48.18.0',
 
     'baseImages': [
-        { repo: 'cloudron/base', tag: 'cloudron/base:2.0.0@sha256:f9fea80513aa7c92fe2e7bf3978b54c8ac5222f47a9a32a7f8833edf0eb5a4f4' }
+        { repo: 'cloudron/base', tag: 'cloudron/base:3.0.0@sha256:455c70428723e3a823198c57472785437eb6eab082e79b3ff04ea584faf46e92' }
     ],
 
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.1.0@sha256:e1dd22aa6eef5beb7339834b200a8bb787ffc2264ce11139857a054108fefb4f' },
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.3.2@sha256:dd624870c7f8ba9b2759f93ce740d1e092a1ac4b2d6af5007a01b30ad6b316d0' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:3.3.0@sha256:0daf1be5320c095077392bf21d247b93ceaddca46c866c17259a335c80d2f357' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:3.0.0@sha256:59e50b1f55e433ffdf6d678f8c658812b4119f631db8325572a52ee40d3bc562' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.3.0@sha256:0e31ec817e235b1814c04af97b1e7cf0053384aca2569570ce92bef0d95e94d2' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.0.1@sha256:ff24c70966937e8c3477d534bbb192e0364d3e9d6924ee0911278009d802b2b0' },
-        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.3.0@sha256:b7bc1ca4f4d0603a01369a689129aa273a938ce195fe43d00d42f4f2d5212f50' },
-        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.0.0@sha256:7e0165f17789192fd4f92efb34aa373450fa859e3b502684b2b121a5582965bf' }
+        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.3.0@sha256:386fb755fc41edd7086f7bcb230f7f28078936f9ae4ead6d97c741df1cc194ae' },
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:3.0.4@sha256:4d688c746f27b195d98f35a7d24ec01f3f754e0ca61e9de0b0bc9793553880f1' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:4.0.2@sha256:424081fd38ebd35f3606c64f8f99138570e5f4d5066f12cfb4142447d249d3e7' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:4.0.1@sha256:ad20a9a5dcb2ab132374a7c8d44b89af0ec37651cf889e570f7625b02ee85fdf' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:3.0.2@sha256:caaa1f7f4055ae8990d8ec65bd100567496df7e4ed5eb427867f3717a8dcbf92' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:3.2.3@sha256:fdc4aa6d2c85aeafe65eaa4243aada0cc2e57b94f6eaee02c9b1a8fb89b01dd7' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:3.0.0@sha256:b00b64b8df4032985d7a1ddd548a2713b6d7d88a54ebe9b7d324cece2bd6829e' },
+        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:3.2.0@sha256:61e8247ded1e07cf882ca478dab180960357c614472e80b938f1f690a46788c2' }
     }
 };

src/ldap.js

@@ -1,12 +1,11 @@
 'use strict';
 
 exports = module.exports = {
-    start: start,
-    stop: stop
+    start,
+    stop
 };
 
-var addons = require('./addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     async = require('async'),
@@ -20,6 +19,7 @@ var addons = require('./addons.js'),
     mailboxdb = require('./mailboxdb.js'),
     path = require('path'),
     safe = require('safetydance'),
+    services = require('./services.js'),
     users = require('./users.js');
 
 var gServer = null;
@@ -288,14 +288,14 @@ function mailboxSearch(req, res, next) {
     } else if (req.dn.rdns[0].attrs.domain) { // legacy ldap mailbox search for old sogo
         var domain = req.dn.rdns[0].attrs.domain.value.toLowerCase();
 
-        mailboxdb.listMailboxes(domain, 1, 1000, function (error, result) {
+        mailboxdb.listMailboxes(domain, 1, 1000, function (error, mailboxes) {
 
             if (error) return next(new ldap.OperationsError(error.toString()));
 
             var results = [];
 
             // send mailbox objects
-            result.forEach(function (mailbox) {
+            mailboxes.forEach(function (mailbox) {
                 var dn = ldap.parseDN(`cn=${mailbox.name}@${domain},domain=${domain},ou=mailboxes,dc=cloudron`);
 
                 var obj = {
@@ -346,23 +346,19 @@ function mailboxSearch(req, res, next) {
                         }
                     };
 
-                    mailboxdb.getAliasesForName(mailbox.name, mailbox.domain, function (error, aliases) {
-                        if (error) return callback(error);
-
-                        aliases.forEach(function (a, idx) {
-                            obj.attributes['mail' + idx] = `${a.name}@${a.domain}`;
-                        });
-
-                        // ensure all filter values are also lowercase
-                        var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
-                        if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
-
-                        if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                            results.push(obj);
-                        }
-
-                        callback();
+                    mailbox.aliases.forEach(function (a, idx) {
+                        obj.attributes['mail' + idx] = `${a.name}@${a.domain}`;
                     });
+
+                    // ensure all filter values are also lowercase
+                    var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
+                    if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
+
+                    if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                        results.push(obj);
+                    }
+
+                    callback();
                 });
             }, function (error) {
                 if (error) return next(new ldap.OperationsError(error.toString()));
@@ -492,7 +488,7 @@ function authorizeUserForApp(req, res, next) {
         // we return no such object, to avoid leakage of a users existence
         if (!hasAccess) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: req.app.id }, { userId: req.user.id, user: users.removePrivateFields(req.user) });
+        eventlog.upsert(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: req.app.id }, { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
         res.end();
     });
@@ -546,7 +542,7 @@ function authenticateUserMailbox(req, res, next) {
                 if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                 if (error) return next(new ldap.OperationsError(error.message));
 
-                eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
+                eventlog.upsert(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
                 res.end();
             });
         });
@@ -576,10 +572,10 @@ function authenticateSftp(req, res, next) {
 }
 
 function loadSftpConfig(req, res, next) {
-    addons.getServicesConfig('sftp', function (error, service, servicesConfig) {
+    services.getServiceConfig('sftp', function (error, serviceConfig) {
         if (error) return next(new ldap.OperationsError(error.toString()));
 
-        req.requireAdmin = servicesConfig['sftp'].requireAdmin;
+        req.requireAdmin = serviceConfig.requireAdmin;
 
         next();
     });
@@ -686,7 +682,7 @@ function authenticateMailAddon(req, res, next) {
                     if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                     if (error) return next(new ldap.OperationsError(error.message));
 
-                    eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
+                    eventlog.upsert(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: email }, { userId: result.id, user: users.removePrivateFields(result) });
                     res.end();
                 });
             });
@@ -744,6 +740,12 @@ function start(callback) {
         res.end();
     });
 
+    // just log that an attempt was made to unknown route, this helps a lot during app packaging
+    gServer.use(function(req, res, next) {
+        debug('not handled: dn %s, scope %s, filter %s (from %s)', req.dn ? req.dn.toString() : '-', req.scope, req.filter ? req.filter.toString() : '-', req.connection.ldap.id);
+        return next();
+    });
+
     gServer.listen(constants.LDAP_PORT, '0.0.0.0', callback);
 }
 

src/mail.js

@@ -55,12 +55,13 @@ exports = module.exports = {
     OWNERTYPE_USER: 'user',
     OWNERTYPE_GROUP: 'group',
 
+    DEFAULT_MEMORY_LIMIT: 512 * 1024 * 1024,
+
     _removeMailboxes: removeMailboxes,
     _readDkimPublicKeySync: readDkimPublicKeySync
 };
 
-var addons = require('./addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
     cloudron = require('./cloudron.js'),
@@ -82,10 +83,12 @@ var addons = require('./addons.js'),
     request = require('request'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
+    services = require('./services.js'),
     settings = require('./settings.js'),
     shell = require('./shell.js'),
     smtpTransport = require('nodemailer-smtp-transport'),
     sysinfo = require('./sysinfo.js'),
+    system = require('./system.js'),
     tasks = require('./tasks.js'),
     users = require('./users.js'),
     validator = require('validator'),
@@ -624,9 +627,10 @@ function createMailConfig(mailFqdn, mailDomain, callback) {
     });
 }
 
-function configureMail(mailFqdn, mailDomain, callback) {
+function configureMail(mailFqdn, mailDomain, serviceConfig, callback) {
     assert.strictEqual(typeof mailFqdn, 'string');
     assert.strictEqual(typeof mailDomain, 'string');
+    assert.strictEqual(typeof serviceConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     // mail (note: 2525 is hardcoded in mail container and app use this port)
@@ -635,7 +639,8 @@ function configureMail(mailFqdn, mailDomain, callback) {
     // mail container uses /app/data for backed up data and /run for restart-able data
 
     const tag = infra.images.mail.tag;
-    const memoryLimit = 4 * 256;
+    const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT;
+    const memory = system.getMemoryAllocation(memoryLimit);
     const cloudronToken = hat(8 * 128), relayToken = hat(8 * 128);
 
     reverseProxy.getCertificate(mailFqdn, mailDomain, function (error, bundle) {
@@ -666,8 +671,8 @@ function configureMail(mailFqdn, mailDomain, callback) {
                             --log-opt syslog-address=udp://127.0.0.1:2514 \
                             --log-opt syslog-format=rfc5424 \
                             --log-opt tag=mail \
-                            -m ${memoryLimit}m \
-                            --memory-swap ${memoryLimit * 2}m \
+                            -m ${memory} \
+                            --memory-swap ${memoryLimit} \
                             --dns 172.18.0.1 \
                             --dns-search=. \
                             -e CLOUDRON_MAIL_TOKEN="${cloudronToken}" \
@@ -714,8 +719,12 @@ function restartMail(callback) {
 
     if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return callback();
 
-    debug(`restartMail: restarting mail container with ${settings.mailFqdn()} ${settings.adminDomain()}`);
-    configureMail(settings.mailFqdn(), settings.adminDomain(), callback);
+    services.getServiceConfig('mail', function (error, serviceConfig) {
+        if (error) return callback(error);
+
+        debug(`restartMail: restarting mail container with ${settings.mailFqdn()} ${settings.adminDomain()}`);
+        configureMail(settings.mailFqdn(), settings.adminDomain(), serviceConfig, callback);
+    });
 }
 
 function restartMailIfActivated(callback) {
@@ -866,37 +875,40 @@ function upsertDnsRecords(domain, mailFqdn, callback) {
 
         if (process.env.BOX_ENV === 'test') return callback();
 
-        var dkimKey = readDkimPublicKeySync(domain);
+        const dkimKey = readDkimPublicKeySync(domain);
         if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, 'Failed to read dkim public key'));
 
         // t=s limits the domainkey to this domain and not it's subdomains
-        var dkimRecord = { subdomain: `${mailDomain.dkimSelector}._domainkey`, domain: domain, type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
+        const dkimRecord = { subdomain: `${mailDomain.dkimSelector}._domainkey`, domain: domain, type: 'TXT', values: [ `"v=DKIM1; t=s; p=${dkimKey}"` ] };
 
-        var records = [ ];
+        let records = [];
         records.push(dkimRecord);
-        if (mailDomain.enabled) {
-            records.push({ subdomain: '_dmarc', domain: domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] });
-            records.push({ subdomain: '', domain: domain, type: 'MX', values: [ '10 ' + mailFqdn + '.' ] });
-        }
+        if (mailDomain.enabled) records.push({ subdomain: '', domain: domain, type: 'MX', values: [ '10 ' + mailFqdn + '.' ] });
 
         txtRecordsWithSpf(domain, mailFqdn, function (error, txtRecords) {
             if (error) return callback(error);
 
             if (txtRecords) records.push({ subdomain: '', domain: domain, type: 'TXT', values: txtRecords });
 
-            debug('upsertDnsRecords: will update %j', records);
+            domains.getDnsRecords('_dmarc', domain, 'TXT', function (error, dmarcRecords) { // only update dmarc if absent. this allows user to set email for reporting
+                if (error) return callback(error);
 
-            async.mapSeries(records, function (record, iteratorCallback) {
-                domains.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values, iteratorCallback);
-            }, function (error, changeIds) {
-                if (error) {
-                    debug(`upsertDnsRecords: failed to update: ${error}`);
-                    return callback(error);
-                }
+                if (dmarcRecords.length === 0) records.push({ subdomain: '_dmarc', domain: domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] });
 
-                debug('upsertDnsRecords: records %j added with changeIds %j', records, changeIds);
+                debug('upsertDnsRecords: will update %j', records);
 
-                callback(null);
+                async.mapSeries(records, function (record, iteratorCallback) {
+                    domains.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values, iteratorCallback);
+                }, function (error, changeIds) {
+                    if (error) {
+                        debug(`upsertDnsRecords: failed to update: ${error}`);
+                        return callback(error);
+                    }
+
+                    debug('upsertDnsRecords: records %j added with changeIds %j', records, changeIds);
+
+                    callback(null);
+                });
             });
         });
     });
@@ -1223,7 +1235,7 @@ function removeSolrIndex(mailbox, callback) {
     assert.strictEqual(typeof mailbox, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    addons.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
+    services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
         if (error) return callback(error);
 
         request.post(`https://${addonDetails.ip}:3000/solr_delete_index?access_token=${addonDetails.token}`, { timeout: 2000, rejectUnauthorized: false, json: { mailbox } }, function (error, response) {
@@ -1412,7 +1424,7 @@ function resolveList(listName, listDomain, callback) {
 
             let result = [], toResolve = list.members.slice(), visited = []; // slice creates a copy of array
 
-            async.whilst(() => toResolve.length != 0, function (iteratorCallback) {
+            async.whilst((testDone) => testDone(null, toResolve.length != 0), function (iteratorCallback) {
                 const toProcess = toResolve.shift();
                 const parts = toProcess.split('@');
                 const memberName = parts[0].split('+')[0], memberDomain = parts[1];

src/mail_templates/app_updates_available.ejs

@@ -3,18 +3,14 @@
 Dear Cloudron Admin,
 
 <% for (var i = 0; i < apps.length; i++) { -%>
-A new version <%= apps[i].updateInfo.manifest.version %> of the app '<%= apps[i].app.manifest.title %>' installed at <%= apps[i].app.fqdn %> is available.
+The app '<%= apps[i].app.manifest.title %>' installed at <%= apps[i].app.fqdn %> has an update available.
 
-Changes:
+<%= apps[i].app.manifest.title %> v<%= apps[i].updateInfo.manifest.version %> changes:
 <%= apps[i].updateInfo.manifest.changelog %>
 
 <% } -%>
 
-<% if (!hasSubscription) { -%>
-*Keep your Cloudron automatically up-to-date and secure by upgrading to a paid plan at* <%= webadminUrl %>/#/settings
-<% } else { -%>
 Update now at <%= webadminUrl %>
-<% } -%>
 
 Powered by https://cloudron.io
 
@@ -33,24 +29,20 @@ Sent at: <%= new Date().toUTCString() %>
     <div style="width: 650px; text-align: left;">
 <% for (var i = 0; i < apps.length; i++) { -%>
         <p>
-            A new version <%= apps[i].updateInfo.manifest.version %> of the app '<%= apps[i].app.manifest.title %>' installed at <a href="https://<%= apps[i].app.fqdn %>"><%= apps[i].app.fqdn %></a> is available.
+            The app '<%= apps[i].app.manifest.title %>' installed at <a href="https://<%= apps[i].app.fqdn %>"><%= apps[i].app.fqdn %></a> has an update available.
         </p>
 
-        <h5>Changelog:</h5>
+        <h5><%= apps[i].app.manifest.title %> v<%= apps[i].updateInfo.manifest.version %> changes:</h5>
         <%- apps[i].changelogHTML %>
 
         <br/>
 <% } -%>
 
-<% if (!hasSubscription) { -%>
-        <p>Keep your Cloudron automatically up-to-date and secure by upgrading to a <a href="<%= webadminUrl %>/#/settings">paid plan</a>.</p>
-<% } else { -%>
         <p>
             <br/>
             <center><a href="<%= webadminUrl %>">Update now</a></center>
             <br/>
         </p>
-<% } -%>
     </div>
 
     <div style="font-size: 10px; color: #333333; background: #ffffff;">

src/mail_templates/box_update_available.ejs

@@ -0,0 +1,45 @@
+<%if (format === 'text') { %>
+
+Dear <%= cloudronName %> Admin,
+
+Cloudron v<%= newBoxVersion %> is now available!
+
+Changes:
+<% for (var i = 0; i < changelog.length; i++) { %>
+  * <%- changelog[i] %>
+<% } %>
+
+Powered by https://cloudron.io
+
+Sent at: <%= new Date().toUTCString() %>
+
+<% } else { %>
+
+<center>
+
+<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
+
+<h3>Dear <%= cloudronName %> Admin,</h3>
+
+<div style="width: 650px; text-align: left;">
+    <p>
+        Cloudron v<%= newBoxVersion %> is now available!
+    </p>
+
+    <h5>Changes:</h5>
+    <ul>
+    <% for (var i = 0; i < changelogHTML.length; i++) { %>
+        <li><%- changelogHTML[i] %></li>
+    <% } %>
+    </ul>
+
+    <br/>
+</div>
+
+<div style="font-size: 10px; color: #333333; background: #ffffff;">
+    Powered by <a href="https://cloudron.io">Cloudron</a>.
+</div>
+
+</center>
+
+<% } %>

src/mail_templates/oom_event.ejs

@@ -2,12 +2,13 @@
 
 Dear <%= cloudronName %> Admin,
 
-<%= program %> was restarted now as it ran out of memory.
-
-If this message appears repeatedly, give the app more memory.
-
-* To increase an app's memory limit - https://docs.cloudron.io/apps/#memory-limit
-* To increase a service's memory limit - https://docs.cloudron.io/troubleshooting/#services
+<%if (app) { %>
+The application at <%= app.fqdn %> ran out of memory. The application has been restarted automatically. If you see this notification often,
+consider increasing the memory limit - <%= webadminUrl %>/#/app/<%= app.id %>/resources .
+<% } else { %>
+The addon <%= addon.name %> service ran out of memory. The service has been restarted automatically. If you see this notification often,
+consider increasing the memory limit - <%= webadminUrl %>/#/services .
+<% } %>
 
 Out of memory event:
 

src/mail_templates/user_added.ejs

@@ -1,30 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear <%= cloudronName %> Admin,
-
-A new user with email <%= user.email %> was added to <%= cloudronName %>.
-
-Powered by https://cloudron.io
-
-<% } else { %>
-
-<center>
-
-<img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
-
-<h3>Dear <%= cloudronName %> Admin,</h3>
-
-<p>
-    A new user with email <%= user.email %> was added to <%= cloudronName %>.
-</p>
-
-<br/>
-<br/>
-
-<div style="font-size: 10px; color: #333333; background: #ffffff;">
-    Powered by <a href="https://cloudron.io">Cloudron</a>.
-</div>
-
-</center>
-
-<% } %>

src/mail_templates/user_event.ejs

@@ -1,14 +0,0 @@
-<%if (format === 'text') { %>
-
-Dear Cloudron Admin,
-
-User <%= user.username || user.email %> <%= event %>.
-
-
-Powered by https://cloudron.io
-
-Sent at: <%= new Date().toUTCString() %>
-
-<% } else { %>
-
-<% } %>

src/mailboxdb.js

@@ -53,6 +53,16 @@ function postProcess(data) {
     return data;
 }
 
+function postProcessAliases(data) {
+    const aliasNames = JSON.parse(data.aliasNames), aliasDomains = JSON.parse(data.aliasDomains);
+    delete data.aliasNames;
+    delete data.aliasDomains;
+    data.aliases = [];
+    for (let i = 0; i < aliasNames.length; i++) { // NOTE: aliasNames is [ null ] when no aliases
+        if (aliasNames[i]) data.aliases[i] = { name: aliasNames[i], domain: aliasDomains[i] };
+    }
+}
+
 function addMailbox(name, domain, ownerId, ownerType, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
@@ -225,14 +235,22 @@ function listMailboxes(domain, search, page, perPage, callback) {
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    let query = `SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? AND domain = ?`;
-    if (search) query += ' AND (name LIKE ' + mysql.escape('%' + search + '%') + ')';
-    query += 'ORDER BY name LIMIT ?,?';
+    const escapedSearch = mysql.escape('%' + search + '%'); // this also quotes the string
+    const searchQuery = search ? ` HAVING (name LIKE ${escapedSearch} OR aliasNames LIKE ${escapedSearch} OR aliasDomains LIKE ${escapedSearch})` : ''; // having instead of where because of aggregated columns use
 
-    database.query(query, [ exports.TYPE_MAILBOX, domain, (page-1)*perPage, perPage ], function (error, results) {
+    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains '
+        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
+        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
+        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
+        + ' WHERE m1.domain = ?'
+        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
+        + searchQuery
+        + ' ORDER BY name LIMIT ?,?';
+
+    database.query(query, [ domain, (page-1)*perPage, perPage ], function (error, results) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
-        results.forEach(function (result) { postProcess(result); });
+        results.forEach(postProcessAliases);
 
         callback(null, results);
     });
@@ -243,14 +261,20 @@ function listAllMailboxes(page, perPage, callback) {
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? ORDER BY name LIMIT ?,?`,
-        [ exports.TYPE_MAILBOX, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains '
+        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
+        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
+        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
+        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
+        + ' ORDER BY name LIMIT ?,?';
 
-            results.forEach(function (result) { postProcess(result); });
+    database.query(query, [ (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
-            callback(null, results);
-        });
+        results.forEach(postProcessAliases);
+
+        callback(null, results);
+    });
 }
 
 function getLists(domain, search, page, perPage, callback) {

src/mailer.js

@@ -1,25 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    userAdded: userAdded,
-    userRemoved: userRemoved,
-    roleChanged: roleChanged,
-    passwordReset: passwordReset,
-    appUpdatesAvailable: appUpdatesAvailable,
+    passwordReset,
+    boxUpdateAvailable,
+    appUpdatesAvailable,
 
-    sendInvite: sendInvite,
+    sendInvite,
 
-    appUp: appUp,
-    appDied: appDied,
-    appUpdated: appUpdated,
-    oomEvent: oomEvent,
+    appUp,
+    appDied,
+    appUpdated,
+    oomEvent,
 
-    backupFailed: backupFailed,
+    backupFailed,
 
-    certificateRenewalError: certificateRenewalError,
-    boxUpdateError: boxUpdateError,
+    certificateRenewalError,
+    boxUpdateError,
 
-    sendTestMail: sendTestMail,
+    sendTestMail,
 
     _mailQueue: [] // accumulate mails in test mode
 };
@@ -35,8 +33,7 @@ var assert = require('assert'),
     settings = require('./settings.js'),
     showdown = require('showdown'),
     translation = require('./translation.js'),
-    smtpTransport = require('nodemailer-smtp-transport'),
-    util = require('util');
+    smtpTransport = require('nodemailer-smtp-transport');
 
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
@@ -114,25 +111,6 @@ function render(templateFile, params, translationAssets) {
     return content;
 }
 
-function mailUserEvent(mailTo, user, event) {
-    assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof event, 'string');
-
-    getMailConfig(function (error, mailConfig) {
-        if (error) return debug('Error getting mail details:', error);
-
-        var mailOptions = {
-            from: mailConfig.notificationFrom,
-            to: mailTo,
-            subject: util.format('[%s] %s %s', mailConfig.cloudronName, user.username || user.fallbackEmail || user.email, event),
-            text: render('user_event.ejs', { user: user, event: event, format: 'text' }),
-        };
-
-        sendMail(mailOptions);
-    });
-}
-
 function sendInvite(user, invitor, inviteLink) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof invitor, 'object');
@@ -168,57 +146,6 @@ function sendInvite(user, invitor, inviteLink) {
     });
 }
 
-function userAdded(mailTo, user) {
-    assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof user, 'object');
-
-    debug(`userAdded: Sending mail for added users ${user.fallbackEmail} to ${mailTo}`);
-
-    getMailConfig(function (error, mailConfig) {
-        if (error) return debug('Error getting mail details:', error);
-
-        var templateData = {
-            user: user,
-            cloudronName: mailConfig.cloudronName,
-            cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
-        };
-
-        var templateDataText = JSON.parse(JSON.stringify(templateData));
-        templateDataText.format = 'text';
-
-        var templateDataHTML = JSON.parse(JSON.stringify(templateData));
-        templateDataHTML.format = 'html';
-
-        var mailOptions = {
-            from: mailConfig.notificationFrom,
-            to: mailTo,
-            subject: util.format('[%s] User %s added', mailConfig.cloudronName, user.fallbackEmail),
-            text: render('user_added.ejs', templateDataText),
-            html: render('user_added.ejs', templateDataHTML)
-        };
-
-        sendMail(mailOptions);
-    });
-}
-
-function userRemoved(mailTo, user) {
-    assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof user, 'object');
-
-    debug('Sending mail for userRemoved.', user.id, user.username, user.email);
-
-    mailUserEvent(mailTo, user, 'was removed');
-}
-
-function roleChanged(mailTo, user) {
-    assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof user, 'object');
-
-    debug('Sending mail for roleChanged');
-
-    mailUserEvent(mailTo, user, `now has the role '${user.role}'`);
-}
-
 function passwordReset(user) {
     assert.strictEqual(typeof user, 'object');
 
@@ -262,7 +189,7 @@ function appUp(mailTo, app) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] App %s is back online', mailConfig.cloudronName, app.fqdn),
+            subject: `[${mailConfig.cloudronName}] App ${app.fqdn} is back online`,
             text: render('app_up.ejs', { title: app.manifest.title, appFqdn: app.fqdn, format: 'text' })
         };
 
@@ -282,7 +209,7 @@ function appDied(mailTo, app) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] App %s is down', mailConfig.cloudronName, app.fqdn),
+            subject: `[${mailConfig.cloudronName}] App ${app.fqdn} is down`,
             text: render('app_down.ejs', { title: app.manifest.title, appFqdn: app.fqdn, supportEmail: mailConfig.supportEmail, format: 'text' })
         };
 
@@ -329,10 +256,46 @@ function appUpdated(mailTo, app, callback) {
     });
 }
 
-function appUpdatesAvailable(mailTo, apps, hasSubscription, callback) {
+function boxUpdateAvailable(mailTo, updateInfo, callback) {
+    assert.strictEqual(typeof mailTo, 'string');
+    assert.strictEqual(typeof updateInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    getMailConfig(function (error, mailConfig) {
+        if (error) return debug('Error getting mail details:', error);
+
+        var converter = new showdown.Converter();
+
+        var templateData = {
+            webadminUrl: settings.adminOrigin(),
+            newBoxVersion: updateInfo.version,
+            changelog: updateInfo.changelog,
+            changelogHTML: updateInfo.changelog.map(function (e) { return converter.makeHtml(e); }),
+            cloudronName: mailConfig.cloudronName,
+            cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
+        };
+
+        var templateDataText = JSON.parse(JSON.stringify(templateData));
+        templateDataText.format = 'text';
+
+        var templateDataHTML = JSON.parse(JSON.stringify(templateData));
+        templateDataHTML.format = 'html';
+
+        var mailOptions = {
+            from: mailConfig.notificationFrom,
+            to: mailTo,
+            subject: `[${mailConfig.cloudronName}] Cloudron update available`,
+            text: render('box_update_available.ejs', templateDataText),
+            html: render('box_update_available.ejs', templateDataHTML)
+        };
+
+        sendMail(mailOptions, callback);
+    });
+}
+
+function appUpdatesAvailable(mailTo, apps, callback) {
     assert.strictEqual(typeof mailTo, 'string');
     assert.strictEqual(typeof apps, 'object');
-    assert.strictEqual(typeof hasSubscription, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
     getMailConfig(function (error, mailConfig) {
@@ -345,7 +308,6 @@ function appUpdatesAvailable(mailTo, apps, hasSubscription, callback) {
 
         var templateData = {
             webadminUrl: settings.adminOrigin(),
-            hasSubscription: hasSubscription,
             apps: apps,
             cloudronName: mailConfig.cloudronName,
             cloudronAvatarUrl: settings.adminOrigin() + '/api/v1/cloudron/avatar'
@@ -360,7 +322,7 @@ function appUpdatesAvailable(mailTo, apps, hasSubscription, callback) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: `New app updates available for ${mailConfig.cloudronName}`,
+            subject: `[${mailConfig.cloudronName}] App update available`,
             text: render('app_updates_available.ejs', templateDataText),
             html: render('app_updates_available.ejs', templateDataHTML)
         };
@@ -378,7 +340,7 @@ function backupFailed(mailTo, errorMessage, logUrl) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] Failed to backup', mailConfig.cloudronName),
+            subject: `[${mailConfig.cloudronName}] Failed to backup`,
             text: render('backup_failed.ejs', { cloudronName: mailConfig.cloudronName, message: errorMessage, logUrl: logUrl, format: 'text' })
         };
 
@@ -397,7 +359,7 @@ function certificateRenewalError(mailTo, domain, message) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] Certificate renewal error', domain),
+            subject: `[${mailConfig.cloudronName}] Certificate renewal error`,
             text: render('certificate_renewal_error.ejs', { domain: domain, message: message, format: 'text' })
         };
 
@@ -415,7 +377,7 @@ function boxUpdateError(mailTo, message) {
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] Cloudron update error', mailConfig.cloudronName),
+            subject: `[${mailConfig.cloudronName}] Cloudron update error`,
             text: render('box_update_error.ejs', { message: message, format: 'text' })
         };
 
@@ -423,19 +385,30 @@ function boxUpdateError(mailTo, message) {
     });
 }
 
-function oomEvent(mailTo, program, event) {
+function oomEvent(mailTo, app, addon, containerId, event) {
     assert.strictEqual(typeof mailTo, 'string');
-    assert.strictEqual(typeof program, 'string');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof addon, 'object');
+    assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof event, 'object');
 
     getMailConfig(function (error, mailConfig) {
         if (error) return debug('Error getting mail details:', error);
 
+        const templateData = {
+            webadminUrl: settings.adminOrigin(),
+            cloudronName: mailConfig.cloudronName,
+            app,
+            addon,
+            event: JSON.stringify(event),
+            format: 'text'
+        };
+
         var mailOptions = {
             from: mailConfig.notificationFrom,
             to: mailTo,
-            subject: util.format('[%s] %s was restarted (OOM)', mailConfig.cloudronName, program),
-            text: render('oom_event.ejs', { cloudronName: mailConfig.cloudronName, program: program, event: JSON.stringify(event), format: 'text' })
+            subject: `[${mailConfig.cloudronName}] ${app ? app.fqdn : addon.name} was restarted (OOM)`,
+            text: render('oom_event.ejs', templateData)
         };
 
         sendMail(mailOptions);
@@ -454,7 +427,7 @@ function sendTestMail(domain, email, callback) {
             authUser: `no-reply@${domain}`,
             from: `"${mailConfig.cloudronName}" <no-reply@${domain}>`,
             to: email,
-            subject: util.format('Test Email from %s', mailConfig.cloudronName),
+            subject: `[${mailConfig.cloudronName}] Test Email`,
             text: render('test.ejs', { cloudronName: mailConfig.cloudronName, format: 'text'})
         };
 

src/nginxconfig.ejs

@@ -28,7 +28,13 @@ server {
         alias /home/yellowtent/platformdata/acme/;
     }
 
-    # for default server, serve the splash page. for other endpoints, redirect to HTTPS
+    location /notfound.html {
+        root <%= sourceDir %>/dashboard/dist;
+        try_files /notfound.html =404;
+        internal;
+    }
+
+    # for default server, serve the notfound page. for other endpoints, redirect to HTTPS
     location / {
 <% if ( endpoint === 'admin' || endpoint === 'setup' ) { %>
         return 301 https://$host$request_uri;
@@ -37,8 +43,8 @@ server {
 <% } else if ( endpoint === 'redirect' ) { %>
         return 301 https://<%= redirectTo %>$request_uri;
 <% } else if ( endpoint === 'ip' ) { %>
-        root <%= sourceDir %>/dashboard/dist;
-        try_files /splash.html =404;
+        root /home/yellowtent/boxdata;
+        try_files /custom_pages/notfound.html /notfound.html;
 <% } %>
     }
 }
@@ -89,12 +95,20 @@ server {
     proxy_hide_header X-Content-Type-Options;
     add_header X-Permitted-Cross-Domain-Policies "none";
     proxy_hide_header X-Permitted-Cross-Domain-Policies;
-    add_header Referrer-Policy "no-referrer-when-downgrade";
+    add_header Referrer-Policy "same-origin";
     proxy_hide_header Referrer-Policy;
 
+    # workaround caching issue after /logout. if max-age is set, browser uses cache and user thinks they have not logged out
+    # have to keep all the add_header here to avoid repeating all add_header in location block
+    <% if (proxyAuth.enabled) { %>
+    proxy_hide_header Cache-Control;
+    add_header Cache-Control no-cache;
+    add_header Set-Cookie $auth_cookie;
+    <% } %>
+
     # gzip responses that are > 50k and not images
     gzip on;
-    gzip_min_length 50k;
+    gzip_min_length 18k;
     gzip_types text/css text/javascript text/xml text/plain application/javascript application/x-javascript application/json;
 
     # enable for proxied requests as well
@@ -154,14 +168,82 @@ server {
     }
 
     # user defined .well-known resources
-    location ~ ^/.well-known/(.*)$ {
-        root /home/yellowtent/boxdata/well-known/$host;
-        try_files /$1 @wellknown-upstream;
+    location /.well-known/ {
+        error_page 404 = @wellknown-upstream;
+        proxy_pass http://127.0.0.1:3000/well-known-handler/;
     }
 
 <% if (proxyAuth.enabled) { %>
     proxy_set_header X-App-ID "<%= proxyAuth.id %>";
+<% } %>
 
+    # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
+    proxy_buffer_size       128k;
+    proxy_buffers           4 256k;
+    proxy_busy_buffers_size 256k;
+
+    # No buffering to temp files, it fails for large downloads
+    proxy_max_temp_file_size 0;
+
+    # Disable check to allow unlimited body sizes. this allows apps to accept whatever size they want
+    client_max_body_size 0;
+
+<% if (robotsTxtQuoted) { %>
+    location = /robots.txt {
+        return 200 <%- robotsTxtQuoted %>;
+    }
+<% } %>
+
+<% if ( endpoint === 'admin' || endpoint === 'setup' ) { %>
+    location /api/ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 1m;
+    }
+
+    location ~ ^/api/v1/(developer|session)/login$ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 1m;
+        limit_req zone=admin_login burst=5;
+    }
+
+    # the read timeout is between successive reads and not the whole connection
+    location ~ ^/api/v1/apps/.*/exec$ {
+        proxy_pass   http://127.0.0.1:3000;
+        proxy_read_timeout 30m;
+    }
+
+    location ~ ^/api/v1/apps/.*/upload$ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 0;
+    }
+
+    location ~ ^/api/v1/apps/.*/files/ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 0;
+    }
+
+    location ~ ^/api/v1/volumes/.*/files/ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 0;
+    }
+
+    # graphite paths (uncomment block below and visit /graphite-web/dashboard)
+    # remember to comment out the CSP policy as well to access the graphite dashboard
+    # location ~ ^/graphite-web/ {
+    #     proxy_pass   http://127.0.0.1:8417;
+    #     client_max_body_size 1m;
+    # }
+
+    location / {
+        root   <%= sourceDir %>/dashboard/dist;
+        index  index.html index.htm;
+    }
+<% } else if ( endpoint === 'app' ) { %>
+    location = /appstatus.html {
+        root /home/yellowtent/box/dashboard/dist;
+    }
+
+    <% if (proxyAuth.enabled) { %>
     location = /proxy-auth {
         internal;
         proxy_pass http://127.0.0.1:3001/auth;
@@ -176,104 +258,55 @@ server {
     }
 
     location @proxy-auth-login {
+        if ($http_user_agent ~* "docker") {
+            return 401;
+        }
         return 302 /login?redirect=$request_uri;
     }
-<% } %>
 
-    location / {
-        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
-        proxy_buffer_size       128k;
-        proxy_buffers           4 256k;
-        proxy_busy_buffers_size 256k;
-
-        # No buffering to temp files, it fails for large downloads
-        proxy_max_temp_file_size 0;
-
-        # Disable check to allow unlimited body sizes. this allows apps to accept whatever size they want
-        client_max_body_size 0;
-
-<% if (robotsTxtQuoted) { %>
-        location = /robots.txt {
-            return 200 <%- robotsTxtQuoted %>;
-        }
-<% } %>
-
-<% if ( endpoint === 'admin' || endpoint === 'setup' ) { %>
-        location /api/ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 1m;
-        }
-
-        location ~ ^/api/v1/(developer|session)/login$ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 1m;
-            limit_req zone=admin_login burst=5;
-        }
-
-        # the read timeout is between successive reads and not the whole connection
-        location ~ ^/api/v1/apps/.*/exec$ {
-            proxy_pass   http://127.0.0.1:3000;
-            proxy_read_timeout 30m;
-        }
-
-        location ~ ^/api/v1/apps/.*/upload$ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 0;
-        }
-
-        location ~ ^/api/v1/apps/.*/files/ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 0;
-        }
-
-        location ~ ^/api/v1/volumes/.*/files/ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 0;
-        }
-
-        # graphite paths (uncomment block below and visit /graphite-web/dashboard)
-        # remember to comment out the CSP policy as well to access the graphite dashboard
-        # location ~ ^/graphite-web/ {
-        #     proxy_pass   http://127.0.0.1:8417;
-        #     client_max_body_size 1m;
-        # }
-
-        location / {
-            root   <%= sourceDir %>/dashboard/dist;
-            index  index.html index.htm;
-        }
-<% } else if ( endpoint === 'app' ) { %>
-    location = /appstatus.html {
-    }
-
-    <% if (proxyAuth.enabled) { %>
-    location "<%= proxyAuth.path %>" {
+    location <%= proxyAuth.location %> {
         auth_request /proxy-auth;
         auth_request_set $auth_cookie $upstream_http_set_cookie;
-        add_header Set-Cookie $auth_cookie;
         error_page 401 = @proxy-auth-login;
 
         proxy_pass http://<%= ip %>:<%= port %>;
     }
+
+    <% if (proxyAuth.location !== '/') { %>
+    location / {
+        proxy_pass http://<%= ip %>:<%= port %>;
+    }
+    <% } %>
+
+    <% } else { %>
+    location / {
+        proxy_pass http://<%= ip %>:<%= port %>;
+    }
     <% } %>
 
     <% Object.keys(httpPaths).forEach(function (path) { -%>
-        location "<%= path %>" {
-            # the trailing / will replace part of the original URI matched by the location.
-            proxy_pass http://<%= ip %>:<%= httpPaths[path] %>/;
-        }
+    location "<%= path %>" {
+        # the trailing / will replace part of the original URI matched by the location.
+        proxy_pass http://<%= ip %>:<%= httpPaths[path] %>/;
+    }
     <% }); %>
 
-        proxy_pass http://<%= ip %>:<%= port %>;
 <% } else if ( endpoint === 'redirect' ) { %>
+    location / {
         # redirect everything to the app. this is temporary because there is no way
         # to clear a permanent redirect on the browser
         return 302 https://<%= redirectTo %>$request_uri;
-<% } else if ( endpoint === 'ip' ) { %>
-        location / {
-            root   <%= sourceDir %>/dashboard/dist;
-            try_files /splash.html =404;
-        }
-<% } %>
     }
+<% } else if ( endpoint === 'ip' ) { %>
+    location /notfound.html {
+        root <%= sourceDir %>/dashboard/dist;
+        try_files /notfound.html =404;
+        internal;
+    }
+
+    location / {
+        root /home/yellowtent/boxdata;
+        try_files /custom_pages/notfound.html /notfound.html;
+    }
+<% } %>
 }

src/notifications.js

@@ -1,11 +1,14 @@
 'use strict';
 
 exports = module.exports = {
-    get: get,
-    ack: ack,
-    getAllPaged: getAllPaged,
+    get,
+    ack,
+    getAllPaged,
 
-    onEvent: onEvent,
+    onEvent,
+
+    appUpdatesAvailable,
+    boxUpdateAvailable,
 
     // NOTE: if you add an alert, be sure to add title below
     ALERT_BACKUP_CONFIG: 'backupConfig',
@@ -20,11 +23,13 @@ exports = module.exports = {
     _add: add
 };
 
-let assert = require('assert'),
+let apps = require('./apps.js'),
+    assert = require('assert'),
     async = require('async'),
     auditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
     changelog = require('./changelog.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:notifications'),
     eventlog = require('./eventlog.js'),
     mailer = require('./mailer.js'),
@@ -94,8 +99,8 @@ function getAllPaged(userId, acknowledged, page, perPage, callback) {
 }
 
 // Calls iterator with (admin, callback)
-function actionForAllAdmins(skippingUserIds, iterator, callback) {
-    assert(Array.isArray(skippingUserIds));
+function forEachAdmin(options, iterator, callback) {
+    assert(Array.isArray(options.skip));
     assert.strictEqual(typeof iterator, 'function');
     assert.strictEqual(typeof callback, 'function');
 
@@ -104,7 +109,7 @@ function actionForAllAdmins(skippingUserIds, iterator, callback) {
         if (error) return callback(error);
 
         // filter out users we want to skip (like the user who did the action or the user the action was performed on)
-        result = result.filter(function (r) { return skippingUserIds.indexOf(r.id) === -1; });
+        result = result.filter(function (r) { return options.skip.indexOf(r.id) === -1; });
 
         async.each(result, iterator, callback);
     });
@@ -116,8 +121,7 @@ function userAdded(performedBy, eventId, user, callback) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([ performedBy, user.id ], function (admin, done) {
-        mailer.userAdded(admin.email, user);
+    forEachAdmin({ skip: [ performedBy, user.id ] }, function (admin, done) {
         add(admin.id, eventId, `User '${user.displayName}' added`, `User '${user.username || user.email || user.fallbackEmail}' was added.`, done);
     }, callback);
 }
@@ -128,8 +132,7 @@ function userRemoved(performedBy, eventId, user, callback) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([ performedBy, user.id ], function (admin, done) {
-        mailer.userRemoved(admin.email, user);
+    forEachAdmin({ skip: [ performedBy, user.id ] }, function (admin, done) {
         add(admin.id, eventId, `User '${user.displayName}' removed`, `User '${user.username || user.email || user.fallbackEmail}' was removed.`, done);
     }, callback);
 }
@@ -139,8 +142,7 @@ function roleChanged(performedBy, eventId, user, callback) {
     assert.strictEqual(typeof user, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([ performedBy, user.id ], function (admin, done) {
-        mailer.roleChanged(admin.email, user);
+    forEachAdmin({ skip: [ performedBy, user.id ] }, function (admin, done) {
         add(admin.id, eventId, `User '${user.displayName}'s role changed`, `User '${user.username || user.email || user.fallbackEmail}' now has the role ${user.role}.`, done);
     }, callback);
 }
@@ -152,23 +154,19 @@ function oomEvent(eventId, app, addon, containerId, event, callback) {
     assert.strictEqual(typeof containerId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let title, message, program;
+    assert(app || addon);
+
+    let title, message;
     if (app) {
-        program = `App ${app.fqdn}`;
         title = `The application at ${app.fqdn} ran out of memory.`;
-        message = 'The application has been restarted automatically. If you see this notification often, consider increasing the [memory limit](https://docs.cloudron.io/apps/#memory-limit)';
+        message = `The application has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.adminOrigin()}/#/app/${app.id}/resources)`;
     } else if (addon) {
-        program = `${addon.name} service`;
         title = `The ${addon.name} service ran out of memory`;
-        message = 'The service has been restarted automatically. If you see this notification often, consider increasing the [memory limit](https://docs.cloudron.io/troubleshooting/#services)';
-    } else { // this never happens currently
-        program = `Container ${containerId}`;
-        title = `The container ${containerId} ran out of memory`;
-        message = 'The container has been restarted automatically. Consider increasing the [memory limit](https://docs.docker.com/v17.09/edge/engine/reference/commandline/update/#update-a-containers-kernel-memory-constraints)';
+        message = `The service has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.adminOrigin()}/#/services)`;
     }
 
-    actionForAllAdmins([], function (admin, done) {
-        mailer.oomEvent(admin.email, program, event);
+    forEachAdmin({ skip: [] }, function (admin, done) {
+        mailer.oomEvent(admin.email, app, addon, containerId, event);
 
         add(admin.id, eventId, title, message, done);
     }, callback);
@@ -179,7 +177,7 @@ function appUp(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, done) {
+    forEachAdmin({ skip: [] }, function (admin, done) {
         mailer.appUp(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is back online`, `The application installed at ${app.fqdn} is back online.`, done);
     }, callback);
@@ -190,7 +188,7 @@ function appDied(eventId, app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, callback) {
+    forEachAdmin({ skip: [] }, function (admin, callback) {
         mailer.appDied(admin.email, app);
         add(admin.id, eventId, `App ${app.fqdn} is down`, `The application installed at ${app.fqdn} is not responding.`, callback);
     }, callback);
@@ -208,7 +206,7 @@ function appUpdated(eventId, app, callback) {
     const title = upstreamVersion ? `${app.manifest.title} at ${app.fqdn} updated to ${upstreamVersion} (package version ${app.manifest.version})`
         : `${app.manifest.title} at ${app.fqdn} updated to package version ${app.manifest.version}`;
 
-    actionForAllAdmins([], function (admin, done) {
+    forEachAdmin({ skip: [] }, function (admin, done) {
         add(admin.id, eventId, title, `The application installed at https://${app.fqdn} was updated.\n\nChangelog:\n${app.manifest.changelog}\n`, function (error) {
             if (error) return callback(error);
 
@@ -220,6 +218,39 @@ function appUpdated(eventId, app, callback) {
     }, callback);
 }
 
+function boxUpdateAvailable(updateInfo, callback) {
+    assert.strictEqual(typeof updateInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getAutoupdatePattern(function (error, result) {
+        if (error) return callback(error);
+
+        if (result !== constants.AUTOUPDATE_PATTERN_NEVER) return callback();
+
+        forEachAdmin({ skip: [] }, function (admin, done) {
+            mailer.boxUpdateAvailable(admin.email, updateInfo, done);
+        }, callback);
+    });
+}
+
+function appUpdatesAvailable(appUpdates, callback) {
+    assert.strictEqual(typeof appUpdates, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getAutoupdatePattern(function (error, result) {
+        if (error) return callback(error);
+
+        // if we are auto updating, then just consider apps that cannot be auto updated
+        if (result !== constants.AUTOUPDATE_PATTERN_NEVER) appUpdates = appUpdates.filter(update => !apps.canAutoupdateApp(update.app, update.updateInfo));
+
+        if (appUpdates.length === 0) return callback();
+
+        forEachAdmin({ skip: [] }, function (admin, done) {
+            mailer.appUpdatesAvailable(admin.email, appUpdates, done);
+        }, callback);
+    });
+}
+
 function boxUpdated(eventId, oldVersion, newVersion, callback) {
     assert.strictEqual(typeof eventId, 'string');
     assert.strictEqual(typeof oldVersion, 'string');
@@ -229,7 +260,7 @@ function boxUpdated(eventId, oldVersion, newVersion, callback) {
     const changes = changelog.getChanges(newVersion);
     const changelogMarkdown = changes.map((m) => `* ${m}\n`).join('');
 
-    actionForAllAdmins([], function (admin, done) {
+    forEachAdmin({ skip: [] }, function (admin, done) {
         add(admin.id, eventId, `Cloudron updated to v${newVersion}`, `Cloudron was updated from v${oldVersion} to v${newVersion}.\n\nChangelog:\n${changelogMarkdown}\n`, done);
     }, callback);
 }
@@ -239,7 +270,7 @@ function boxUpdateError(eventId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, done) {
+    forEachAdmin({ skip: [] }, function (admin, done) {
         mailer.boxUpdateError(admin.email, errorMessage);
         add(admin.id, eventId, 'Cloudron update failed', `Failed to update Cloudron: ${errorMessage}.`, done);
     }, callback);
@@ -251,7 +282,7 @@ function certificateRenewalError(eventId, vhost, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, callback) {
+    forEachAdmin({ skip: [] }, function (admin, callback) {
         mailer.certificateRenewalError(admin.email, vhost, errorMessage);
         add(admin.id, eventId, `Certificate renewal of ${vhost} failed`, `Failed to new certs of ${vhost}: ${errorMessage}. Renewal will be retried in 12 hours`, callback);
     }, callback);
@@ -263,7 +294,7 @@ function backupFailed(eventId, taskId, errorMessage, callback) {
     assert.strictEqual(typeof errorMessage, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    actionForAllAdmins([], function (admin, callback) {
+    forEachAdmin({ skip: [] }, function (admin, callback) {
         mailer.backupFailed(admin.email, errorMessage, `${settings.adminOrigin()}/logs.html?taskId=${taskId}`);
         add(admin.id, eventId, 'Backup failed', `Backup failed: ${errorMessage}. Logs are available [here](/logs.html?taskId=${taskId}).`, callback);
     }, callback);
@@ -275,11 +306,10 @@ function alert(id, title, message, callback) {
     assert.strictEqual(typeof message, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`alert: id=${id} title=${title}`);
-
     const acknowledged = !message;
+    debug(`alert: id=${id} title=${title} ack=${acknowledged}`);
 
-    actionForAllAdmins([], function (admin, callback) {
+    forEachAdmin({ skip: [] }, function (admin, callback) {
         const data = {
             userId: admin.id,
             eventId: null,

src/paths.js

@@ -19,6 +19,7 @@ exports = module.exports = {
     DASHBOARD_DIR: constants.TEST ? path.join(__dirname, '../../dashboard/src') : path.join(baseDir(), 'box/dashboard/dist'),
 
     PROVIDER_FILE: '/etc/cloudron/PROVIDER',
+    SETUP_TOKEN_FILE: '/etc/cloudron/SETUP_TOKEN',
 
     PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'),
     APPS_DATA_DIR: path.join(baseDir(), 'appsdata'),
@@ -59,6 +60,8 @@ exports = module.exports = {
 
     GHOST_USER_FILE: path.join(baseDir(), 'platformdata/cloudron_ghost.json'),
 
+    SWAP_RATIO_FILE: path.join(baseDir(), 'platformdata/swap-ratio'),
+
     // this pattern is for the cloudron logs API route to work
     BACKUP_LOG_FILE: path.join(baseDir(), 'platformdata/logs/backup/app.log'),
     UPDATER_LOG_FILE: path.join(baseDir(), 'platformdata/logs/updater/app.log'),

src/platform.js

@@ -1,32 +1,30 @@
 'use strict';
 
 exports = module.exports = {
-    start: start,
-    stopAllTasks: stopAllTasks,
+    start,
+    stopAllTasks,
 
     // exported for testing
     _isReady: false
 };
 
-var addons = require('./addons.js'),
-    apps = require('./apps.js'),
+const apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     debug = require('debug')('box:platform'),
     fs = require('fs'),
-    graphs = require('./graphs.js'),
     infra = require('./infra_version.js'),
     locker = require('./locker.js'),
     paths = require('./paths.js'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
-    settings = require('./settings.js'),
-    sftp = require('./sftp.js'),
+    services = require('./services.js'),
     shell = require('./shell.js'),
     tasks = require('./tasks.js'),
     _ = require('underscore');
 
-function start(callback) {
+function start(options, callback) {
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return callback();
@@ -55,10 +53,8 @@ function start(callback) {
 
     async.series([
         (next) => { if (existingInfra.version !== infra.version) removeAllContainers(next); else next(); },
-        markApps.bind(null, existingInfra), // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
-        graphs.startGraphite.bind(null, existingInfra),
-        sftp.startSftp.bind(null, existingInfra),
-        addons.startServices.bind(null, existingInfra),
+        markApps.bind(null, existingInfra, options), // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
+        services.startServices.bind(null, existingInfra),
         fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra, null, 4))
     ], function (error) {
         if (error) return callback(error);
@@ -80,7 +76,7 @@ function onPlatformReady(infraChanged) {
     exports._isReady = true;
 
     let tasks = [ apps.schedulePendingTasks ];
-    if (infraChanged) tasks.push(applyPlatformConfig, pruneInfraImages);
+    if (infraChanged) tasks.push(pruneInfraImages);
 
     async.series(async.reflectAll(tasks), function (error, results) {
         results.forEach((result, idx) => {
@@ -89,14 +85,6 @@ function onPlatformReady(infraChanged) {
     });
 }
 
-function applyPlatformConfig(callback) {
-    settings.getPlatformConfig(function (error, platformConfig) {
-        if (error) return callback(error);
-
-        addons.updateServiceConfig(platformConfig, callback);
-    });
-}
-
 function pruneInfraImages(callback) {
     debug('pruneInfraImages: checking existing images');
 
@@ -131,10 +119,14 @@ function removeAllContainers(callback) {
     ], callback);
 }
 
-function markApps(existingInfra, callback) {
+function markApps(existingInfra, options, callback) {
+    assert.strictEqual(typeof existingInfra, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     if (existingInfra.version === 'none') { // cloudron is being restored from backup
         debug('markApps: restoring installed apps');
-        apps.restoreInstalledApps(callback);
+        apps.restoreInstalledApps(options, callback);
     } else if (existingInfra.version !== infra.version) {
         debug('markApps: reconfiguring installed apps');
         reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start

src/provision.js

@@ -149,16 +149,17 @@ function activate(username, password, email, displayName, ip, auditSource, callb
                 expires: result.expires
             });
 
-            setImmediate(cloudron.onActivated.bind(null, NOOP_CALLBACK)); // hack for now to not block the above http response
+            setImmediate(cloudron.onActivated.bind(null, {}, NOOP_CALLBACK)); // hack for now to not block the above http response
         });
     });
 }
 
-function restore(backupConfig, backupId, version, sysinfoConfig, auditSource, callback) {
+function restore(backupConfig, backupId, version, sysinfoConfig, options, auditSource, callback) {
     assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof version, 'string');
     assert.strictEqual(typeof sysinfoConfig, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -203,7 +204,10 @@ function restore(backupConfig, backupId, version, sysinfoConfig, auditSource, ca
                     (done) => {
                         const adminDomain = settings.adminDomain(); // load this fresh from after the backup.restore
                         async.series([
-                            cloudron.setupDnsAndCert.bind(null, constants.ADMIN_LOCATION, adminDomain, auditSource, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)),
+                            (next) => {
+                                if (options.skipDnsSetup) return next();
+                                cloudron.setupDnsAndCert(constants.ADMIN_LOCATION, adminDomain, auditSource, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK), next);
+                            },
                             cloudron.setDashboardDomain.bind(null, adminDomain, auditSource)
                         ], done);
                     },
@@ -213,7 +217,7 @@ function restore(backupConfig, backupId, version, sysinfoConfig, auditSource, ca
                     gProvisionStatus.restore.active = false;
                     gProvisionStatus.restore.errorMessage = error ? error.message : '';
 
-                    if (!error) cloudron.onActivated(NOOP_CALLBACK);
+                    if (!error) cloudron.onActivated(options, NOOP_CALLBACK);
                 });
             });
         });

src/proxyauth.js

@@ -24,6 +24,7 @@ const apps = require('./apps.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
+    speakeasy = require('speakeasy'),
     translation = require('./translation.js'),
     users = require('./users.js');
 
@@ -55,11 +56,17 @@ function basicAuthVerify(req, res, next) {
 
     const api = credentials.name.indexOf('@') !== -1 ? users.verifyWithEmail : users.verifyWithUsername;
 
-    api(credentials.name, credentials.pass, appId, function (error, user) {
-        if (error) return next(new HttpError(403, 'Invalid username or password' ));
+    apps.get(appId, function (error, app) {
+        if (error) return next(new HttpError(503, error.message));
 
-        req.user = user;
-        next();
+        if (!app.manifest.addons.proxyAuth.basicAuth) return next();
+
+        api(credentials.name, credentials.pass, appId, function (error, user) {
+            if (error) return next(new HttpError(403, 'Invalid username or password' ));
+
+            req.user = user;
+            next();
+        });
     });
 }
 
@@ -98,9 +105,24 @@ function loginPage(req, res, next) {
     });
 }
 
-// called by nginx to authorize any protected route
+// someday this can be more sophisticated and check for a real browser
+function isBrowser(req) {
+    const userAgent = req.get('user-agent');
+    if (!userAgent) return false;
+
+    // https://github.com/docker/engine/blob/master/dockerversion/useragent.go#L18
+    return !userAgent.toLowerCase().includes('docker');
+}
+
+// called by nginx to authorize any protected route. this route must return only 2xx or 401/403 (http://nginx.org/en/docs/http/ngx_http_auth_request_module.html)
 function auth(req, res, next) {
-    if (!req.user) return next(new HttpError(401, 'Unauthorized'));
+    if (!req.user) {
+        if (isBrowser(req)) return next(new HttpError(401, 'Unauthorized'));
+
+        // the header has to be generated here and cannot be set in nginx config - https://forum.nginx.org/read.php?2,171461,171469#msg-171469
+        res.set('www-authenticate', 'Basic realm="Cloudron"');
+        return next(new HttpError(401, 'Unauthorized'));
+    }
 
     // user is already authenticated, refresh cookie
     const token = jwt.sign({ user: req.user }, TOKEN_SECRET, { expiresIn: `${EXPIRY_DAYS}d` });
@@ -115,7 +137,7 @@ function auth(req, res, next) {
 }
 
 // endpoint called by login page, username and password posted as JSON body
-function authenticate(req, res, next) {
+function passwordAuth(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
     const appId = req.headers['x-app-id'] || '';
@@ -123,14 +145,22 @@ function authenticate(req, res, next) {
 
     if (typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be non empty string' ));
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be non empty string' ));
+    if ('totpToken' in req.body && typeof req.body.totpToken !== 'string') return next(new HttpError(400, 'totpToken must be a string' ));
 
-    const { username, password } = req.body;
+    const { username, password, totpToken } = req.body;
 
     const api = username.indexOf('@') !== -1 ? users.verifyWithEmail : users.verifyWithUsername;
 
     api(username, password, appId, function (error, user) {
         if (error) return next(new HttpError(403, 'Invalid username or password' ));
 
+        if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
+            if (!totpToken) return next(new HttpError(403, 'A totpToken must be provided'));
+
+            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
+            if (!verified) return next(new HttpError(403, 'Invalid totpToken'));
+        }
+
         req.user = user;
         next();
     });
@@ -155,14 +185,24 @@ function authorize(req, res, next) {
                 secure: true
             });
 
-            res.redirect('/');
+            res.redirect(302, '/');
         });
     });
 }
 
-function logoutPage(req, res) {
-    res.clearCookie('authToken');
-    res.redirect('/'); // do not redirect to '/login' as it may not be protected
+function logoutPage(req, res, next) {
+    const appId = req.headers['x-app-id'] || '';
+    if (!appId) return next(new HttpError(503, 'Nginx misconfiguration'));
+
+    apps.get(appId, function (error, app) {
+        if (error) return next(new HttpError(503, error.message));
+
+        res.clearCookie('authToken');
+
+        // when we have no path, redirect to the login page. we cannot redirect to '/' because browsers will immediately serve up the cached page
+        // if a path is set, we can assume '/' is a public page
+        res.redirect(302, app.manifest.addons.proxyAuth.path ? '/' : '/login');
+    });
 }
 
 function logout(req, res, next) {
@@ -193,7 +233,7 @@ function initializeAuthwallExpressSync() {
 
     router.get ('/login',     loginPage);
     router.get ('/auth',      jwtVerify, basicAuthVerify, auth);
-    router.post('/login',     json, authenticate, authorize);
+    router.post('/login',     json, passwordAuth, authorize);
     router.get ('/logout',    logoutPage);
     router.post('/logout',    json, logout);
 

src/reverseproxy.js

@@ -54,8 +54,16 @@ var acme2 = require('./cert/acme2.js'),
     users = require('./users.js'),
     util = require('util');
 
-var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/nginxconfig.ejs', { encoding: 'utf8' }),
-    RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh');
+const NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/nginxconfig.ejs', { encoding: 'utf8' });
+const RESTART_SERVICE_CMD = path.join(__dirname, 'scripts/restartservice.sh');
+
+function nginxLocation(s) {
+    if (!s.startsWith('!')) return s;
+
+    let re = s.replace(/[\^$\\.*+?()[\]{}|]/g, '\\$&'); // https://github.com/es-shims/regexp.escape/blob/master/implementation.js
+
+    return `~ ^(?!(${re.slice(1)}))`; // negative regex assertion - https://stackoverflow.com/questions/16302897/nginx-location-not-equal-to-regex
+}
 
 function getAcmeApi(domainObject, callback) {
     assert.strictEqual(typeof domainObject, 'object');
@@ -164,7 +172,7 @@ function validateCertificate(location, domainObject, certificate) {
 function reload(callback) {
     if (constants.TEST) return callback();
 
-    shell.sudo('reload', [ RELOAD_NGINX_CMD ], {}, function (error) {
+    shell.sudo('reload', [ RESTART_SERVICE_CMD, 'nginx' ], {}, function (error) {
         if (error) return callback(new BoxError(BoxError.NGINX_ERROR, `Error reloading nginx: ${error.message}`));
 
         callback();
@@ -249,22 +257,22 @@ function setAppCertificateSync(location, domainObject, certificate) {
     return null;
 }
 
-function getAcmeCertificate(hostname, domainObject, callback) {
-    assert.strictEqual(typeof hostname, 'string');
+function getAcmeCertificate(vhost, domainObject, callback) {
+    assert.strictEqual(typeof vhost, 'string'); // this can contain wildcard domain (for alias domains)
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     let certFilePath, keyFilePath;
 
-    if (hostname !== domainObject.domain && domainObject.tlsConfig.wildcard) { // bare domain is not part of wildcard SAN
-        let certName = domains.makeWildcard(hostname).replace('*.', '_.');
+    if (vhost !== domainObject.domain && domainObject.tlsConfig.wildcard) { // bare domain is not part of wildcard SAN
+        let certName = domains.makeWildcard(vhost).replace('*.', '_.');
         certFilePath = path.join(paths.APP_CERTS_DIR, `${certName}.cert`);
         keyFilePath = path.join(paths.APP_CERTS_DIR, `${certName}.key`);
 
         if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath });
     } else {
-        certFilePath = path.join(paths.APP_CERTS_DIR, `${hostname}.cert`);
-        keyFilePath = path.join(paths.APP_CERTS_DIR, `${hostname}.key`);
+        certFilePath = path.join(paths.APP_CERTS_DIR, `${vhost}.cert`);
+        keyFilePath = path.join(paths.APP_CERTS_DIR, `${vhost}.key`);
 
         if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath });
     }
@@ -383,7 +391,7 @@ function writeDashboardNginxConfig(bundle, configFileName, vhost, callback) {
         certFilePath: bundle.certFilePath,
         keyFilePath: bundle.keyFilePath,
         robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n'),
-        proxyAuth: { enabled: false, id: null, path: '/' }
+        proxyAuth: { enabled: false, id: null, location: nginxLocation('/') }
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, configFileName);
@@ -430,8 +438,9 @@ function writeDashboardConfig(domain, callback) {
     });
 }
 
-function writeAppNginxConfig(app, bundle, callback) {
+function writeAppNginxConfig(app, fqdn, bundle, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof fqdn, 'string');
     assert.strictEqual(typeof bundle, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -450,7 +459,7 @@ function writeAppNginxConfig(app, bundle, callback) {
     var data = {
         sourceDir: sourceDir,
         adminOrigin: settings.adminOrigin(),
-        vhost: app.fqdn,
+        vhost: fqdn,
         hasIPv6: sysinfo.hasIPv6(),
         ip: app.containerIp,
         port: app.manifest.httpPort,
@@ -463,14 +472,15 @@ function writeAppNginxConfig(app, bundle, callback) {
         proxyAuth: {
             enabled: app.sso && app.manifest.addons && app.manifest.addons.proxyAuth,
             id: app.id,
-            path: safe.query(app.manifest, 'addons.proxyAuth.path') || '/'
+            location: nginxLocation(safe.query(app.manifest, 'addons.proxyAuth.path') || '/')
         },
         httpPaths: app.manifest.httpPaths || {}
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 
-    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-    debug('writeAppNginxConfig: writing config for "%s" to %s with options %j', app.fqdn, nginxConfigFilename, data);
+    const aliasSuffix = app.fqdn === fqdn ? '' : `-alias-${fqdn.replace('*', '_')}`;
+    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}${aliasSuffix}.conf`);
+    debug('writeAppNginxConfig: writing config for "%s" to %s with options %j', fqdn, nginxConfigFilename, data);
 
     if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
         debug('Error creating nginx config for "%s" : %s', app.fqdn, safe.error.message);
@@ -497,7 +507,7 @@ function writeAppRedirectNginxConfig(app, fqdn, bundle, callback) {
         robotsTxtQuoted: null,
         cspQuoted: null,
         hideHeaders: [],
-        proxyAuth: { enabled: false, id: app.id, path: '/' }
+        proxyAuth: { enabled: false, id: app.id, location: nginxLocation('/') }
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 
@@ -517,21 +527,30 @@ function writeAppConfig(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    getCertificate(app.fqdn, app.domain, function (error, bundle) {
-        if (error) return callback(error);
+    let appDomains = [];
+    appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary' });
 
-        writeAppNginxConfig(app, bundle, function (error) {
-            if (error) return callback(error);
-
-            async.eachSeries(app.alternateDomains, function (alternateDomain, iteratorDone) {
-                getCertificate(alternateDomain.fqdn, alternateDomain.domain, function (error, bundle) {
-                    if (error) return iteratorDone(error);
-
-                    writeAppRedirectNginxConfig(app, alternateDomain.fqdn, bundle, iteratorDone);
-                });
-            }, callback);
-        });
+    app.alternateDomains.forEach(function (alternateDomain) {
+        appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate' });
     });
+
+    app.aliasDomains.forEach(function (aliasDomain) {
+        appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias' });
+    });
+
+    async.eachSeries(appDomains, function (appDomain, iteratorDone) {
+        getCertificate(appDomain.fqdn, appDomain.domain, function (error, bundle) {
+            if (error) return iteratorDone(error);
+
+            if (appDomain.type === 'primary') {
+                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            } else if (appDomain.type === 'alternate') {
+                writeAppRedirectNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            } else if (appDomain.type === 'alias') {
+                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            }
+        });
+    }, callback);
 }
 
 function configureApp(app, auditSource, callback) {
@@ -539,21 +558,30 @@ function configureApp(app, auditSource, callback) {
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    ensureCertificate(app.fqdn, app.domain, auditSource, function (error, bundle) {
-        if (error) return callback(error);
+    let appDomains = [];
+    appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary' });
 
-        writeAppNginxConfig(app, bundle, function (error) {
-            if (error) return callback(error);
-
-            async.eachSeries(app.alternateDomains, function (alternateDomain, iteratorDone) {
-                ensureCertificate(alternateDomain.fqdn, alternateDomain.domain, auditSource, function (error, bundle) {
-                    if (error) return iteratorDone(error);
-
-                    writeAppRedirectNginxConfig(app, alternateDomain.fqdn, bundle, iteratorDone);
-                });
-            }, callback);
-        });
+    app.alternateDomains.forEach(function (alternateDomain) {
+        appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate' });
     });
+
+    app.aliasDomains.forEach(function (aliasDomain) {
+        appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias' });
+    });
+
+    async.eachSeries(appDomains, function (appDomain, iteratorDone) {
+        ensureCertificate(appDomain.fqdn, appDomain.domain, auditSource, function (error, bundle) {
+            if (error) return iteratorDone(error);
+
+            if (appDomain.type === 'primary') {
+                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            } else if (appDomain.type === 'alternate') {
+                writeAppRedirectNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            } else if (appDomain.type === 'alias') {
+                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
+            }
+        });
+    }, callback);
 }
 
 function unconfigureApp(app, callback) {
@@ -577,7 +605,7 @@ function renewCerts(options, auditSource, progressCallback, callback) {
     apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        var appDomains = [];
+        let appDomains = [];
 
         // add webadmin and mail domain
         if (settings.mailFqdn() === settings.adminFqdn()) {
@@ -587,16 +615,20 @@ function renewCerts(options, auditSource, progressCallback, callback) {
             appDomains.push({ domain: settings.mailDomain(), fqdn: settings.mailFqdn(), type: 'mail' });
         }
 
-        // add app main
         allApps.forEach(function (app) {
             if (app.runState === apps.RSTATE_STOPPED) return; // do not renew certs of stopped apps
 
-            appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'main', app: app, nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf') });
+            appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary', app: app, nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf') });
 
             app.alternateDomains.forEach(function (alternateDomain) {
                 const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-redirect-${alternateDomain.fqdn}.conf`);
                 appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate', app: app, nginxConfigFilename });
             });
+
+            app.aliasDomains.forEach(function (aliasDomain) {
+                const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-alias-${aliasDomain.fqdn.replace('*', '_')}.conf`);
+                appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias', app: app, nginxConfigFilename });
+            });
         });
 
         if (options.domain) appDomains = appDomains.filter(function (appDomain) { return appDomain.domain === options.domain; });
@@ -628,10 +660,12 @@ function renewCerts(options, auditSource, progressCallback, callback) {
                         mail.handleCertChanged,
                         writeDashboardNginxConfig.bind(null, bundle, `${settings.adminFqdn()}.conf`, settings.adminFqdn())
                     ], iteratorCallback);
-                } else if (appDomain.type === 'main') {
-                    return writeAppNginxConfig(appDomain.app, bundle, iteratorCallback);
+                } else if (appDomain.type === 'primary') {
+                    return writeAppNginxConfig(appDomain.app, appDomain.fqdn, bundle, iteratorCallback);
                 } else if (appDomain.type === 'alternate') {
                     return writeAppRedirectNginxConfig(appDomain.app, appDomain.fqdn, bundle, iteratorCallback);
+                } else if (appDomain.type === 'alias') {
+                    return writeAppNginxConfig(appDomain.app, appDomain.fqdn, bundle, iteratorCallback);
                 }
 
                 iteratorCallback(new BoxError(BoxError.INTERNAL_ERROR, `Unknown domain type for ${appDomain.fqdn}. This should never happen`));
@@ -644,7 +678,13 @@ function renewCerts(options, auditSource, progressCallback, callback) {
 
             async.series([
                 (next) => { return renewed.includes(settings.mailFqdn()) ? mail.handleCertChanged(next) : next(); },// mail cert renewed
-                reload // reload nginx if any certs were updated but the config was not rewritten
+                reload, // reload nginx if any certs were updated but the config was not rewritten
+                (next) => { // restart tls apps on cert change
+                    const tlsApps = allApps.filter(app => app.manifest.addons && app.manifest.addons.tls && renewed.includes(app.fqdn));
+                    async.eachSeries(tlsApps, function (app, iteratorDone) {
+                        apps.restart(app, auditSource, () => iteratorDone());
+                    }, next);
+                }
             ], callback);
         });
     });
@@ -685,7 +725,7 @@ function writeDefaultConfig(options, callback) {
         certFilePath,
         keyFilePath,
         robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n'),
-        proxyAuth: { enabled: false, id: null, path: '/' }
+        proxyAuth: { enabled: false, id: null, location: nginxLocation('/') }
     };
     const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
     const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, constants.NGINX_DEFAULT_CONFIG_FILE_NAME);

src/routes/accesscontrol.js

@@ -1,11 +1,11 @@
 'use strict';
 
 exports = module.exports = {
-    passwordAuth: passwordAuth,
-    tokenAuth: tokenAuth,
+    passwordAuth,
+    tokenAuth,
 
-    authorize: authorize,
-    websocketAuth: websocketAuth
+    authorize,
+    websocketAuth
 };
 
 var accesscontrol = require('../accesscontrol.js'),
@@ -21,17 +21,17 @@ function passwordAuth(req, res, next) {
 
     if (!req.body.username || typeof req.body.username !== 'string') return next(new HttpError(400, 'A username must be non-empty string'));
     if (!req.body.password || typeof req.body.password !== 'string') return next(new HttpError(400, 'A password must be non-empty string'));
+    if ('totpToken' in req.body && typeof req.body.totpToken !== 'string') return next(new HttpError(400, 'totpToken must be a string' ));
 
-    const username = req.body.username;
-    const password = req.body.password;
+    const { username, password, totpToken } = req.body;
 
     function check2FA(user) {
         assert.strictEqual(typeof user, 'object');
 
         if (!user.ghost && !user.appPassword && user.twoFactorAuthenticationEnabled) {
-            if (!req.body.totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
+            if (!totpToken) return next(new HttpError(401, 'A totpToken must be provided'));
 
-            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: req.body.totpToken, window: 2 });
+            let verified = speakeasy.totp.verify({ secret: user.twoFactorAuthenticationSecret, encoding: 'base32', token: totpToken, window: 2 });
             if (!verified) return next(new HttpError(401, 'Invalid totpToken'));
         }
 
@@ -99,6 +99,7 @@ function tokenAuth(req, res, next) {
         if (error && error.reason === BoxError.INVALID_CREDENTIALS) return next(new HttpError(401, 'Unauthorized'));
         if (error) return next(new HttpError(500, error.message));
 
+        req.access_token = token; // used in logout route
         req.user = user;
 
         next();

src/routes/apps.js

@@ -44,6 +44,7 @@ exports = module.exports = {
     uploadFile,
     downloadFile,
 
+
     load
 };
 
@@ -139,12 +140,18 @@ function install(req, res, next) {
         if (data.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
     }
 
+    if ('aliasDomains' in data) {
+        if (!Array.isArray(data.aliasDomains)) return next(new HttpError(400, 'aliasDomains must be an array'));
+        if (data.aliasDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'aliasDomains array must contain objects with domain and subdomain strings'));
+    }
+
     if ('env' in data) {
         if (!data.env || typeof data.env !== 'object') return next(new HttpError(400, 'env must be an object'));
         if (Object.keys(data.env).some(function (key) { return typeof data.env[key] !== 'string'; })) return next(new HttpError(400, 'env must contain values as strings'));
     }
 
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
+    if ('skipDnsSetup' in req.body && typeof req.body.skipDnsSetup !== 'boolean') return next(new HttpError(400, 'skipDnsSetup must be boolean'));
 
     apps.downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
         if (error) return next(BoxError.toHttpError(error));
@@ -353,7 +360,13 @@ function setLocation(req, res, next) {
         if (req.body.alternateDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'alternateDomains array must contain objects with domain and subdomain strings'));
     }
 
+    if ('aliasDomains' in req.body) {
+        if (!Array.isArray(req.body.aliasDomains)) return next(new HttpError(400, 'aliasDomains must be an array'));
+        if (req.body.aliasDomains.some(function (d) { return (typeof d.domain !== 'string' || typeof d.subdomain !== 'string'); })) return next(new HttpError(400, 'aliasDomains array must contain objects with domain and subdomain strings'));
+    }
+
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
+    if ('skipDnsSetup' in req.body && typeof req.body.skipDnsSetup !== 'boolean') return next(new HttpError(400, 'skipDnsSetup must be boolean'));
 
     apps.setLocation(req.resource, req.body, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
@@ -467,6 +480,7 @@ function clone(req, res, next) {
     if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
 
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
+    if ('skipDnsSetup' in req.body && typeof req.body.skipDnsSetup !== 'boolean') return next(new HttpError(400, 'skipDnsSetup must be boolean'));
 
     apps.clone(req.resource, data, req.user, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));

src/routes/cloudron.js

@@ -21,7 +21,8 @@ exports = module.exports = {
     renewCerts,
     getServerIp,
     getLanguages,
-    syncExternalLdap
+    syncExternalLdap,
+    syncDnsRecords
 };
 
 let assert = require('assert'),
@@ -64,24 +65,11 @@ function login(req, res, next) {
 }
 
 function logout(req, res) {
-    var token;
+    assert.strictEqual(typeof req.access_token, 'string');
 
-    // this determines the priority
-    if (req.body && req.body.access_token) token = req.body.access_token;
-    if (req.query && req.query.access_token) token = req.query.access_token;
-    if (req.headers && req.headers.authorization) {
-        var parts = req.headers.authorization.split(' ');
-        if (parts.length == 2) {
-            var scheme = parts[0];
-            var credentials = parts[1];
+    eventlog.add(eventlog.ACTION_USER_LOGOUT, auditSource.fromRequest(req), { userId: req.user.id, user: users.removePrivateFields(req.user) });
 
-            if (/^Bearer$/i.test(scheme)) token = credentials;
-        }
-    }
-
-    if (!token) return res.redirect('/login.html');
-
-    tokendb.delByAccessToken(token, function () { res.redirect('/login.html'); });
+    tokendb.delByAccessToken(req.access_token, function () { res.redirect('/login.html'); });
 }
 
 function passwordResetRequest(req, res, next) {
@@ -293,6 +281,8 @@ function prepareDashboardDomain(req, res, next) {
 }
 
 function renewCerts(req, res, next) {
+    if ('domain' in req.body && typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
+
     cloudron.renewCerts({ domain: req.body.domain || null }, auditSource.fromRequest(req), function (error, taskId) {
         if (error) return next(BoxError.toHttpError(error));
 
@@ -304,7 +294,7 @@ function syncExternalLdap(req, res, next) {
     externalLdap.startSyncer(function (error, taskId) {
         if (error) return next(new HttpError(500, error.message));
 
-        next(new HttpSuccess(202, { taskId: taskId }));
+        next(new HttpSuccess(202, { taskId }));
     });
 }
 
@@ -323,3 +313,17 @@ function getLanguages(req, res, next) {
         next(new HttpSuccess(200, { languages }));
     });
 }
+
+function syncDnsRecords(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if ('domain' in req.body && typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
+    if ('type' in req.body && typeof req.body.type !== 'string') return next(new HttpError(400, 'type must be a string'));
+
+    cloudron.syncDnsRecords(req.body, function (error, taskId) {
+        if (error && error.reason === BoxError.ACCESS_DENIED) return next(new HttpSuccess(200, { error: { reason: error.reason, message: error.message }}));
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(201, { taskId }));
+    });
+}

src/routes/domains.js

@@ -1,13 +1,13 @@
 'use strict';
 
 exports = module.exports = {
-    add: add,
-    get: get,
-    getAll: getAll,
-    update: update,
-    del: del,
+    add,
+    get,
+    getAll,
+    update,
+    del,
 
-    checkDnsRecords: checkDnsRecords,
+    checkDnsRecords,
 };
 
 var assert = require('assert'),
@@ -99,6 +99,13 @@ function update(req, res, next) {
         if (!req.body.tlsConfig.provider || typeof req.body.tlsConfig.provider !== 'string') return next(new HttpError(400, 'tlsConfig.provider must be a string'));
     }
 
+    if ('wellKnown' in req.body) {
+        if (typeof req.body.wellKnown !== 'object') return next(new HttpError(400, 'wellKnown must be an object'));
+        if (req.body.wellKnown) {
+            if (Object.keys(req.body.wellKnown).some(k => typeof req.body.wellKnown[k] !== 'string')) return next(new HttpError(400, 'wellKnown is a map of strings'));
+        }
+    }
+
     // some DNS providers like DigitalOcean take a really long time to verify credentials (https://github.com/expressjs/timeout/issues/26)
     req.clearTimeout();
 
@@ -107,7 +114,8 @@ function update(req, res, next) {
         provider: req.body.provider,
         config: req.body.config,
         fallbackCertificate: req.body.fallbackCertificate || null,
-        tlsConfig: req.body.tlsConfig || { provider: 'letsencrypt-prod' }
+        tlsConfig: req.body.tlsConfig || { provider: 'letsencrypt-prod' },
+        wellKnown: req.body.wellKnown || null
     };
 
     domains.update(req.params.domain, data, auditSource.fromRequest(req), function (error) {

src/routes/filemanager.js

@@ -4,11 +4,11 @@ exports = module.exports = {
     proxy
 };
 
-var addons = require('../addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     middleware = require('../middleware/index.js'),
     HttpError = require('connect-lastmile').HttpError,
+    services = require('../services.js'),
     url = require('url');
 
 function proxy(req, res, next) {
@@ -18,7 +18,7 @@ function proxy(req, res, next) {
 
     req.clearTimeout();
 
-    addons.getContainerDetails('sftp', 'CLOUDRON_SFTP_TOKEN', function (error, result) {
+    services.getContainerDetails('sftp', 'CLOUDRON_SFTP_TOKEN', function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         let parsedUrl = url.parse(req.url, true /* parseQueryString */);

src/routes/graphs.js

@@ -1,7 +1,7 @@
 'use strict';
 
 exports = module.exports = {
-    getGraphs: getGraphs
+    getGraphs
 };
 
 var middleware = require('../middleware/index.js'),

src/routes/index.js

@@ -25,5 +25,6 @@ exports = module.exports = {
     tasks: require('./tasks.js'),
     tokens: require('./tokens.js'),
     users: require('./users.js'),
-    volumes: require('./volumes.js')
+    volumes: require('./volumes.js'),
+    wellknown: require('./wellknown.js')
 };

src/routes/mail.js

@@ -3,8 +3,6 @@
 exports = module.exports = {
     getDomain,
 
-    setDnsRecords,
-
     getStatus,
 
     setMailFromValidation,
@@ -50,21 +48,6 @@ function getDomain(req, res, next) {
     });
 }
 
-function setDnsRecords(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.params.domain, 'string');
-
-    // can take a setup all the DNS entries. this is mostly because some backends try to list DNS entries (DO)
-    // for upsert and this takes a lot of time
-    req.clearTimeout();
-
-    mail.setDnsRecords(req.params.domain, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(201));
-    });
-}
-
 function getStatus(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 

src/routes/mailserver.js

@@ -8,8 +8,7 @@ exports = module.exports = {
     setLocation
 };
 
-var addons = require('../addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:routes/mailserver'),
@@ -17,6 +16,7 @@ var addons = require('../addons.js'),
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     mail = require('../mail.js'),
     middleware = require('../middleware/index.js'),
+    services = require('../services.js'),
     url = require('url');
 
 function restart(req, res, next) {
@@ -33,7 +33,7 @@ function proxy(req, res, next) {
     delete req.headers['authorization'];
     delete req.headers['cookies'];
 
-    addons.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
+    services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
         if (error) return next(BoxError.toHttpError(error));
 
         parsedUrl.query['access_token'] = addonDetails.token;

src/routes/provision.js

@@ -1,11 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    providerTokenAuth: providerTokenAuth,
-    setup: setup,
-    activate: activate,
-    restore: restore,
-    getStatus: getStatus
+    providerTokenAuth,
+    setup,
+    activate,
+    restore,
+    getStatus,
+    setupTokenAuth
 };
 
 var assert = require('assert'),
@@ -15,10 +16,24 @@ var assert = require('assert'),
     debug = require('debug')('box:routes/setup'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    paths = require('../paths.js'),
     provision = require('../provision.js'),
     request = require('request'),
+    safe = require('safetydance'),
     settings = require('../settings.js');
 
+function setupTokenAuth(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    const setupToken = safe.fs.readFileSync(paths.SETUP_TOKEN_FILE, 'utf8');
+    if (!setupToken) return next();
+
+    if (!req.body.setupToken) return next(new HttpError(400, 'setup token required'));
+    if (setupToken.trim() !== req.body.setupToken) return next(new HttpError(422, 'setup token does not match'));
+
+    return next();
+}
+
 function providerTokenAuth(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
@@ -96,7 +111,7 @@ function restore(req, res, next) {
 
     if (!req.body.backupConfig || typeof req.body.backupConfig !== 'object') return next(new HttpError(400, 'backupConfig is required'));
 
-    var backupConfig = req.body.backupConfig;
+    const backupConfig = req.body.backupConfig;
     if (typeof backupConfig.provider !== 'string') return next(new HttpError(400, 'provider is required'));
     if ('password' in backupConfig && typeof backupConfig.password !== 'string') return next(new HttpError(400, 'password must be a string'));
     if (typeof backupConfig.format !== 'string') return next(new HttpError(400, 'format must be a string'));
@@ -106,8 +121,13 @@ function restore(req, res, next) {
     if (typeof req.body.version !== 'string') return next(new HttpError(400, 'version must be a string'));
 
     if ('sysinfoConfig' in req.body && typeof req.body.sysinfoConfig !== 'object') return next(new HttpError(400, 'sysinfoConfig must be an object'));
+    if ('skipDnsSetup' in req.body && typeof req.body.skipDnsSetup !== 'boolean') return next(new HttpError(400, 'skipDnsSetup must be a boolean'));
 
-    provision.restore(backupConfig, req.body.backupId, req.body.version, req.body.sysinfoConfig || { provider: 'generic' }, auditSource.fromRequest(req), function (error) {
+    const options = {
+        skipDnsSetup: req.body.skipDnsSetup || false
+    };
+
+    provision.restore(backupConfig, req.body.backupId, req.body.version, req.body.sysinfoConfig || { provider: 'generic' }, options, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));

src/routes/services.js

@@ -1,24 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    getAll: getAll,
-    get: get,
-    configure: configure,
-    getLogs: getLogs,
-    getLogStream: getLogStream,
-    restart: restart
+    getAll,
+    get,
+    configure,
+    getLogs,
+    getLogStream,
+    restart,
+    rebuild
 };
 
-var addons = require('../addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess;
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    services = require('../services.js');
 
 function getAll(req, res, next) {
-    req.clearTimeout(); // can take a while to get status of all services
-
-    addons.getServices(function (error, result) {
+    services.getServiceIds(function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { services: result }));
@@ -28,7 +27,9 @@ function getAll(req, res, next) {
 function get(req, res, next) {
     assert.strictEqual(typeof req.params.service, 'string');
 
-    addons.getService(req.params.service, function (error, result) {
+    req.clearTimeout();
+
+    services.getServiceStatus(req.params.service, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { service: result }));
@@ -38,12 +39,10 @@ function get(req, res, next) {
 function configure(req, res, next) {
     assert.strictEqual(typeof req.params.service, 'string');
 
-    if (typeof req.body.memory !== 'number') return next(new HttpError(400, 'memory must be a number'));
-    if (typeof req.body.memorySwap !== 'number') return next(new HttpError(400, 'memorySwap must be a number'));
+    if (typeof req.body.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit must be a number'));
 
     const data = {
-        memory: req.body.memory,
-        memorySwap: req.body.memorySwap
+        memoryLimit: req.body.memoryLimit
     };
 
     if (req.params.service === 'sftp') {
@@ -51,7 +50,7 @@ function configure(req, res, next) {
         data.requireAdmin = req.body.requireAdmin;
     }
 
-    addons.configureService(req.params.service, data, function (error) {
+    services.configureService(req.params.service, data, function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, {}));
@@ -70,7 +69,7 @@ function getLogs(req, res, next) {
         format: req.query.format || 'json'
     };
 
-    addons.getServiceLogs(req.params.service, options, function (error, logStream) {
+    services.getServiceLogs(req.params.service, options, function (error, logStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.writeHead(200, {
@@ -100,7 +99,7 @@ function getLogStream(req, res, next) {
         format: 'json'
     };
 
-    addons.getServiceLogs(req.params.service, options, function (error, logStream) {
+    services.getServiceLogs(req.params.service, options, function (error, logStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.writeHead(200, {
@@ -124,7 +123,17 @@ function getLogStream(req, res, next) {
 function restart(req, res, next) {
     assert.strictEqual(typeof req.params.service, 'string');
 
-    addons.restartService(req.params.service, function (error) {
+    services.restartService(req.params.service, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
+function rebuild(req, res, next) {
+    assert.strictEqual(typeof req.params.service, 'string');
+
+    services.rebuildService(req.params.service, function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, {}));

src/routes/settings.js

@@ -116,32 +116,6 @@ function setBackupConfig(req, res, next) {
     });
 }
 
-function getPlatformConfig(req, res, next) {
-    settings.getPlatformConfig(function (error, config) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, config));
-    });
-}
-
-function setPlatformConfig(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    for (let addon of [ 'mysql', 'postgresql', 'mail', 'mongodb' ]) {
-        if (!(addon in req.body)) continue;
-        if (typeof req.body[addon] !== 'object') return next(new HttpError(400, 'addon config must be an object'));
-
-        if (typeof req.body[addon].memory !== 'number') return next(new HttpError(400, 'memory must be a number'));
-        if (typeof req.body[addon].memorySwap !== 'number') return next(new HttpError(400, 'memorySwap must be a number'));
-    }
-
-    settings.setPlatformConfig(req.body, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, {}));
-    });
-}
-
 function getExternalLdapConfig(req, res, next) {
     settings.getExternalLdapConfig(function (error, config) {
         if (error) return next(BoxError.toHttpError(error));
@@ -220,10 +194,13 @@ function getRegistryConfig(req, res, next) {
 function setRegistryConfig(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (typeof req.body.serverAddress !== 'string') return next(new HttpError(400, 'serverAddress is required'));
-    if ('username' in req.body && typeof req.body.username !== 'string') return next(new HttpError(400, 'username is required'));
-    if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email is required'));
-    if ('password' in req.body && typeof req.body.password !== 'string') return next(new HttpError(400, 'password is required'));
+    if (!req.body.provider || typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider is required'));
+    if (req.body.provider !== 'noop') {
+        if (typeof req.body.serverAddress !== 'string') return next(new HttpError(400, 'serverAddress is required'));
+        if ('username' in req.body && typeof req.body.username !== 'string') return next(new HttpError(400, 'username is required'));
+        if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email is required'));
+        if ('password' in req.body && typeof req.body.password !== 'string') return next(new HttpError(400, 'password is required'));
+    }
 
     settings.setRegistryConfig(req.body, function (error) {
         if (error) return next(BoxError.toHttpError(error));
@@ -299,7 +276,6 @@ function get(req, res, next) {
     switch (req.params.setting) {
     case settings.DYNAMIC_DNS_KEY: return getDynamicDnsConfig(req, res, next);
     case settings.BACKUP_CONFIG_KEY: return getBackupConfig(req, res, next);
-    case settings.PLATFORM_CONFIG_KEY: return getPlatformConfig(req, res, next);
     case settings.EXTERNAL_LDAP_KEY: return getExternalLdapConfig(req, res, next);
     case settings.UNSTABLE_APPS_KEY: return getUnstableAppsConfig(req, res, next);
     case settings.REGISTRY_CONFIG_KEY: return getRegistryConfig(req, res, next);
@@ -321,7 +297,6 @@ function set(req, res, next) {
 
     switch (req.params.setting) {
     case settings.DYNAMIC_DNS_KEY: return setDynamicDnsConfig(req, res, next);
-    case settings.PLATFORM_CONFIG_KEY: return setPlatformConfig(req, res, next);
     case settings.EXTERNAL_LDAP_KEY: return setExternalLdapConfig(req, res, next);
     case settings.UNSTABLE_APPS_KEY: return setUnstableAppsConfig(req, res, next);
     case settings.REGISTRY_CONFIG_KEY: return setRegistryConfig(req, res, next);

src/routes/test/groups-test.js

@@ -145,6 +145,16 @@ describe('Groups API', function () {
             });
     });
 
+    it('cannot set duplicate groups for a user', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
+            .query({ access_token: token })
+            .send({ groupIds: [ groupObject.id, group1Object.id, groupObject.id ]})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(409);
+                done();
+            });
+    });
+
     it('can set users of a group', function (done) {
         superagent.put(SERVER_URL + '/api/v1/groups/' + groupObject.id + '/members')
             .query({ access_token: token })
@@ -155,6 +165,17 @@ describe('Groups API', function () {
             });
     });
 
+    it('cannot set duplicate users of a group', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/groups/' + groupObject.id + '/members')
+            .query({ access_token: token })
+            .send({ userIds: [ userId, userId_1, userId ]})
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(409);
+                done();
+            });
+    });
+
+
     it('cannot get non-existing group', function (done) {
         superagent.get(SERVER_URL + '/api/v1/groups/nope')
             .query({ access_token: token })
@@ -180,8 +201,8 @@ describe('Groups API', function () {
                 expect(result.statusCode).to.equal(200);
                 expect(result.body.name).to.be(groupObject.name);
                 expect(result.body.userIds.length).to.be(2);
-                expect(result.body.userIds[0]).to.be(userId);
-                expect(result.body.userIds[1]).to.be(userId_1);
+                expect(result.body.userIds).to.contain(userId);
+                expect(result.body.userIds).to.contain(userId_1);
                 done();
             });
     });

src/routes/test/mail-test.js

@@ -618,8 +618,7 @@ describe('Mail API', function () {
                     expect(res.body.mailboxes[0].name).to.equal(MAILBOX_NAME);
                     expect(res.body.mailboxes[0].ownerId).to.equal(userId);
                     expect(res.body.mailboxes[0].ownerType).to.equal('user');
-                    expect(res.body.mailboxes[0].aliasName).to.equal(null);
-                    expect(res.body.mailboxes[0].aliasDomain).to.equal(null);
+                    expect(res.body.mailboxes[0].aliases).to.eql([]);
                     expect(res.body.mailboxes[0].domain).to.equal(DOMAIN_0.domain);
                     done();
                 });

src/routes/test/users-test.js

@@ -932,5 +932,44 @@ describe('Users API', function () {
                 });
         });
     });
+
+    describe('transfer ownership', function () {
+
+        before(function (done) {
+            superagent.post(SERVER_URL + '/api/v1/users')
+                .query({ access_token: token })
+                .send({ username: USERNAME_1, email: EMAIL_1 })
+                .end(function (error, result) {
+                    expect(error).to.not.be.ok();
+                    expect(result.statusCode).to.equal(201);
+
+                    user_1 = result.body;
+                    token_1 = hat(8 * 32);
+
+                    // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+                    tokendb.add({ id: 'tid-3', accessToken: token_1, identifier: user_1.id, clientId: 'test-client-id', expires: Date.now() + 10000, scope: 'unused', name: 'fromtest' }, done);
+                });
+        });
+
+        it('succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/users/' + user_1.id + '/make_owner')
+                .query({ access_token: token })
+                .send({})
+                .end(function (error, result) {
+                    expect(error).to.not.be.ok();
+                    expect(result.statusCode).to.equal(204);
+
+                    superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
+                        .query({ access_token: token_1 })
+                        .end(function (error, result) {
+                            expect(error).to.not.be.ok();
+                            expect(result.statusCode).to.equal(200);
+                            expect(result.body.role).to.equal('user');
+
+                            done();
+                    });
+            });
+        });
+    });
 });
 

src/routes/test/volumes-test.js

@@ -55,7 +55,7 @@ describe('Volumes API', function () {
     it('cannot create volume with bad name', function (done) {
         superagent.post(SERVER_URL + '/api/v1/volumes')
             .query({ access_token: token })
-            .send({ name: 'music#/ ', hostPath: '/media/music' })
+            .send({ name: 'music#/ ', hostPath: '/media/cloudron-test-music' })
             .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -75,7 +75,7 @@ describe('Volumes API', function () {
     it('can create volume', function (done) {
         superagent.post(SERVER_URL + '/api/v1/volumes')
             .query({ access_token: token })
-            .send({ name: 'music', hostPath: '/media/music' })
+            .send({ name: 'music', hostPath: '/media/cloudron-test-music' })
             .end(function (err, res) {
                 expect(res.statusCode).to.equal(201);
                 expect(res.body.id).to.be.a('string');
@@ -91,7 +91,7 @@ describe('Volumes API', function () {
                 expect(res.statusCode).to.equal(200);
                 expect(res.body.volumes.length).to.be(1);
                 expect(res.body.volumes[0].id).to.be(volumeId);
-                expect(res.body.volumes[0].hostPath).to.be('/media/music');
+                expect(res.body.volumes[0].hostPath).to.be('/media/cloudron-test-music');
                 done();
             });
     });
@@ -111,7 +111,7 @@ describe('Volumes API', function () {
             .end(function (err, res) {
                 expect(res.statusCode).to.equal(200);
                 expect(res.body.id).to.be(volumeId);
-                expect(res.body.hostPath).to.be('/media/music');
+                expect(res.body.hostPath).to.be('/media/cloudron-test-music');
                 done();
             });
     });

src/routes/users.js

@@ -13,6 +13,7 @@ exports = module.exports = {
     setGroups,
     setAvatar,
     clearAvatar,
+    makeOwner,
 
     load
 };
@@ -216,3 +217,19 @@ function clearAvatar(req, res, next) {
         next(new HttpSuccess(202, {}));
     });
 }
+
+// This route transfers ownership from token user to user specified in path param
+function makeOwner(req, res, next) {
+    assert.strictEqual(typeof req.resource, 'object');
+
+    // first make new one owner, then devote current one
+    users.update(req.resource, { role: users.ROLE_OWNER }, auditSource.fromRequest(req), function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        users.update(req.user, { role: users.ROLE_USER }, auditSource.fromRequest(req), function (error) {
+            if (error) return next(BoxError.toHttpError(error));
+
+            next(new HttpSuccess(204));
+        });
+    });
+}

src/routes/wellknown.js

@@ -0,0 +1,19 @@
+'use strict';
+
+exports = module.exports = {
+    get
+};
+
+const HttpError = require('connect-lastmile').HttpError,
+    wellknown = require('../wellknown.js');
+
+function get(req, res, next) {
+    const host = req.headers['host'];
+    const location = req.params[0];
+
+    wellknown.get(host, location, function (error, result) {
+        if (error) return next(new HttpError(404, error.message));
+
+        res.status(200).set('content-type', result.type).send(result.body);
+    });
+}

src/scheduler.js

@@ -76,10 +76,10 @@ function createJobs(app, schedulerConfig, callback) {
         // stopJobs only deletes jobs since previous run. This means that when box code restarts, none of the containers
         // are removed. The deleteContainer here ensures we re-create the cron containers with the latest config
         docker.deleteContainer(containerName, function ( /* ignoredError */) {
-            docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], { } /* options */, function (error, container) {
+            docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], { } /* options */, function (error) {
                 if (error && error.reason !== BoxError.ALREADY_EXISTS) return iteratorDone(error);
 
-                debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${container.id}`);
+                debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`);
 
                 var cronJob = new CronJob({
                     cronTime: cronTime, // at this point, the pattern has been validated

src/scripts/reloadnginx.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-if [[ $# == 1 && "$1" == "--check" ]]; then
-    echo "OK"
-    exit 0
-fi
-
-if [[ "${BOX_ENV}" == "cloudron" ]]; then
-    nginx -s reload
-fi

src/scripts/restartdocker.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-if [[ $# == 1 && "$1" == "--check" ]]; then
-    echo "OK"
-    exit 0
-fi
-
-if [[ "${BOX_ENV}" == "cloudron" ]]; then
-    systemctl restart docker
-fi
-

src/scripts/restartservice.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
+if [[ $# -eq 0 ]]; then
+    echo "No arguments supplied"
+    exit 1
+fi
+
+if [[ "$1" == "--check" ]]; then
+    echo "OK"
+    exit 0
+fi
+
+[[ "${BOX_ENV}" != "cloudron" ]] && exit
+
+service="$1"
+
+if [[ "${service}" == "unbound" ]]; then
+    unbound-anchor -a /var/lib/unbound/root.key
+    systemctl restart unbound
+elif [[ "${service}" == "nginx" ]]; then
+    nginx -s reload
+elif [[ "${service}" == "docker" ]]; then
+    systemctl restart docker
+elif [[ "${service}" == "collectd" ]]; then
+    systemctl restart collectd
+else
+    echo "Unknown service ${service}"
+    exit 1
+fi
+

src/scripts/restartunbound.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-if [[ $# == 1 && "$1" == "--check" ]]; then
-    echo "OK"
-    exit 0
-fi
-
-if [[ "${BOX_ENV}" == "cloudron" ]]; then
-    unbound-anchor -a /var/lib/unbound/root.key
-    systemctl restart unbound
-fi
-

src/scripts/retire.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-readonly BOX_SRC_DIR=/home/yellowtent/box
-
-if [[ $# == 1 && "$1" == "--check" ]]; then
-    echo "OK"
-    exit 0
-fi
-
-echo "Retiring cloudron"
-
-if [[ "${BOX_ENV}" != "cloudron" ]]; then
-	exit 0
-fi
-
-echo "Stopping apps"
-systemctl stop docker # stop the apps
-
-# do this at the end since stopping the box will kill this script as well
-echo "Stopping Cloudron Smartserver"
-"${BOX_SRC_DIR}/setup/stop.sh"

src/scripts/rmaddondir.sh

@@ -19,13 +19,17 @@ fi
 
 addon="$1"
 appid="${2:-}"   # only valid for redis
-if [[ "${addon}" != "postgresql" && "${addon}" != "mysql" && "${addon}" != "mongodb" && "${addon}" != "redis" ]]; then
-    echo "${addon} must be postgresql/mysql/mongodb/redis"
+if [[ "${addon}" != "postgresql" && "${addon}" != "mysql" && "${addon}" != "mongodb" && "${addon}" != "redis" && "${addon}" != "graphite" ]]; then
+    echo "${addon} must be postgresql/mysql/mongodb/redis/graphite"
     exit 1
 fi
 
 if [[ "${BOX_ENV}" == "cloudron" ]]; then
-    readonly addon_dir="${HOME}/platformdata/${addon}/${appid}"
+    if [[ "${addon}" == "graphite" ]]; then
+        readonly addon_dir="${HOME}/platformdata/graphite"
+    else
+        readonly addon_dir="${HOME}/platformdata/${addon}/${appid}"
+    fi
 else
     readonly addon_dir="${HOME}/.cloudron_test/platformdata/${addon}/${appid}"
 fi

src/server.js

@@ -86,19 +86,19 @@ function initializeExpressSync() {
     const authorizeUserManager = routes.accesscontrol.authorize(users.ROLE_USER_MANAGER);
 
     // public routes
-    router.post('/api/v1/cloudron/setup',    json, routes.provision.providerTokenAuth, routes.provision.setup);    // only available until no-domain
-    router.post('/api/v1/cloudron/restore',  json, routes.provision.restore);    // only available until activated
-    router.post('/api/v1/cloudron/activate', json, routes.provision.activate);
+    router.post('/api/v1/cloudron/setup',    json, routes.provision.setupTokenAuth, routes.provision.providerTokenAuth, routes.provision.setup);    // only available until no-domain
+    router.post('/api/v1/cloudron/restore',  json, routes.provision.setupTokenAuth, routes.provision.restore);    // only available until activated
+    router.post('/api/v1/cloudron/activate', json, routes.provision.setupTokenAuth, routes.provision.activate);
     router.get ('/api/v1/cloudron/status',         routes.provision.getStatus);
     router.get ('/api/v1/cloudron/languages',      routes.cloudron.getLanguages);
     router.get ('/api/v1/cloudron/avatar',         routes.branding.getCloudronAvatar); // this is a public alias for /api/v1/branding/cloudron_avatar
 
     // login/logout routes
     router.post('/api/v1/cloudron/login',                  json, password, routes.cloudron.login);
-    router.get ('/api/v1/cloudron/logout',                       routes.cloudron.logout); // this will invalidate the token if any and redirect to /login.html always
-    router.post('/api/v1/cloudron/password_reset_request', json, routes.cloudron.passwordResetRequest);
-    router.post('/api/v1/cloudron/password_reset',         json, routes.cloudron.passwordReset);
-    router.post('/api/v1/cloudron/setup_account',          json, routes.cloudron.setupAccount);
+    router.get ('/api/v1/cloudron/logout',                       token,    routes.cloudron.logout); // this will invalidate the token if any and redirect to /login.html always
+    router.post('/api/v1/cloudron/password_reset_request', json,           routes.cloudron.passwordResetRequest);
+    router.post('/api/v1/cloudron/password_reset',         json,           routes.cloudron.passwordReset);
+    router.post('/api/v1/cloudron/setup_account',          json,           routes.cloudron.setupAccount);
 
     // developer routes
     router.post('/api/v1/developer/login', json, password, routes.cloudron.login); // DEPRECATED we should use the regular /api/v1/cloudron/login
@@ -109,6 +109,7 @@ function initializeExpressSync() {
     router.post('/api/v1/cloudron/prepare_dashboard_domain', json, token, authorizeAdmin, routes.cloudron.prepareDashboardDomain);
     router.post('/api/v1/cloudron/set_dashboard_domain',     json, token, authorizeAdmin, routes.cloudron.updateDashboardDomain);
     router.post('/api/v1/cloudron/renew_certs',              json, token, authorizeAdmin, routes.cloudron.renewCerts);
+    router.post('/api/v1/cloudron/sync_dns',                 json, token, authorizeAdmin, routes.cloudron.syncDnsRecords);
     router.post('/api/v1/cloudron/check_for_updates',        json, token, authorizeAdmin, routes.cloudron.checkForUpdates);
     router.get ('/api/v1/cloudron/reboot',                         token, authorizeAdmin, routes.cloudron.isRebootRequired);
     router.post('/api/v1/cloudron/reboot',                   json, token, authorizeAdmin, routes.cloudron.reboot);
@@ -174,6 +175,7 @@ function initializeExpressSync() {
     router.post('/api/v1/users/:userId',               json, token, authorizeUserManager, routes.users.load, routes.users.update);
     router.post('/api/v1/users/:userId/password',      json, token, authorizeUserManager, routes.users.load, routes.users.changePassword);
     router.put ('/api/v1/users/:userId/groups',        json, token, authorizeUserManager, routes.users.load, routes.users.setGroups);
+    router.post('/api/v1/users/:userId/make_owner',    json, token, authorizeOwner,       routes.users.load, routes.users.makeOwner);
     router.post('/api/v1/users/:userId/send_invite',   json, token, authorizeUserManager, routes.users.load, routes.users.sendInvite);
     router.post('/api/v1/users/:userId/create_invite', json, token, authorizeUserManager, routes.users.load, routes.users.createInvite);
     router.post('/api/v1/users/:userId/avatar',        json, token, authorizeUserManager, routes.users.load, multipart, routes.users.setAvatar);
@@ -234,6 +236,7 @@ function initializeExpressSync() {
     router.post('/api/v1/apps/:id/upload',                       json, token, authorizeAdmin, multipart, routes.apps.load, routes.apps.uploadFile);
     router.use ('/api/v1/apps/:id/files/*',                            token, authorizeAdmin, routes.filemanager.proxy);
     router.get ('/api/v1/apps/:id/exec',                               token, authorizeAdmin, routes.apps.load, routes.apps.exec);
+
     // websocket cannot do bearer authentication
     router.get ('/api/v1/apps/:id/execws',    routes.accesscontrol.websocketAuth.bind(null, users.ROLE_ADMIN), routes.apps.load, routes.apps.execWebSocket);
 
@@ -273,7 +276,6 @@ function initializeExpressSync() {
     router.post('/api/v1/mail/:domain/catch_all',               json, token, authorizeAdmin, routes.mail.setCatchAllAddress);
     router.post('/api/v1/mail/:domain/relay',                   json, token, authorizeAdmin, routes.mail.setMailRelay);
     router.post('/api/v1/mail/:domain/enable',                  json, token, authorizeAdmin, routes.mail.setMailEnabled);
-    router.post('/api/v1/mail/:domain/dns',                     json, token, authorizeAdmin, routes.mail.setDnsRecords);
     router.post('/api/v1/mail/:domain/banner',                  json, token, authorizeAdmin, routes.mail.setBanner);
     router.post('/api/v1/mail/:domain/send_test_mail',          json, token, authorizeAdmin, routes.mail.sendTestMail);
     router.get ('/api/v1/mail/:domain/mailbox_count',                 token, authorizeAdmin, routes.mail.getMailboxCount);
@@ -296,12 +298,12 @@ function initializeExpressSync() {
     router.post('/api/v1/support/remote_support', json, token, authorizeAdmin, routes.support.canEnableRemoteSupport, routes.support.enableRemoteSupport);
 
     // domain routes
-    router.post('/api/v1/domains',                   json, token, authorizeAdmin, routes.domains.add);
-    router.get ('/api/v1/domains',                         token,                 routes.domains.getAll);
-    router.get ('/api/v1/domains/:domain',                 token, authorizeAdmin, routes.domains.get);  // this is manage scope because it returns non-restricted fields
-    router.put ('/api/v1/domains/:domain',           json, token, authorizeAdmin, routes.domains.update);
-    router.del ('/api/v1/domains/:domain',                 token, authorizeAdmin, routes.domains.del);
-    router.get ('/api/v1/domains/:domain/dns_check',       token, authorizeAdmin, routes.domains.checkDnsRecords);
+    router.post('/api/v1/domains',                    json, token, authorizeAdmin, routes.domains.add);
+    router.get ('/api/v1/domains',                          token,                 routes.domains.getAll);
+    router.get ('/api/v1/domains/:domain',                  token, authorizeAdmin, routes.domains.get);  // this is manage scope because it returns non-restricted fields
+    router.put ('/api/v1/domains/:domain',            json, token, authorizeAdmin, routes.domains.update);
+    router.del ('/api/v1/domains/:domain',                  token, authorizeAdmin, routes.domains.del);
+    router.get ('/api/v1/domains/:domain/dns_check',        token, authorizeAdmin, routes.domains.checkDnsRecords);
 
     // volume routes
     router.post('/api/v1/volumes',                   json, token, authorizeAdmin, routes.volumes.add);
@@ -310,13 +312,17 @@ function initializeExpressSync() {
     router.del ('/api/v1/volumes/:id',                     token, authorizeAdmin, routes.volumes.load, routes.volumes.del);
     router.use ('/api/v1/volumes/:id/files/*',             token, authorizeAdmin, routes.filemanager.proxy);
 
-    // addon routes
+    // service routes
     router.get ('/api/v1/services',                          token, authorizeAdmin, routes.services.getAll);
     router.get ('/api/v1/services/:service',                 token, authorizeAdmin, routes.services.get);
     router.post('/api/v1/services/:service',           json, token, authorizeAdmin, routes.services.configure);
     router.get ('/api/v1/services/:service/logs',            token, authorizeAdmin, routes.services.getLogs);
     router.get ('/api/v1/services/:service/logstream',       token, authorizeAdmin, routes.services.getLogStream);
     router.post('/api/v1/services/:service/restart',   json, token, authorizeAdmin, routes.services.restart);
+    router.post('/api/v1/services/:service/rebuild',   json, token, authorizeAdmin, routes.services.rebuild);
+
+    // well known
+    router.get ('/well-known-handler/*', routes.wellknown.get);
 
     // disable server socket "idle" timeout. we use the timeout middleware to handle timeouts on a route level
     // we rely on nginx for timeouts on the TCP level (see client_header_timeout)

src/services.js

@@ -1,11 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    getServices,
-    getService,
-    getServicesConfig,
-    configureService,
+    getServiceIds,
+    getServiceStatus,
+    getServiceConfig,
     getServiceLogs,
+
+    configureService,
     restartService,
     rebuildService,
 
@@ -13,7 +14,6 @@ exports = module.exports = {
     stopAppServices,
 
     startServices,
-    updateServiceConfig,
 
     setupAddons,
     teardownAddons,
@@ -22,7 +22,6 @@ exports = module.exports = {
     clearAddons,
 
     getEnvironment,
-    getMountsSync,
     getContainerNamesSync,
 
     getContainerDetails,
@@ -39,10 +38,9 @@ var appdb = require('./appdb.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     crypto = require('crypto'),
-    debug = require('debug')('box:addons'),
+    debug = require('debug')('box:services'),
     docker = require('./docker.js'),
     fs = require('fs'),
-    graphs = require('./graphs.js'),
     hat = require('./hat.js'),
     infra = require('./infra_version.js'),
     mail = require('./mail.js'),
@@ -59,11 +57,13 @@ var appdb = require('./appdb.js'),
     spawn = require('child_process').spawn,
     split = require('split'),
     request = require('request'),
+    system = require('./system.js'),
     util = require('util');
 
 const NOOP = function (app, options, callback) { return callback(); };
 const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 const RMADDONDIR_CMD = path.join(__dirname, 'scripts/rmaddondir.sh');
+const RESTART_SERVICE_CMD = path.join(__dirname, 'scripts/restartservice.sh');
 
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
@@ -159,6 +159,13 @@ var ADDONS = {
         restore: NOOP,
         clear: NOOP,
     },
+    tls: {
+        setup: NOOP,
+        teardown: NOOP,
+        backup: NOOP,
+        restore: NOOP,
+        clear: NOOP,
+    },
     oauth: { // kept for backward compatibility. keep teardown for uninstall to work
         setup: NOOP,
         teardown: teardownOauth,
@@ -172,27 +179,27 @@ var ADDONS = {
 const SERVICES = {
     turn: {
         status: statusTurn,
-        restart: restartContainer.bind(null, 'turn'),
+        restart: docker.restartContainer.bind(null, 'turn'),
         defaultMemoryLimit: 256 * 1024 * 1024
     },
     mail: {
         status: containerStatus.bind(null, 'mail', 'CLOUDRON_MAIL_TOKEN'),
         restart: mail.restartMail,
-        defaultMemoryLimit: Math.max((1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 128, 256) * 1024 * 1024
+        defaultMemoryLimit: mail.DEFAULT_MEMORY_LIMIT
     },
     mongodb: {
         status: containerStatus.bind(null, 'mongodb', 'CLOUDRON_MONGODB_TOKEN'),
-        restart: restartContainer.bind(null, 'mongodb'),
+        restart: docker.restartContainer.bind(null, 'mongodb'),
         defaultMemoryLimit: (1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 256 * 1024 * 1024
     },
     mysql: {
         status: containerStatus.bind(null, 'mysql', 'CLOUDRON_MYSQL_TOKEN'),
-        restart: restartContainer.bind(null, 'mysql'),
+        restart: docker.restartContainer.bind(null, 'mysql'),
         defaultMemoryLimit: (1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 256 * 1024 * 1024
     },
     postgresql: {
         status: containerStatus.bind(null, 'postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'),
-        restart: restartContainer.bind(null, 'postgresql'),
+        restart: docker.restartContainer.bind(null, 'postgresql'),
         defaultMemoryLimit: (1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 256 * 1024 * 1024
     },
     docker: {
@@ -207,13 +214,13 @@ const SERVICES = {
     },
     sftp: {
         status: statusSftp,
-        restart: restartContainer.bind(null, 'sftp'),
-        defaultMemoryLimit: 256 * 1024 * 1024
+        restart: docker.restartContainer.bind(null, 'sftp'),
+        defaultMemoryLimit: sftp.DEFAULT_MEMORY_LIMIT
     },
     graphite: {
         status: statusGraphite,
-        restart: restartContainer.bind(null, 'graphite'),
-        defaultMemoryLimit: 75 * 1024 * 1024
+        restart: restartGraphite,
+        defaultMemoryLimit: 256 * 1024 * 1024
     },
     nginx: {
         status: statusNginx,
@@ -227,7 +234,7 @@ const APP_SERVICES = {
         status: (instance, done) => containerStatus(`redis-${instance}`, 'CLOUDRON_REDIS_TOKEN', done),
         start: (instance, done) => docker.startContainer(`redis-${instance}`, done),
         stop: (instance, done) => docker.stopContainer(`redis-${instance}`, done),
-        restart: (instance, done) => restartContainer(`redis-${instance}`, done),
+        restart: (instance, done) => docker.restartContainer(`redis-${instance}`, done),
         defaultMemoryLimit: 150 * 1024 * 1024
     }
 };
@@ -262,38 +269,6 @@ function dumpPath(addon, appId) {
     }
 }
 
-function rebuildService(serviceName, callback) {
-    assert.strictEqual(typeof serviceName, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    // this attempts to recreate the service docker container if they don't exist but platform infra version is unchanged
-    // passing an infra version of 'none' will not attempt to purge existing data, not sure if this is good or bad
-    if (serviceName === 'turn') return startTurn({ version: 'none' }, callback);
-    if (serviceName === 'mongodb') return startMongodb({ version: 'none' }, callback);
-    if (serviceName === 'postgresql') return startPostgresql({ version: 'none' }, callback);
-    if (serviceName === 'mysql') return startMysql({ version: 'none' }, callback);
-    if (serviceName === 'sftp') return sftp.startSftp({ version: 'none' }, callback);
-    if (serviceName === 'graphite') return graphs.startGraphite({ version: 'none' }, callback);
-
-    // nothing to rebuild for now
-    callback();
-}
-
-function restartContainer(name, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    docker.restartContainer(name, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) {
-            callback(null); // callback early since rebuilding takes long
-            return rebuildService(name, function (error) { if (error) debug(`restartContainer: Unable to rebuild service ${name}`, error); });
-        }
-        if (error) return callback(error);
-
-        callback(error);
-    });
-}
-
 function getContainerDetails(containerName, tokenEnvName, callback) {
     assert.strictEqual(typeof containerName, 'string');
     assert.strictEqual(typeof tokenEnvName, 'string');
@@ -326,7 +301,7 @@ function containerStatus(containerName, tokenEnvName, callback) {
         if (error && (error.reason === BoxError.NOT_FOUND || error.reason === BoxError.INACTIVE)) return callback(null, { status: exports.SERVICE_STATUS_STOPPED });
         if (error) return callback(error);
 
-        request.get(`https://${addonDetails.ip}:3000/healthcheck?access_token=${addonDetails.token}`, { json: true, rejectUnauthorized: false, timeout: 3000 }, function (error, response) {
+        request.get(`https://${addonDetails.ip}:3000/healthcheck?access_token=${addonDetails.token}`, { json: true, rejectUnauthorized: false, timeout: 20000 }, function (error, response) {
             if (error) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for ${containerName}: ${error.message}` });
             if (response.statusCode !== 200 || !response.body.status) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}` });
 
@@ -346,32 +321,32 @@ function containerStatus(containerName, tokenEnvName, callback) {
     });
 }
 
-function getServices(callback) {
+function getServiceIds(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    let services = Object.keys(SERVICES);
+    let serviceIds = Object.keys(SERVICES);
 
     appdb.getAll(function (error, apps) {
         if (error) return callback(error);
 
         for (let app of apps) {
-            if (app.manifest.addons && app.manifest.addons['redis']) services.push(`redis:${app.id}`);
+            if (app.manifest.addons && app.manifest.addons['redis']) serviceIds.push(`redis:${app.id}`);
         }
 
-        callback(null, services);
+        callback(null, serviceIds);
     });
 }
 
-function getServicesConfig(id, callback) {
+function getServiceConfig(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    const [name, instance ] = id.split(':');
+    const [name, instance] = id.split(':');
     if (!instance) {
-        settings.getPlatformConfig(function (error, platformConfig) {
+        settings.getServicesConfig(function (error, servicesConfig) {
             if (error) return callback(error);
 
-            callback(null, SERVICES[name], platformConfig);
+            callback(null, servicesConfig[name] || {});
         });
 
         return;
@@ -380,22 +355,24 @@ function getServicesConfig(id, callback) {
     appdb.get(instance, function (error, app) {
         if (error) return callback(error);
 
-        callback(null, APP_SERVICES[name], app.servicesConfig);
+        callback(null, app.servicesConfig[name] || {});
     });
 }
 
-function getService(id, callback) {
+function getServiceStatus(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     const [name, instance ] = id.split(':');
-    let containerStatusFunc;
+    let containerStatusFunc, service;
 
     if (instance) {
-        if (!APP_SERVICES[name]) return callback(new BoxError(BoxError.NOT_FOUND));
-        containerStatusFunc = APP_SERVICES[name].status.bind(null, instance);
+        service = APP_SERVICES[name];
+        if (!service) return callback(new BoxError(BoxError.NOT_FOUND));
+        containerStatusFunc = service.status.bind(null, instance);
     } else if (SERVICES[name]) {
-        containerStatusFunc = SERVICES[name].status;
+        service = SERVICES[name];
+        containerStatusFunc = service.status;
     } else {
         return callback(new BoxError(BoxError.NOT_FOUND));
     }
@@ -407,11 +384,7 @@ function getService(id, callback) {
         memoryPercent: 0,
         error: null,
         healthcheck: null,
-        config: {
-            // If a property is not set then we cannot change it through the api, see below
-            // memory: 0,
-            // memorySwap: 0
-        }
+        config: {}
     };
 
     containerStatusFunc(function (error, result) {
@@ -423,15 +396,13 @@ function getService(id, callback) {
         tmp.error = result.error || null;
         tmp.healthcheck = result.healthcheck || null;
 
-        getServicesConfig(id, function (error, service, servicesConfig) {
+        getServiceConfig(id, function (error, serviceConfig) {
             if (error) return callback(error);
 
-            const serviceConfig = servicesConfig[name];
-            tmp.config = Object.assign({}, serviceConfig);
+            tmp.config = serviceConfig;
 
-            if ((!tmp.config.memory || !tmp.config.memorySwap) && service.defaultMemoryLimit) {
-                tmp.config.memory = service.defaultMemoryLimit;
-                tmp.config.memorySwap = tmp.config.memory * 2;
+            if (!tmp.config.memoryLimit && service.defaultMemoryLimit) {
+                tmp.config.memoryLimit = service.defaultMemoryLimit;
             }
 
             callback(null, tmp);
@@ -448,37 +419,34 @@ function configureService(id, data, callback) {
 
     if (instance) {
         if (!APP_SERVICES[name]) return callback(new BoxError(BoxError.NOT_FOUND));
-    } else if (!SERVICES[name]) {
-        return callback(new BoxError(BoxError.NOT_FOUND));
-    }
 
-    getServicesConfig(id, function (error, service, servicesConfig) {
-        if (error) return callback(error);
+        apps.get(instance, function (error, app) {
+            if (error) return callback(error);
 
-        if (!servicesConfig[name]) servicesConfig[name] = {};
-
-        // if not specified we clear the entry and use defaults
-        if (!data.memory || !data.memorySwap) {
-            delete servicesConfig[name].memory;
-            delete servicesConfig[name].memorySwap;
-        } else {
+            const servicesConfig = app.servicesConfig;
             servicesConfig[name] = data;
-        }
 
-        if (instance) {
             appdb.update(instance, { servicesConfig }, function (error) {
                 if (error) return callback(error);
 
-                updateAppServiceConfig(name, instance, servicesConfig, callback);
+                applyServiceConfig(id, data, callback);
             });
-        } else {
-            settings.setPlatformConfig(servicesConfig, function (error) {
+        });
+    } else if (SERVICES[name]) {
+        settings.getServicesConfig(function (error, servicesConfig) {
+            if (error) return callback(error);
+
+            servicesConfig[name] = data;
+
+            settings.setServicesConfig(servicesConfig, function (error) {
                 if (error) return callback(error);
 
-                callback(null);
+                applyServiceConfig(id, data, callback);
             });
-        }
-    });
+        });
+    } else {
+        return callback(new BoxError(BoxError.NOT_FOUND));
+    }
 }
 
 function getServiceLogs(id, options, callback) {
@@ -559,6 +527,29 @@ function getServiceLogs(id, options, callback) {
     callback(null, transformStream);
 }
 
+function rebuildService(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // this attempts to recreate the service docker container if they don't exist but platform infra version is unchanged
+    // passing an infra version of 'none' will not attempt to purge existing data, not sure if this is good or bad
+    getServiceConfig(id, function (error, serviceConfig) {
+        if (error) return callback(error);
+
+        if (id === 'turn') return startTurn({ version: 'none' }, serviceConfig, callback);
+        if (id === 'mongodb') return startMongodb({ version: 'none' }, callback);
+        if (id === 'postgresql') return startPostgresql({ version: 'none' }, callback);
+        if (id === 'mysql') return startMysql({ version: 'none' }, callback);
+        if (id === 'sftp') return sftp.rebuild(serviceConfig, { /* options */ }, callback);
+        if (id === 'graphite') return startGraphite({ version: 'none' }, serviceConfig, callback);
+
+        // nothing to rebuild for now.
+        // TODO: mongo/postgresql/mysql need to be scaled down.
+        // TODO: missing redis container is not created
+        callback();
+    });
+}
+
 function restartService(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -568,9 +559,9 @@ function restartService(id, callback) {
     if (instance) {
         if (!APP_SERVICES[name]) return callback(new BoxError(BoxError.NOT_FOUND));
 
-        APP_SERVICES[name].restart(instance, callback);
+        return APP_SERVICES[name].restart(instance, callback);
     } else if (SERVICES[name]) {
-        SERVICES[name].restart(callback);
+        return SERVICES[name].restart(callback);
     } else {
         return callback(new BoxError(BoxError.NOT_FOUND));
     }
@@ -623,7 +614,7 @@ function waitForContainer(containerName, tokenEnvName, callback) {
         if (error) return callback(error);
 
         async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
-            request.get(`https://${result.ip}:3000/healthcheck?access_token=${result.token}`, { json: true, rejectUnauthorized: false, timeout: 3000 }, function (error, response) {
+            request.get(`https://${result.ip}:3000/healthcheck?access_token=${result.token}`, { json: true, rejectUnauthorized: false, timeout: 5000 }, function (error, response) {
                 if (error) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Network error waiting for ${containerName}: ${error.message}`));
                 if (response.statusCode !== 200 || !response.body.status) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`));
 
@@ -778,18 +769,20 @@ function exportDatabase(addon, callback) {
         return callback(null);
     }
 
-    appdb.getAll(function (error, apps) {
+    appdb.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        async.eachSeries(apps, function iterator (app, iteratorCallback) {
+        async.eachSeries(allApps, function iterator (app, iteratorCallback) {
             if (!app.manifest.addons || !(addon in app.manifest.addons)) return iteratorCallback(); // app doesn't use the addon
+            if (app.installationState === apps.ISTATE_ERROR) return iteratorCallback(); // missing db causes crash in old app addon containers
 
             debug(`exportDatabase: Exporting addon ${addon} of app ${app.id}`);
 
             ADDONS[addon].backup(app, app.manifest.addons[addon], function (error) {
                 if (error) {
                     debug(`exportDatabase: Error exporting ${addon} of app ${app.id}.`, error);
-                    return iteratorCallback(error);
+                    // for errored apps, we can ignore if export had an error
+                    return iteratorCallback(app.installationState === apps.ISTATE_ERROR ? null : error);
                 }
 
                 iteratorCallback();
@@ -807,82 +800,84 @@ function exportDatabase(addon, callback) {
     });
 }
 
-function updateServiceConfig(platformConfig, callback) {
-    assert.strictEqual(typeof platformConfig, 'object');
+function applyServiceConfig(id, serviceConfig, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof serviceConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    async.eachSeries([ 'mysql', 'postgresql', 'mail', 'mongodb', 'graphite' ], function iterator(serviceName, iteratorCallback) {
-        const containerConfig = platformConfig[serviceName];
-        let memory, memorySwap;
-        if (containerConfig && containerConfig.memory && containerConfig.memorySwap) {
-            memory = containerConfig.memory;
-            memorySwap = containerConfig.memorySwap;
-        } else {
-            memory = SERVICES[serviceName].defaultMemoryLimit;
-            memorySwap = memory * 2;
-        }
+    const [name, instance] = id.split(':');
+    let containerName, memoryLimit;
 
-        const args = `update --memory ${memory} --memory-swap ${memorySwap} ${serviceName}`.split(' ');
-        // scale back db containers, if possible. this is retried because updating memory constraints can fail
-        // with failed to write to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/xx/memory.memsw.limit_in_bytes: device or resource busy
-        async.retry({ times: 10, interval: 60 * 1000 }, function (retryCallback) {
-            shell.spawn(`updateServiceConfig(${serviceName})`, '/usr/bin/docker', args, { }, retryCallback);
-        }, iteratorCallback);
-    }, callback);
-}
+    if (instance) {
+        if (!APP_SERVICES[name]) return callback(new BoxError(BoxError.NOT_FOUND));
 
-function updateAppServiceConfig(name, instance, servicesConfig, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof instance, 'string');
-    assert.strictEqual(typeof servicesConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`updateAppServiceConfig: ${name}-${instance} ${JSON.stringify(servicesConfig)}`);
-
-    const serviceConfig = servicesConfig[name];
-    let memory, memorySwap;
-    if (serviceConfig && serviceConfig.memory && serviceConfig.memorySwap) {
-        memory = serviceConfig.memory;
-        memorySwap = serviceConfig.memorySwap;
+        containerName = `${name}-${instance}`;
+        memoryLimit = serviceConfig && serviceConfig.memoryLimit ? serviceConfig.memoryLimit : APP_SERVICES[name].defaultMemoryLimit;
+    } else if (SERVICES[name]) {
+        containerName = name;
+        memoryLimit = serviceConfig && serviceConfig.memoryLimit ? serviceConfig.memoryLimit : SERVICES[name].defaultMemoryLimit;
     } else {
-        memory = APP_SERVICES[name].defaultMemoryLimit;
-        memorySwap = memory * 2;
+        return callback(new BoxError(BoxError.NOT_FOUND));
     }
 
-    const args = `update --memory ${memory} --memory-swap ${memorySwap} ${name}-${instance}`.split(' ');
-    shell.spawn(`updateAppServiceConfig${name}`, '/usr/bin/docker', args, { }, callback);
+    debug(`updateServiceConfig: ${containerName} ${JSON.stringify(serviceConfig)}`);
+
+    const memory = system.getMemoryAllocation(memoryLimit);
+    docker.update(containerName, memory, memoryLimit, callback);
 }
 
 function startServices(existingInfra, callback) {
     assert.strictEqual(typeof existingInfra, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let startFuncs = [ ];
+    settings.getServicesConfig(function (error, servicesConfig) {
+        if (error) return callback(error);
 
-    // always start addons on any infra change, regardless of minor or major update
-    if (existingInfra.version !== infra.version) {
-        debug(`startServices: ${existingInfra.version} -> ${infra.version}. starting all services`);
-        startFuncs.push(
-            startTurn.bind(null, existingInfra),
-            startMysql.bind(null, existingInfra),
-            startPostgresql.bind(null, existingInfra),
-            startMongodb.bind(null, existingInfra),
-            startRedis.bind(null, existingInfra),
-            mail.startMail);
-    } else {
-        assert.strictEqual(typeof existingInfra.images, 'object');
+        let startFuncs = [ ];
 
-        if (infra.images.turn.tag !== existingInfra.images.turn.tag) startFuncs.push(startTurn.bind(null, existingInfra));
-        if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) startFuncs.push(startMysql.bind(null, existingInfra));
-        if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) startFuncs.push(startPostgresql.bind(null, existingInfra));
-        if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) startFuncs.push(startMongodb.bind(null, existingInfra));
-        if (infra.images.mail.tag !== existingInfra.images.mail.tag) startFuncs.push(mail.startMail);
-        if (infra.images.redis.tag !== existingInfra.images.redis.tag) startFuncs.push(startRedis.bind(null, existingInfra));
+        // always start addons on any infra change, regardless of minor or major update
+        if (existingInfra.version !== infra.version) {
+            debug(`startServices: ${existingInfra.version} -> ${infra.version}. starting all services`);
+            startFuncs.push(
+                mail.startMail, // start this first to reduce email downtime
+                startTurn.bind(null, existingInfra, servicesConfig['turn'] || {}),
+                startMysql.bind(null, existingInfra),
+                startPostgresql.bind(null, existingInfra),
+                startMongodb.bind(null, existingInfra),
+                startRedis.bind(null, existingInfra),
+                startGraphite.bind(null, existingInfra, servicesConfig['graphite'] || {}),
+                sftp.start.bind(null, existingInfra, servicesConfig['sftp'] || {}),
+            );
+        } else {
+            assert.strictEqual(typeof existingInfra.images, 'object');
 
-        debug('startServices: existing infra. incremental service create %j', startFuncs.map(function (f) { return f.name; }));
-    }
+            if (infra.images.mail.tag !== existingInfra.images.mail.tag) startFuncs.push(mail.startMail);  // start this first to reduce email downtime
+            if (infra.images.turn.tag !== existingInfra.images.turn.tag) startFuncs.push(startTurn.bind(null, existingInfra, servicesConfig['turn'] || {}));
+            if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) startFuncs.push(startMysql.bind(null, existingInfra));
+            if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) startFuncs.push(startPostgresql.bind(null, existingInfra));
+            if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) startFuncs.push(startMongodb.bind(null, existingInfra));
+            if (infra.images.redis.tag !== existingInfra.images.redis.tag) startFuncs.push(startRedis.bind(null, existingInfra));
+            if (infra.images.graphite.tag !== existingInfra.images.graphite.tag) startFuncs.push(startGraphite.bind(null, existingInfra, servicesConfig['graphite'] || {}));
+            if (infra.images.sftp.tag !== existingInfra.images.sftp.tag) startFuncs.push(sftp.start.bind(null, existingInfra, servicesConfig['sftp'] || {}));
 
-    async.series(startFuncs, callback);
+            debug('startServices: existing infra. incremental service create %j', startFuncs.map(function (f) { return f.name; }));
+        }
+
+        async.series(startFuncs, function (error) {
+            if (error) return callback(error);
+
+            // we always start db containers with unlimited memory. we then scale them down per configuration
+            let updateFuncs = [
+                applyServiceConfig.bind(null, 'mysql', servicesConfig['mysql'] || {}),
+                applyServiceConfig.bind(null, 'postgresql', servicesConfig['postgresql'] || {}),
+                applyServiceConfig.bind(null, 'mongodb', servicesConfig['mongodb'] || {}),
+            ];
+
+            async.series(updateFuncs, NOOP_CALLBACK); // it's ok if applying service configs fails
+
+            callback();
+        });
+    });
 }
 
 function getEnvironment(app, callback) {
@@ -898,31 +893,6 @@ function getEnvironment(app, callback) {
     });
 }
 
-function getMountsSync(app, addons) {
-    assert.strictEqual(typeof app, 'object');
-    assert(!addons || typeof addons === 'object');
-
-    let mounts = [ ];
-
-    if (!addons) return mounts;
-
-    for (let addon in addons) {
-        switch (addon) {
-        case 'localstorage':
-            mounts.push({
-                Target: '/app/data',
-                Source: `${app.id}-localstorage`,
-                Type: 'volume',
-                ReadOnly: false
-            });
-            break;
-        default: break;
-        }
-    }
-
-    return mounts;
-}
-
 function getContainerNamesSync(app, addons) {
     assert.strictEqual(typeof app, 'object');
     assert(!addons || typeof addons === 'object');
@@ -1577,8 +1547,9 @@ function restorePostgreSql(app, options, callback) {
     });
 }
 
-function startTurn(existingInfra, callback) {
+function startTurn(existingInfra, serviceConfig, callback) {
     assert.strictEqual(typeof existingInfra, 'object');
+    assert.strictEqual(typeof serviceConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     // get and ensure we have a turn secret
@@ -1589,7 +1560,8 @@ function startTurn(existingInfra, callback) {
     }
 
     const tag = infra.images.turn.tag;
-    const memoryLimit = 256;
+    const memoryLimit = serviceConfig.memoryLimit || SERVICES['turn'].defaultMemoryLimit;
+    const memory = system.getMemoryAllocation(memoryLimit);
     const realm = settings.adminFqdn();
 
     // this exports 3478/tcp, 5349/tls and 50000-51000/udp. note that this runs on the host network!
@@ -1600,8 +1572,8 @@ function startTurn(existingInfra, callback) {
                 --log-opt syslog-address=udp://127.0.0.1:2514 \
                 --log-opt syslog-format=rfc5424 \
                 --log-opt tag=turn \
-                -m ${memoryLimit}m \
-                --memory-swap ${memoryLimit * 2}m \
+                -m ${memory} \
+                --memory-swap ${memoryLimit} \
                 --dns 172.18.0.1 \
                 --dns-search=. \
                 -e CLOUDRON_TURN_SECRET="${turnSecret}" \
@@ -1809,6 +1781,49 @@ function restoreMongoDb(app, options, callback) {
     });
 }
 
+function startGraphite(existingInfra, serviceConfig, callback) {
+    assert.strictEqual(typeof existingInfra, 'object');
+    assert.strictEqual(typeof serviceConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const tag = infra.images.graphite.tag;
+    const memoryLimit = serviceConfig.memoryLimit || 256 * 1024 * 1024;
+    const memory = system.getMemoryAllocation(memoryLimit);
+
+    const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.graphite.tag, tag);
+
+    if (upgrading) debug('startGraphite: graphite will be upgraded');
+
+    const cmd = `docker run --restart=always -d --name="graphite" \
+        --hostname graphite \
+        --net cloudron \
+        --net-alias graphite \
+        --log-driver syslog \
+        --log-opt syslog-address=udp://127.0.0.1:2514 \
+        --log-opt syslog-format=rfc5424 \
+        --log-opt tag=graphite \
+        -m ${memory} \
+        --memory-swap ${memoryLimit} \
+        --dns 172.18.0.1 \
+        --dns-search=. \
+        -p 127.0.0.1:2003:2003 \
+        -p 127.0.0.1:2004:2004 \
+        -p 127.0.0.1:8417:8000 \
+        -v "${paths.PLATFORM_DATA_DIR}/graphite:/var/lib/graphite" \
+        --label isCloudronManaged=true \
+        --read-only -v /tmp -v /run "${tag}"`;
+
+    async.series([
+        shell.exec.bind(null, 'stopGraphite', 'docker stop graphite || true'),
+        shell.exec.bind(null, 'removeGraphite', 'docker rm -f graphite || true'),
+        (done) => {
+            if (!upgrading) return done();
+            shell.sudo('removeGraphiteDir', [ RMADDONDIR_CMD, 'graphite' ], {}, done);
+        },
+        shell.exec.bind(null, 'startGraphite', cmd)
+    ], callback);
+}
+
 function setupProxyAuth(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
@@ -1877,7 +1892,8 @@ function setupRedis(app, options, callback) {
         const redisServiceToken = hat(4 * 48);
 
         // Compute redis memory limit based on app's memory limit (this is arbitrary)
-        const memoryLimit = app.servicesConfig['redis'] ? app.servicesConfig['redis'].memory : APP_SERVICES['redis'].defaultMemoryLimit;
+        const memoryLimit = app.servicesConfig['redis'] ? app.servicesConfig['redis'].memoryLimit : APP_SERVICES['redis'].defaultMemoryLimit;
+        const memory = system.getMemoryAllocation(memoryLimit);
 
         const tag = infra.images.redis.tag;
         const label = app.fqdn;
@@ -1891,7 +1907,7 @@ function setupRedis(app, options, callback) {
                     --log-opt syslog-address=udp://127.0.0.1:2514 \
                     --log-opt syslog-format=rfc5424 \
                     --log-opt tag="${redisName}" \
-                    -m ${memoryLimit/2} \
+                    -m ${memory} \
                     --memory-swap ${memoryLimit} \
                     --dns 172.18.0.1 \
                     --dns-search=. \
@@ -2047,7 +2063,7 @@ function statusDocker(callback) {
 function restartDocker(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    shell.sudo('restartdocker', [ path.join(__dirname, 'scripts/restartdocker.sh') ], {}, NOOP_CALLBACK);
+    shell.sudo('restartdocker', [ RESTART_SERVICE_CMD, 'docker' ], {}, NOOP_CALLBACK);
 
     callback(null);
 }
@@ -2063,7 +2079,7 @@ function statusUnbound(callback) {
 function restartUnbound(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    shell.sudo('restartunbound', [ path.join(__dirname, 'scripts/restartunbound.sh') ], {}, NOOP_CALLBACK);
+    shell.sudo('restartunbound', [ RESTART_SERVICE_CMD, 'unbound' ], {}, NOOP_CALLBACK);
 
     callback(null);
 }
@@ -2079,7 +2095,7 @@ function statusNginx(callback) {
 function restartNginx(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    shell.sudo('reloadnginx', [ path.join(__dirname, 'scripts/reloadnginx.sh') ], {}, NOOP_CALLBACK);
+    shell.sudo('restartnginx', [ RESTART_SERVICE_CMD, 'nginx' ], {}, NOOP_CALLBACK);
 
     callback(null);
 }
@@ -2112,7 +2128,7 @@ function statusGraphite(callback) {
         if (error && error.reason === BoxError.NOT_FOUND) return callback(null, { status: exports.SERVICE_STATUS_STOPPED });
         if (error) return callback(error);
 
-        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { json: true, timeout: 3000 }, function (error, response) {
+        request.get('http://127.0.0.1:8417/graphite-web/dashboard', { json: true, timeout: 20000 }, function (error, response) {
             if (error) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite: ${error.message}` });
             if (response.statusCode !== 200) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite. Status code: ${response.statusCode} message: ${response.body.message}` });
 
@@ -2131,6 +2147,17 @@ function statusGraphite(callback) {
     });
 }
 
+function restartGraphite(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.restartContainer('graphite', callback);
+
+    setTimeout(function () {
+        // wait for graphite to startup and then restart collectd
+        shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {}, NOOP_CALLBACK);
+    }, 10000);
+}
+
 function teardownOauth(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');

src/settings.js

@@ -23,8 +23,8 @@ exports = module.exports = {
     setBackupConfig,
     setBackupCredentials,
 
-    getPlatformConfig,
-    setPlatformConfig,
+    getServicesConfig,
+    setServicesConfig,
 
     getExternalLdapConfig,
     setExternalLdapConfig,
@@ -83,7 +83,7 @@ exports = module.exports = {
 
     // json. if you add an entry here, be sure to fix getAll
     BACKUP_CONFIG_KEY: 'backup_config',
-    PLATFORM_CONFIG_KEY: 'platform_config',
+    SERVICES_CONFIG_KEY: 'services_config',
     EXTERNAL_LDAP_KEY: 'external_ldap_config',
     REGISTRY_CONFIG_KEY: 'registry_config',
     SYSINFO_CONFIG_KEY: 'sysinfo_config',
@@ -117,10 +117,7 @@ exports = module.exports = {
     _setApiServerOrigin: setApiServerOrigin
 };
 
-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
-var addons = require('./addons.js'),
-    assert = require('assert'),
+const assert = require('assert'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
@@ -157,12 +154,14 @@ let gDefaults = (function () {
         retentionPolicy: { keepWithinSecs: 2 * 24 * 60 * 60 }, // 2 days
         schedulePattern: '00 00 23 * * *' // every day at 11pm
     };
-    result[exports.PLATFORM_CONFIG_KEY] = {};
+    result[exports.SERVICES_CONFIG_KEY] = {};
     result[exports.EXTERNAL_LDAP_KEY] = {
         provider: 'noop',
         autoCreate: false
     };
-    result[exports.REGISTRY_CONFIG_KEY] = {};
+    result[exports.REGISTRY_CONFIG_KEY] = {
+        provider: 'noop'
+    };
     result[exports.SYSINFO_CONFIG_KEY] = {
         provider: 'generic'
     };
@@ -282,7 +281,8 @@ function setCloudronName(name, callback) {
     if (!name) return callback(new BoxError(BoxError.BAD_FIELD, 'name is empty', { field: 'name' }));
 
     // some arbitrary restrictions (for sake of ui layout)
-    if (name.length > 32) return callback(new BoxError(BoxError.BAD_FIELD, 'name cannot exceed 32 characters', { field: 'name' }));
+    // if this is changed, adjust dashboard/branding.html
+    if (name.length > 64) return callback(new BoxError(BoxError.BAD_FIELD, 'name cannot exceed 64 characters', { field: 'name' }));
 
     settingsdb.set(exports.CLOUDRON_NAME_KEY, name, function (error) {
         if (error) return callback(error);
@@ -436,31 +436,26 @@ function setBackupCredentials(credentials, callback) {
     });
 }
 
-function getPlatformConfig(callback) {
+function getServicesConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    settingsdb.get(exports.PLATFORM_CONFIG_KEY, function (error, value) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, gDefaults[exports.PLATFORM_CONFIG_KEY]);
+    settingsdb.get(exports.SERVICES_CONFIG_KEY, function (error, value) {
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(null, gDefaults[exports.SERVICES_CONFIG_KEY]);
         if (error) return callback(error);
 
         callback(null, JSON.parse(value));
     });
 }
 
-function setPlatformConfig(platformConfig, callback) {
+function setServicesConfig(platformConfig, callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    for (let addon of [ 'mysql', 'postgresql', 'mail', 'mongodb' ]) {
-        if (!platformConfig[addon]) continue;
-        if (platformConfig[addon].memorySwap < platformConfig[addon].memory) return callback(new BoxError(BoxError.BAD_FIELD, 'memorySwap must be larger than memory', { field: 'memory', addon }));
-    }
-
-    settingsdb.set(exports.PLATFORM_CONFIG_KEY, JSON.stringify(platformConfig), function (error) {
+    settingsdb.set(exports.SERVICES_CONFIG_KEY, JSON.stringify(platformConfig), function (error) {
         if (error) return callback(error);
 
-        callback(null); // updating service config can take a while
+        notifyChange(exports.SERVICES_CONFIG_KEY, platformConfig);
 
-        addons.updateServiceConfig(platformConfig, NOOP_CALLBACK);
+        callback(null);
     });
 }
 
@@ -748,7 +743,7 @@ function getAll(callback) {
         result[exports.DEMO_KEY] = !!result[exports.DEMO_KEY];
 
         // convert JSON objects
-        [exports.BACKUP_CONFIG_KEY, exports.DIRECTORY_CONFIG_KEY, exports.PLATFORM_CONFIG_KEY, exports.EXTERNAL_LDAP_KEY, exports.REGISTRY_CONFIG_KEY, exports.SYSINFO_CONFIG_KEY ].forEach(function (key) {
+        [exports.BACKUP_CONFIG_KEY, exports.DIRECTORY_CONFIG_KEY, exports.SERVICES_CONFIG_KEY, exports.EXTERNAL_LDAP_KEY, exports.REGISTRY_CONFIG_KEY, exports.SYSINFO_CONFIG_KEY ].forEach(function (key) {
             result[key] = typeof result[key] === 'object' ? result[key] : safe.JSON.parse(result[key]);
         });
 

src/sftp.js

@@ -1,64 +1,50 @@
 'use strict';
 
 exports = module.exports = {
-    startSftp,
-    rebuild
+    start,
+    rebuild,
+
+    DEFAULT_MEMORY_LIMIT: 256 * 1024 * 1024
 };
 
 var apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     debug = require('debug')('box:sftp'),
+    docker = require('./docker.js'),
     hat = require('./hat.js'),
     infra = require('./infra_version.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
+    system = require('./system.js'),
     volumes = require('./volumes.js'),
     _ = require('underscore');
 
-function startSftp(existingInfra, callback) {
-    assert.strictEqual(typeof existingInfra, 'object');
+function rebuild(serviceConfig, options, callback) {
+    assert.strictEqual(typeof serviceConfig, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (existingInfra.version === infra.version && infra.images.sftp.tag === existingInfra.images.sftp.tag) return callback();
-
-    rebuild(callback);
-}
-
-var rebuildInProgress = false;
-function rebuild(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    if (rebuildInProgress) {
-        debug('waiting for other rebuild to finish');
-        return setTimeout(function () { rebuild(callback); }, 5000);
-    }
-
-    rebuildInProgress = true;
-
-    function done(error) {
-        rebuildInProgress = false;
-        callback(error);
-    }
-
     debug('rebuilding container');
 
+    const force = !!options.force;
     const tag = infra.images.sftp.tag;
-    const memoryLimit = 256;
+    const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT;
+    const memory = system.getMemoryAllocation(memoryLimit);
     const cloudronToken = hat(8 * 128);
 
     apps.getAll(function (error, result) {
-        if (error) return done(error);
+        if (error) return callback(error);
 
         let dataDirs = [];
         result.forEach(function (app) {
             if (!app.manifest.addons['localstorage']) return;
 
             const hostDir = apps.getDataDir(app, app.dataDir), mountDir = `/app/data/${app.id}`;
-            if (!safe.fs.existsSync(hostDir)) {
+            if (!safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user
                 // do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail
-                debug(`Ignoring volume for ${app.id} since it does not exist`);
+                debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`);
                 return;
             }
 
@@ -78,9 +64,9 @@ function rebuild(callback) {
                 dataDirs.push({ hostDir: volume.hostPath, mountDir: `/app/data/${volume.id}` });
             });
 
-            shell.exec('inspectSftp', 'docker inspect --format="{{json .Mounts }}" sftp', function (error, result) {
-                if (!error && result) {
-                    let currentDataDirs = safe.JSON.parse(result);
+            docker.inspect('sftp', function (error, data) {
+                if (!error && data && data.Mounts) {
+                    let currentDataDirs = data.Mounts;
                     if (currentDataDirs) {
                         currentDataDirs = currentDataDirs.filter(function (d) { return d.Destination.indexOf('/app/data/') === 0; }).map(function (d) { return { hostDir: d.Source, mountDir: d.Destination }; });
 
@@ -88,9 +74,9 @@ function rebuild(callback) {
                         currentDataDirs.sort(function (a, b) { return a.hostDir < b.hostDir ? -1 : 1; });
                         dataDirs.sort(function (a, b) { return a.hostDir < b.hostDir ? -1 : 1; });
 
-                        if (_.isEqual(currentDataDirs, dataDirs)) {
+                        if (!force && _.isEqual(currentDataDirs, dataDirs)) {
                             debug('Skipping rebuild, no changes');
-                            return done();
+                            return callback();
                         }
                     }
                 }
@@ -104,8 +90,8 @@ function rebuild(callback) {
                             --log-opt syslog-address=udp://127.0.0.1:2514 \
                             --log-opt syslog-format=rfc5424 \
                             --log-opt tag=sftp \
-                            -m ${memoryLimit}m \
-                            --memory-swap ${memoryLimit * 2}m \
+                            -m ${memory} \
+                            --memory-swap ${memoryLimit} \
                             --dns 172.18.0.1 \
                             --dns-search=. \
                             -p 222:22 \
@@ -120,8 +106,16 @@ function rebuild(callback) {
                     shell.exec.bind(null, 'stopSftp', 'docker stop sftp || true'),
                     shell.exec.bind(null, 'removeSftp', 'docker rm -f sftp || true'),
                     shell.exec.bind(null, 'startSftp', cmd)
-                ], done);
+                ], callback);
             });
         });
     });
 }
+
+function start(existingInfra, serviceConfig, callback) {
+    assert.strictEqual(typeof existingInfra, 'object');
+    assert.strictEqual(typeof serviceConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    rebuild(serviceConfig, { force: true }, callback); // force rebuild when infra changed
+}

src/storage/filesystem.js

@@ -1,22 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    getBackupPath: getBackupPath,
-    checkPreconditions: checkPreconditions,
+    getBackupPath,
+    checkPreconditions,
 
-    upload: upload,
-    download: download,
+    upload,
+    download,
 
-    copy: copy,
+    copy,
 
-    listDir: listDir,
+    exists,
+    listDir,
 
-    remove: remove,
-    removeDir: removeDir,
+    remove,
+    removeDir,
 
-    testConfig: testConfig,
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields
+    testConfig,
+    removePrivateFields,
+    injectPrivateFields
 };
 
 const PROVIDER_FILESYSTEM = 'filesystem';
@@ -111,8 +112,10 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
             const BACKUP_UID = parseInt(process.env.SUDO_UID, 10) || process.getuid();
 
             // sshfs and cifs handle ownership through the mount args
-            if (apiConfig.provider === PROVIDER_FILESYSTEM && !safe.fs.chownSync(backupFilePath, BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
-            if (apiConfig.provider === PROVIDER_FILESYSTEM && !safe.fs.chownSync(path.dirname(backupFilePath), BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
+            if (apiConfig.provider === PROVIDER_FILESYSTEM || apiConfig.provider === PROVIDER_NFS) {
+                if (!safe.fs.chownSync(backupFilePath, BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
+                if (!safe.fs.chownSync(path.dirname(backupFilePath), BACKUP_UID, BACKUP_UID)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unable to chown:' + safe.error.message));
+            }
 
             debug('upload %s: done.', backupFilePath);
 
@@ -134,6 +137,20 @@ function download(apiConfig, sourceFilePath, callback) {
     callback(null, fileStream);
 }
 
+function exists(apiConfig, sourceFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof sourceFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // do not use existsSync because it does not return EPERM etc
+    if (!safe.fs.statSync(sourceFilePath)) {
+        if (safe.error && safe.error.code === 'ENOENT') return callback(null, false);
+        if (safe.error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Exists ${sourceFilePath}: ${safe.error.message}`));
+    }
+
+    callback(null, true);
+}
+
 function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof dir, 'string');
@@ -260,8 +277,10 @@ function testConfig(apiConfig, callback) {
         if (error) return callback(error);
 
         if (typeof apiConfig.prefix !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'prefix must be a string', { field: 'prefix' }));
-        if (path.isAbsolute(apiConfig.prefix)) return new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path', { field: 'backupFolder' });
-        if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) return callback(new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path', { field: 'prefix' }));
+        if (apiConfig.prefix !== '') {
+            if (path.isAbsolute(apiConfig.prefix)) return new BoxError(BoxError.BAD_FIELD, 'prefix must be a relative path', { field: 'backupFolder' });
+            if (path.normalize(apiConfig.prefix) !== apiConfig.prefix) return callback(new BoxError(BoxError.BAD_FIELD, 'prefix must contain a normalized relative path', { field: 'prefix' }));
+        }
 
         const mounts = safe.fs.readFileSync('/proc/mounts', 'utf8');
         const mountInfo = mounts.split('\n').filter(function (l) { return l.indexOf(apiConfig.mountPoint) !== -1; })[0];

src/storage/gcs.js

@@ -1,21 +1,22 @@
 'use strict';
 
 exports = module.exports = {
-    getBackupPath: getBackupPath,
-    checkPreconditions: checkPreconditions,
+    getBackupPath,
+    checkPreconditions,
 
-    upload: upload,
-    download: download,
-    copy: copy,
+    upload,
+    exists,
+    download,
+    copy,
 
-    listDir: listDir,
+    listDir,
 
-    remove: remove,
-    removeDir: removeDir,
+    remove,
+    removeDir,
 
-    testConfig: testConfig,
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
+    testConfig,
+    removePrivateFields,
+    injectPrivateFields,
 
     // Used to mock GCS
     _mockInject: mockInject,
@@ -100,6 +101,36 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
     sourceStream.pipe(uploadStream);
 }
 
+function exists(apiConfig, backupFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const bucket = getBucket(apiConfig);
+
+    if (!backupFilePath.endsWith('/')) {
+        const file = bucket.file(backupFilePath);
+        file.getMetadata(function (error) {
+            if (error && error.code === 404) return callback(null, false);
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message));
+
+            callback(null, true);
+        });
+    } else {
+        const query = {
+            prefix: backupFilePath,
+            maxResults: 1,
+            autoPaginate: true
+        };
+
+        bucket.getFiles(query, function (error, files) {
+            if (error) return callback(error);
+
+            callback(null, files.length !== 0);
+        });
+    }
+}
+
 function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');
@@ -135,7 +166,7 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac
 
     let done = false;
 
-    async.whilst(() => !done, function listAndDownload(whilstCallback) {
+    async.whilst((testDone) => testDone(null, !done), function listAndDownload(whilstCallback) {
         bucket.getFiles(query, function (error, files, nextQuery) {
             if (error) return whilstCallback(error);
 
@@ -212,7 +243,7 @@ function removeDir(apiConfig, pathPrefix) {
 
     var events = new EventEmitter();
 
-    const batchSize = 1000, concurrency = 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles
+    const batchSize = 1000, concurrency = apiConfig.deleteConcurrency || 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles
     var total = 0;
 
     listDir(apiConfig, pathPrefix, batchSize, function (entries, done) {

src/storage/interface.js

@@ -11,23 +11,25 @@
 //     for the other API calls we leave it to the backend to retry. this allows
 //     them to tune the concurrency based on failures/rate limits accordingly
 exports = module.exports = {
-    getBackupPath: getBackupPath,
-    checkPreconditions: checkPreconditions,
+    getBackupPath,
+    checkPreconditions,
 
-    upload: upload,
+    upload,
 
-    download: download,
-    downloadDir: downloadDir,
-    copy: copy,
+    exists,
 
-    listDir: listDir,
+    download,
+    downloadDir,
+    copy,
 
-    remove: remove,
-    removeDir: removeDir,
+    listDir,
 
-    testConfig: testConfig,
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields
+    remove,
+    removeDir,
+
+    testConfig,
+    removePrivateFields,
+    injectPrivateFields
 };
 
 var assert = require('assert'),
@@ -72,6 +74,14 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
     callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'upload is not implemented'));
 }
 
+function exists(apiConfig, backupFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'exists is not implemented'));
+}
+
 function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');

src/storage/noop.js

@@ -1,22 +1,23 @@
 'use strict';
 
 exports = module.exports = {
-    getBackupPath: getBackupPath,
-    checkPreconditions: checkPreconditions,
+    getBackupPath,
+    checkPreconditions,
 
-    upload: upload,
-    download: download,
-    downloadDir: downloadDir,
-    copy: copy,
+    upload,
+    exists,
+    download,
+    downloadDir,
+    copy,
 
-    listDir: listDir,
+    listDir,
 
-    remove: remove,
-    removeDir: removeDir,
+    remove,
+    removeDir,
 
-    testConfig: testConfig,
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields
+    testConfig,
+    removePrivateFields,
+    injectPrivateFields
 };
 
 var assert = require('assert'),
@@ -49,6 +50,16 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
     callback(null);
 }
 
+function exists(apiConfig, backupFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('exists: %s', backupFilePath);
+
+    callback(null, false);
+}
+
 function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');

src/storage/s3.js

@@ -1,21 +1,22 @@
 'use strict';
 
 exports = module.exports = {
-    getBackupPath: getBackupPath,
-    checkPreconditions: checkPreconditions,
+    getBackupPath,
+    checkPreconditions,
 
-    upload: upload,
-    download: download,
-    copy: copy,
+    upload,
+    exists,
+    download,
+    copy,
 
-    listDir: listDir,
+    listDir,
 
-    remove: remove,
-    removeDir: removeDir,
+    remove,
+    removeDir,
 
-    testConfig: testConfig,
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
+    testConfig,
+    removePrivateFields,
+    injectPrivateFields,
 
     // Used to mock AWS
     _mockInject: mockInject,
@@ -56,7 +57,7 @@ function getS3Config(apiConfig, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var credentials = {
+    let credentials = {
         signatureVersion: apiConfig.signatureVersion || 'v4',
         s3ForcePathStyle: false, // Use vhost style instead of path style - https://forums.aws.amazon.com/ann.jspa?annID=6776
         accessKeyId: apiConfig.accessKeyId,
@@ -64,10 +65,10 @@ function getS3Config(apiConfig, callback) {
         region: apiConfig.region || 'us-east-1',
         maxRetries: 10,
         retryDelayOptions: {
-            customBackoff: () => 20000 // constant backoff - https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#retryDelayOptions-property
+            customBackoff: (/* retryCount, error */) => 20000 // constant backoff - https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#retryDelayOptions-property
         },
         httpOptions: {
-            connectTimeout: 20000, // https://github.com/aws/aws-sdk-js/pull/1446
+            connectTimeout: 60000, // https://github.com/aws/aws-sdk-js/pull/1446
             timeout: 0 // https://github.com/aws/aws-sdk-js/issues/1704 (allow unlimited time for chunk upload)
         }
     };
@@ -137,6 +138,45 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
     });
 }
 
+function exists(apiConfig, backupFilePath, callback) {
+    assert.strictEqual(typeof apiConfig, 'object');
+    assert.strictEqual(typeof backupFilePath, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getS3Config(apiConfig, function (error, credentials) {
+        if (error) return callback(error);
+
+        const s3 = new AWS.S3(_.omit(credentials, 'retryDelayOptions', 'maxRetries'));
+
+        if (!backupFilePath.endsWith('/')) { // check for file
+            const params = {
+                Bucket: apiConfig.bucket,
+                Key: backupFilePath
+            };
+
+            s3.headObject(params, function (error) {
+                if (!Object.keys(this.httpResponse.headers).some(h => h.startsWith('x-amz'))) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'not a s3 endpoint'));
+                if (error && S3_NOT_FOUND(error)) return callback(null, false);
+                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error headObject ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`));
+
+                callback(null, true);
+            });
+        } else { // list dir contents
+            const listParams = {
+                Bucket: apiConfig.bucket,
+                Prefix: backupFilePath,
+                MaxKeys: 1
+            };
+
+            s3.listObjects(listParams, function (error, listData) {
+                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error listing objects ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`));
+
+                callback(null, listData.Contents.length !== 0);
+            });
+        }
+    });
+}
+
 function download(apiConfig, backupFilePath, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof backupFilePath, 'string');
@@ -160,7 +200,7 @@ function download(apiConfig, backupFilePath, callback) {
                 ps.emit('error', new BoxError(BoxError.NOT_FOUND, `Backup not found: ${backupFilePath}`));
             } else {
                 debug(`download: ${apiConfig.bucket}:${backupFilePath} s3 stream error.`, error);
-                ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, error.message || error.code)); // DO sets 'code'
+                ps.emit('error', new BoxError(BoxError.EXTERNAL_ERROR, `Error multipartDownload ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`));
             }
         });
 
@@ -189,9 +229,9 @@ function listDir(apiConfig, dir, batchSize, iteratorCallback, callback) {
 
         let done = false;
 
-        async.whilst(() => !done, function listAndDownload(whilstCallback) {
+        async.whilst((testDone) => testDone(null, !done), function listAndDownload(whilstCallback) {
             s3.listObjects(listParams, function (error, listData) {
-                if (error) return whilstCallback(new BoxError(BoxError.EXTERNAL_ERROR, error.message || error.code));
+                if (error) return whilstCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Error listing objects in ${dir}. Message: ${error.message} HTTP Code: ${error.code}`));
 
                 if (listData.Contents.length === 0) { done = true; return whilstCallback(); }
 
@@ -256,7 +296,7 @@ function copy(apiConfig, oldFilePath, newFilePath) {
 
             // S3 copyObject has a file size limit of 5GB so if we have larger files, we do a multipart copy
             // Exoscale and B2 take too long to copy 5GB
-            const largeFileLimit = (apiConfig.provider === 'exoscale-sos' || apiConfig.provider === 'backblaze-b2') ? 1024 * 1024 * 1024 : 5 * 1024 * 1024 * 1024;
+            const largeFileLimit = (apiConfig.provider === 'exoscale-sos' || apiConfig.provider === 'backblaze-b2' || apiConfig.provider === 'digitalocean-spaces') ? 1024 * 1024 * 1024 : 5 * 1024 * 1024 * 1024;
 
             if (entry.size < largeFileLimit) {
                 events.emit('progress', `Copying ${relativePath || oldFilePath}`);
@@ -288,7 +328,7 @@ function copy(apiConfig, oldFilePath, newFilePath) {
                 }
                 ranges.push({ startBytes: cur, endBytes: entry.size-1 });
 
-                async.eachOfLimit(ranges, 5, function copyChunk(range, index, iteratorDone) {
+                async.eachOfLimit(ranges, 3, function copyChunk(range, index, iteratorDone) {
                     const partCopyParams = {
                         Bucket: apiConfig.bucket,
                         Key: path.join(newFilePath, relativePath),
@@ -303,7 +343,7 @@ function copy(apiConfig, oldFilePath, newFilePath) {
                     s3.uploadPartCopy(partCopyParams, function (error, part) {
                         if (error) return iteratorDone(error);
 
-                        events.emit('progress', `Uploaded part ${partCopyParams.PartNumber} - Etag: ${part.CopyPartResult.ETag}`);
+                        events.emit('progress', `Copying part ${partCopyParams.PartNumber} - Etag: ${part.CopyPartResult.ETag}`);
 
                         if (!part.CopyPartResult.ETag) return iteratorDone(new Error('Multi-part copy is broken or not implemented by the S3 storage provider'));
 
@@ -465,7 +505,7 @@ function testConfig(apiConfig, callback) {
 
         var s3 = new AWS.S3(_.omit(credentials, 'retryDelayOptions', 'maxRetries'));
         s3.putObject(params, function (error) {
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message || error.code)); // DO sets 'code'
+            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error put object cloudron-testfile. Message: ${error.message} HTTP Code: ${error.code}`));
 
             var params = {
                 Bucket: apiConfig.bucket,
@@ -473,7 +513,7 @@ function testConfig(apiConfig, callback) {
             };
 
             s3.deleteObject(params, function (error) {
-                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error.message || error.code)); // DO sets 'code'
+                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Error del object cloudron-testfile. Message: ${error.message} HTTP Code: ${error.code}`));
 
                 callback();
             });