Add to changes

See "Losing context" https://masteringjs.io/tutorials/node/promisify

.eslintrc.json

@@ -5,7 +5,7 @@
     },
     "extends": "eslint:recommended",
     "parserOptions": {
-        "ecmaVersion": 8
+        "ecmaVersion": 2020
     },
     "rules": {
         "indent": [

.jshintrc

@@ -0,0 +1,5 @@
+{
+    "node": true,
+    "unused": true,
+    "esversion": 8
+}

CHANGES

@@ -2186,3 +2186,191 @@
 [6.1.1]
 * Fix bug where platform does not start if memory limits could not be applied
 
+[6.1.2]
+* App disk usage was not shown in graphs
+* Email autoconfig
+* Fix SOGo login
+
+[6.2.0]
+* ovh: object storage URL has changed from s3 to storage subdomain
+* ionos: add profit bricks object storage
+* update node to 14.15.4
+* update docker to 20.10.3
+* new base image 3.0.0
+* postgresql updated to 12.5
+* redis updated to 5.0.7
+* dovecot updated to 2.3.7
+* proxyAuth: fix docker UA detection
+* registry config: add UI to disable it
+* update solr to 8.8.1
+* firewall: fix issue where script errored when having more than 15 wl/bl ports
+* If groups are used, do not allow app installation without choosing the access settings
+* tls addon
+* Do not overwrite existing DMARC record
+* Sync dns records
+* Dry run restore
+* linode: show cloudron is installing when user SSHs
+* mysql: disable bin logs
+* Show cancel task button if task is still running after 2 minutes
+* filemanager: fix various bugs involving file names with spaces
+* Change Referrer-policy default to 'same-origin'
+* rsync: preserve and restore symlinks
+* Clean up backups function now removes missing backups
+
+[6.2.1]
+* Avoid updown notifications on full restore
+* Add retries to downloader logic in installer
+
+[6.2.2]
+* Fix ENOBUFS issue with backups when collecting fs metadata
+
+[6.2.3]
+* Fix addon crashes with missing databases
+* Update mail container for LMTP cert fix
+* Fix services view showing yellow icon
+
+[6.2.4]
+* Another addon crash fix
+
+[6.2.5]
+* update: set memory limit properly
+* Fix bug where renew certs button did not work
+* sftp: fix rebuild condition
+* Fix display of user management/dashboard visiblity for email apps
+* graphite: disable tagdb and reduce log noise
+
+[6.2.6]
+* Fix issue where collectd is restarted too quickly before graphite
+
+[6.2.7]
+* redis: backup before upgrade
+
+[6.2.8]
+* linode object storage: update aws sdk to make it work again
+* Fix crash in blocklist setting when source and list have mixed ip versions
+* mysql: bump connection limit to 200
+* namecheap: fix issue where DNS updates and del were not working
+* turn: turn off verbose logging
+* Fix crash when parsing df output (set LC_ALL for box service)
+
+[6.3.0]
+* mail: allow TLS from internal hosts
+* tokens: add lastUsedTime
+* update: set memory limit properly
+* addons: better error handling
+* filemanager: various enhancements
+* sftp: fix rebuild condition
+* app mailbox is now optional
+* Fix display of user management/dashboard visiblity for email apps
+* graphite: disable tagdb and reduce log noise
+* hsts: change max-age to 2 years
+* clone: copy over redis memory limit
+* namecheap: fix bug where records were not removed
+* add UI to disable 2FA of a user
+* mail: add active flag to mailboxes and lists
+* Implement OCSP stapling
+* security: send new browser login location notification email
+* backups: add fqdn to the backup filename
+* import all boxdata settings into the database
+* volumes: generate systemd mount configs based on type
+* postgresql: set max conn limit per db
+* ubuntu 16: add alert about EOL
+* clone: save and restore app config
+* app import: restore icon, tag, label, proxy configs etc
+* sieve: fix redirects to not do SRS
+* notifications are now system level instead of per-user
+* vultr DNS
+* vultr object storage
+* mail: do not forward spam to mailing lists
+
+[6.3.1]
+* Fix cert migration issues
+
+[6.3.2]
+* Avatar was migrated as base64 instead of binary
+* Fix issue where filemanager came up empty for CIFS mounts
+
+[6.3.3]
+* volumes: add filesystem volume type for shared folders
+* mail: enable sieve extension editheader
+* mail: update solr to 8.9.0
+
+[6.3.4]
+* Fix issue where old nginx configs where not removed before upgrade
+
+[6.3.5]
+* Fix permission issues with sshfs
+* filemanager: reset selection if directory has changed
+* branding: fix error highlight with empty cloudron name
+* better text instead of "Cloudron in the wild"
+* Make sso login hint translatable
+* Give unread notifications a small left border
+* Fix issue where clicking update indicator opened app in new tab
+* Ensure notifications are only fetched and shown for at least admins
+* setupaccount: Show input field errors below input field
+* Set focus automatically for new alias or redirect
+* eventlog: fix issue where old events are not periodically removed
+* ssfs: fix chown
+
+[6.3.6]
+* Fix broken reboot button
+* app updated notification shown despite failure
+* Update translation for sso login information
+* Hide groups/tags/state filter in app listing for normal users
+* filemanager: Ensure breadcrumbs and hash are correctly updated on folder navigation
+* cloudron-setup: check if nginx/docker is already installed
+* Use the addresses of all available interfaces for port 53 binding
+* refresh config on appstore login
+* password reset: check 2fa when enabled
+
+[7.0.0]
+* Ubuntu 16 is not supported anymore
+* Do not use Gravatar as the default but only an option
+* redis: suppress password warning
+* setup UI: fix dark mode
+* wellknown: response to .wellknown/matrix/client
+* purpose field is not required anymore during appstore signup
+* sftp: fix symlink deletion
+* Show correct/new app version info in updated finished notification
+* Make new login email translatable
+* Hide ticket form if cloudron.io mail is not verified
+* Refactor code to use async/await
+* postgresql: bump shm size and disable parallel queries
+* update nodejs to 14.17.6
+* external ldap: If we detect a local user with the same username as found on LDAP/AD we map it
+* add basic eventlog for apps in app view
+* Enable sshfs/cifs/nfs in app import UI
+* Require password for fallback email change
+* Make password reset logic translatable
+* support: only verified email address can open support tickets
+* Logout users without 2FA when mandatory 2fa is enabled
+* notifications: better oom message for redis
+* Add way to impersonate users for presetup
+* mail: open up port 465 for mail submission (TLS)
+* Implement operator role for apps
+* sftp: normal users do not have SFTP access anymore. Use operator role instead
+* eventlog: add service rebuild/restart/configure events
+* upcloud: add object storage integration
+* Each app can now have a custom crontab
+* services: add recovery mode
+* postgresql: fix restore issue with long table names
+* recvmail: make the addon work again
+* mail: update solr to 8.10.0
+* mail: POP3 support
+* update docker to 20.10.7
+* volumes: add remount button
+* mail: add spam eventlog filter type
+* mail: configure dnsbl
+* mail: add duplication detection for lists
+* mail: add SRS for Sieve Forwarding
+
+[7.0.1]
+* Fix matrix wellKnown client migration
+
+[7.0.2]
+* mail: POP3 flag was not returned correctly
+* external ldap: fix crash preventing users from logging in
+* volumes: ensure we don't crash if mount status is unexpected
+* backups: set default backup memory limit to 800
+* users: allow admins to specify password recovery email
+

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2020 Cloudron UG
+Copyright (c) 2021 Cloudron UG
 
 With regard to the Cloudron Software:
 

README.md

@@ -1,3 +1,5 @@
+![Translation status](https://translate.cloudron.io/widgets/cloudron/-/svg-badge.svg)
+
 # Cloudron
 
 [Cloudron](https://cloudron.io) is the best way to run apps on your server.
@@ -70,8 +72,13 @@ Just to give some heads up, we are a bit restrictive in merging changes. We are
 would like to keep our maintenance burden low. It might be best to discuss features first in the [forum](https://forum.cloudron.io),
 to also figure out how many other people will use it to justify maintenance for a feature.
 
+# Localization
+
+![Translation status](https://translate.cloudron.io/widgets/cloudron/-/287x66-white.png)
+
 ## Support
 
 * [Documentation](https://docs.cloudron.io/)
 * [Forum](https://forum.cloudron.io/)
 
+

baseimage/createAMI

@@ -1,193 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-assertNotEmpty() {
-    : "${!1:? "$1 is not set."}"
-}
-
-readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-export JSON="${SOURCE_DIR}/node_modules/.bin/json"
-
-INSTANCE_TYPE="t2.micro"
-BLOCK_DEVICE="DeviceName=/dev/sda1,Ebs={VolumeSize=20,DeleteOnTermination=true,VolumeType=gp2}"
-SSH_KEY_NAME="id_rsa_yellowtent"
-
-revision=$(git rev-parse HEAD)
-ami_name=""
-server_id=""
-server_ip=""
-destroy_server="yes"
-deploy_env="prod"
-image_id=""
-
-args=$(getopt -o "" -l "revision:,name:,no-destroy,env:,region:" -n "$0" -- "$@")
-eval set -- "${args}"
-
-while true; do
-    case "$1" in
-    --env) deploy_env="$2"; shift 2;;
-    --revision) revision="$2"; shift 2;;
-    --name) ami_name="$2"; shift 2;;
-    --no-destroy) destroy_server="no"; shift 2;;
-    --region)
-        case "$2" in
-        "us-east-1")
-            image_id="ami-6edd3078"
-            security_group="sg-a5e17fd9"
-            subnet_id="subnet-b8fbc0f1"
-            ;;
-        "eu-central-1")
-            image_id="ami-5aee2235"
-            security_group="sg-19f5a770" # everything open on eu-central-1
-            subnet_id=""
-            ;;
-        *)
-            echo "Unknown aws region $2"
-            exit 1
-            ;;
-        esac
-        export AWS_DEFAULT_REGION="$2"    # used by the aws cli tool
-        shift 2
-        ;;
-    --) break;;
-    *) echo "Unknown option $1"; exit 1;;
-    esac
-done
-
-# TODO fix this
-export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY}"
-export AWS_SECRET_ACCESS_KEY="${AWS_ACCESS_SECRET}"
-
-readonly ssh_keys="${HOME}/.ssh/id_rsa_yellowtent"
-readonly SSH="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-
-if [[ ! -f "${ssh_keys}" ]]; then
-    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
-    exit 1
-fi
-
-if [[ -z "${image_id}" ]]; then
-    echo "--region is required (us-east-1 or eu-central-1)"
-    exit 1
-fi
-
-function get_pretty_revision() {
-    local git_rev="$1"
-    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
-
-    echo "${sha1}"
-}
-
-function wait_for_ssh() {
-    echo "=> Waiting for ssh connection"
-    while true; do
-        echo -n "."
-
-        if $SSH ubuntu@${server_ip} echo "hello"; then
-            echo ""
-            break
-        fi
-
-        sleep 5
-    done
-}
-
-now=$(date "+%Y-%m-%d-%H%M%S")
-pretty_revision=$(get_pretty_revision "${revision}")
-
-if [[ -z "${ami_name}" ]]; then
-    ami_name="box-${deploy_env}-${pretty_revision}-${now}"
-fi
-
-echo "=> Create EC2 instance"
-id=$(aws ec2 run-instances --image-id "${image_id}" --instance-type "${INSTANCE_TYPE}" --security-group-ids "${security_group}" --block-device-mappings "${BLOCK_DEVICE}" --key-name "${SSH_KEY_NAME}" --subnet-id "${subnet_id}" --associate-public-ip-address \
-    | $JSON Instances \
-    | $JSON 0.InstanceId)
-
-[[ -z "$id" ]] && exit 1
-echo "Instance created  ID $id"
-
-echo "=> Waiting for instance to get a public IP"
-while true; do
-    server_ip=$(aws ec2 describe-instances --instance-ids ${id} \
-        | $JSON Reservations.0.Instances \
-        | $JSON 0.PublicIpAddress)
-
-    if [[ ! -z "${server_ip}" ]]; then
-        echo ""
-        break
-    fi
-
-    echo -n "."
-    sleep 1
-done
-
-echo "Got public IP ${server_ip}"
-
-wait_for_ssh
-
-echo "=> Fetching cloudron-setup"
-while true; do
-
-    if $SSH ubuntu@${server_ip} wget "https://cloudron.io/cloudron-setup" -O "cloudron-setup"; then
-        echo ""
-        break
-    fi
-
-    echo -n "."
-    sleep 5
-done
-
-echo "=> Running cloudron-setup"
-$SSH ubuntu@${server_ip} sudo /bin/bash "cloudron-setup" --env "${deploy_env}" --provider "ami" --skip-reboot
-
-wait_for_ssh
-
-echo "=> Removing ssh key"
-$SSH ubuntu@${server_ip} sudo rm /home/ubuntu/.ssh/authorized_keys /root/.ssh/authorized_keys
-
-echo "=> Creating AMI"
-image_id=$(aws ec2 create-image --instance-id "${id}" --name "${ami_name}" | $JSON ImageId)
-[[ -z "$id" ]] && exit 1
-echo "Creating AMI with Id ${image_id}"
-
-echo "=> Waiting for AMI to be created"
-while true; do
-    state=$(aws ec2 describe-images --image-ids ${image_id} \
-        | $JSON Images \
-        | $JSON 0.State)
-
-    if [[ "${state}" == "available" ]]; then
-        echo ""
-        break
-    fi
-
-    echo -n "."
-    sleep 5
-done
-
-if [[ "${destroy_server}" == "yes" ]]; then
-    echo "=> Deleting EC2 instance"
-
-    while true; do
-        state=$(aws ec2 terminate-instances --instance-id "${id}" \
-            | $JSON TerminatingInstances \
-            | $JSON 0.CurrentState.Name)
-
-        if [[ "${state}" == "shutting-down" ]]; then
-            echo ""
-            break
-        fi
-
-        echo -n "."
-        sleep 5
-    done
-fi
-
-echo ""
-echo "Done."
-echo ""
-echo "New AMI is: ${image_id}"
-echo ""

baseimage/digitalocean.sh

@@ -1,261 +0,0 @@
-#!/bin/bash
-
-if [[ -z "${DIGITAL_OCEAN_TOKEN}" ]]; then
-    echo "Script requires DIGITAL_OCEAN_TOKEN env to be set"
-    exit 1
-fi
-
-if [[ -z "${JSON}" ]]; then
-    echo "Script requires JSON env to be set to path of JSON binary"
-    exit 1
-fi
-
-readonly CURL="curl --retry 5 -s -u ${DIGITAL_OCEAN_TOKEN}:"
-
-function debug() {
-    echo "$@" >&2
-}
-
-function get_ssh_key_id() {
-    id=$($CURL "https://api.digitalocean.com/v2/account/keys" \
-        | $JSON ssh_keys \
-        | $JSON -c "this.name === \"$1\"" \
-        | $JSON 0.id)
-    [[ -z "$id" ]] && exit 1
-    echo "$id"
-}
-
-function create_droplet() {
-    local ssh_key_id="$1"
-    local box_name="$2"
-
-    local image_region="sfo2"
-    local ubuntu_image_slug="ubuntu-16-04-x64"
-    local box_size="1gb"
-
-    local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"
-
-    id=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets" | $JSON droplet.id)
-    [[ -z "$id" ]] && exit 1
-    echo "$id"
-}
-
-function get_droplet_ip() {
-    local droplet_id="$1"
-    ip=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}" | $JSON "droplet.networks.v4[0].ip_address")
-    [[ -z "$ip" ]] && exit 1
-    echo "$ip"
-}
-
-function get_droplet_id() {
-    local droplet_name="$1"
-    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=200" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
-    [[ -z "$id" ]] && exit 1
-    echo "$id"
-}
-
-function power_off_droplet() {
-    local droplet_id="$1"
-    local data='{"type":"power_off"}'
-    local response=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions")
-    local event_id=`echo "${response}" | $JSON action.id`
-
-    if [[ -z "${event_id}" ]]; then
-        debug "Got no event id, assuming already powered off."
-        debug "Response: ${response}"
-        return
-    fi
-
-    debug "Powered off droplet. Event id: ${event_id}"
-    debug -n "Waiting for droplet to power off"
-
-    while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
-        if [[ "${event_status}" == "completed" ]]; then
-            break
-        fi
-        debug -n "."
-        sleep 10
-    done
-    debug ""
-}
-
-function power_on_droplet() {
-    local droplet_id="$1"
-    local data='{"type":"power_on"}'
-    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
-
-    debug "Powered on droplet. Event id: ${event_id}"
-
-    if [[ -z "${event_id}" ]]; then
-        debug "Got no event id, assuming already powered on"
-        return
-    fi
-
-    debug -n "Waiting for droplet to power on"
-
-    while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
-        if [[ "${event_status}" == "completed" ]]; then
-            break
-        fi
-        debug -n "."
-        sleep 10
-    done
-    debug ""
-}
-
-function get_image_id() {
-    local snapshot_name="$1"
-    local image_id=""
-
-    if ! response=$($CURL "https://api.digitalocean.com/v2/images?per_page=200"); then
-        echo "Failed to get image listing. ${response}"
-        return 1
-    fi
-
-    if ! image_id=$(echo "$response" \
-       | $JSON images \
-       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id); then
-        echo "Failed to parse curl response: ${response}"
-        return 1
-    fi
-
-    if [[ -z "${image_id}" ]]; then
-        echo "Failed to get image id of ${snapshot_name}. reponse: ${response}"
-        return 1
-    fi
-
-    echo "${image_id}"
-}
-
-function snapshot_droplet() {
-    local droplet_id="$1"
-    local snapshot_name="$2"
-    local data="{\"type\":\"snapshot\",\"name\":\"${snapshot_name}\"}"
-    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
-
-    debug "Droplet snapshotted as ${snapshot_name}. Event id: ${event_id}"
-    debug -n "Waiting for snapshot to complete"
-
-    while true; do
-        if ! response=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}"); then
-            echo "Could not get action status. ${response}"
-            continue
-        fi
-        if ! event_status=$(echo "${response}" | $JSON action.status); then
-            echo "Could not parse action.status from response. ${response}"
-            continue
-        fi
-        if [[ "${event_status}" == "completed" ]]; then
-            break
-        fi
-        debug -n "."
-        sleep 10
-    done
-    debug "! done"
-
-    if ! image_id=$(get_image_id "${snapshot_name}"); then
-        return 1
-    fi
-    echo "${image_id}"
-}
-
-function destroy_droplet() {
-    local droplet_id="$1"
-    # TODO: check for 204 status
-    $CURL -X DELETE "https://api.digitalocean.com/v2/droplets/${droplet_id}"
-    debug "Droplet destroyed"
-    debug ""
-}
-
-function transfer_image() {
-    local image_id="$1"
-    local region_slug="$2"
-    local data="{\"type\":\"transfer\",\"region\":\"${region_slug}\"}"
-    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/images/${image_id}/actions" | $JSON action.id`
-    echo "${event_id}"
-}
-
-function wait_for_image_event() {
-    local image_id="$1"
-    local event_id="$2"
-
-    debug -n "Waiting for ${event_id}"
-
-    while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/images/${image_id}/actions/${event_id}" | $JSON action.status`
-        if [[ "${event_status}" == "completed" ]]; then
-            break
-        fi
-        debug -n "."
-        sleep 10
-    done
-    debug ""
-}
-
-function transfer_image_to_all_regions() {
-    local image_id="$1"
-
-    xfer_events=()
-    image_regions=(ams2) ## sfo1 is where the image is created
-    for image_region in ${image_regions[@]}; do
-        xfer_event=$(transfer_image ${image_id} ${image_region})
-        echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}"
-        xfer_events+=("${xfer_event}")
-        sleep 1
-    done
-
-    echo "Image transfer initiated, but they will take some time to get transferred."
-
-    for xfer_event in ${xfer_events[@]}; do
-        $vps wait_for_image_event "${image_id}" "${xfer_event}"
-    done
-}
-
-if [[ $# -lt 1 ]]; then
-    debug "<command> <params...>"
-    exit 1
-fi
-
-case $1 in
-get_ssh_key_id)
-    get_ssh_key_id "${@:2}"
-    ;;
-
-create)
-    create_droplet "${@:2}"
-    ;;
-
-get_id)
-    get_droplet_id "${@:2}"
-    ;;
-
-get_ip)
-    get_droplet_ip "${@:2}"
-    ;;
-
-power_on)
-    power_on_droplet "${@:2}"
-    ;;
-
-power_off)
-    power_off_droplet "${@:2}"
-    ;;
-
-snapshot)
-    snapshot_droplet "${@:2}"
-    ;;
-
-destroy)
-    destroy_droplet "${@:2}"
-    ;;
-
-transfer_image_to_all_regions)
-    transfer_image_to_all_regions "${@:2}"
-    ;;
-
-*)
-    echo "Unknown command $1"
-    exit 1
-esac

baseimage/initializeBaseUbuntuImage.sh

@@ -32,6 +32,7 @@ debconf-set-selections <<< 'mysql-server mysql-server/root_password_again passwo
 
 gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
 mysql_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "mysql-server-8.0" || echo "mysql-server-5.7")
+ntpd_package=$([[ "${ubuntu_version}" == "20.04" ]] && echo "systemd-timesyncd" || echo "")
 apt-get -y install --no-install-recommends \
     acl \
     apparmor \
@@ -48,9 +49,12 @@ apt-get -y install --no-install-recommends \
     linux-generic \
     logrotate \
     $mysql_package \
+    nfs-common \
+    $ntpd_package \
     openssh-server \
     pwgen \
     resolvconf \
+    sshfs \
     swaks \
     tzdata \
     unattended-upgrades \
@@ -72,10 +76,11 @@ apt-get -o Dpkg::Options::="--force-confold" install -y --no-install-recommends
 cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
 
 echo "==> Installing node.js"
-mkdir -p /usr/local/node-10.18.1
-curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-10.18.1
-ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
-ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
+readonly node_version=14.17.6
+mkdir -p /usr/local/node-${node_version}
+curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-${node_version}
+ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
+ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
 apt-get install -y --no-install-recommends python   # Install python which is required for npm rebuild
 [[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
@@ -87,9 +92,10 @@ mkdir -p /etc/systemd/system/docker.service.d
 echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
 
 # there are 3 packages for docker - containerd, CLI and the daemon
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.13-2_amd64.deb" -o /tmp/containerd.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
-curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
+readonly docker_version=20.10.7
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.6-1_amd64.deb" -o /tmp/containerd.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 # apt install with install deps (as opposed to dpkg -i)
 apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
 rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb

box.js

@@ -2,65 +2,74 @@
 
 'use strict';
 
-let async = require('async'),
-    dockerProxy = require('./src/dockerproxy.js'),
+const dockerProxy = require('./src/dockerproxy.js'),
     fs = require('fs'),
     ldap = require('./src/ldap.js'),
     paths = require('./src/paths.js'),
     proxyAuth = require('./src/proxyauth.js'),
+    safe = require('safetydance'),
     server = require('./src/server.js');
 
-const NOOP_CALLBACK = function () { };
+let logFd;
 
-function setupLogging(callback) {
-    if (process.env.BOX_ENV === 'test') return callback();
+async function setupLogging() {
+    if (process.env.BOX_ENV === 'test') return;
 
-    var logfileStream = fs.createWriteStream(paths.BOX_LOG_FILE, { flags:'a' });
-    process.stdout.write = process.stderr.write = logfileStream.write.bind(logfileStream);
-
-    callback();
+    logFd = fs.openSync(paths.BOX_LOG_FILE, 'a');
+    // we used to write using a stream before but it caches internally and there is no way to flush it when things crash
+    process.stdout.write = process.stderr.write = function (...args) {
+        const callback = typeof args[args.length-1] === 'function' ? args.pop() : function () {}; // callback is required for fs.write
+        fs.write.apply(fs, [logFd, ...args, callback]);
+    };
 }
 
-async.series([
-    setupLogging,
-    server.start, // do this first since it also inits the database
-    proxyAuth.start,
-    ldap.start,
-    dockerProxy.start
-], function (error) {
-    if (error) {
-        console.log('Error starting server', error);
-        process.exit(1);
-    }
+// this is also used as the 'uncaughtException' handler which can only have synchronous functions
+function exitSync(status) {
+    if (status.error) fs.write(logFd, status.error.stack + '\n', function () {});
+    fs.fsyncSync(logFd);
+    fs.closeSync(logFd);
+    process.exit(status.code);
+}
+
+async function startServers() {
+    await setupLogging();
+    await server.start(); // do this first since it also inits the database
+    await proxyAuth.start();
+    await ldap.start();
+    await dockerProxy.start();
+}
+
+async function main() {
+    const [error] = await safe(startServers());
+    if (error) return exitSync({ error: new Error(`Error starting server: ${JSON.stringify(error)}`), code: 1 });
 
     // require those here so that logging handler is already setup
     require('supererror');
     const debug = require('debug')('box:box');
 
-    process.on('SIGINT', function () {
+    process.on('SIGINT', async function () {
         debug('Received SIGINT. Shutting down.');
 
-        proxyAuth.stop(NOOP_CALLBACK);
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
+        await proxyAuth.stop();
+        await server.stop();
+        await ldap.stop();
+        await dockerProxy.stop();
         setTimeout(process.exit.bind(process), 3000);
     });
 
-    process.on('SIGTERM', function () {
+    process.on('SIGTERM', async function () {
         debug('Received SIGTERM. Shutting down.');
 
-        proxyAuth.stop(NOOP_CALLBACK);
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
+        await proxyAuth.stop();
+        await server.stop();
+        await ldap.stop();
+        await dockerProxy.stop();
         setTimeout(process.exit.bind(process), 3000);
     });
 
-    process.on('uncaughtException', function (error) {
-        console.error((error && error.stack) ? error.stack : error);
-        setTimeout(process.exit.bind(process, 1), 3000);
-    });
+    process.on('uncaughtException', (error) => exitSync({ error, code: 1 }));
 
     console.log(`Cloudron is up and running. Logs are at ${paths.BOX_LOG_FILE}`); // this goes to journalctl
-});
+}
+
+main();

crashnotifierservice.js

@@ -2,27 +2,21 @@
 
 'use strict';
 
-var database = require('./src/database.js');
+const database = require('./src/database.js');
 
-var crashNotifier = require('./src/crashnotifier.js');
+const crashNotifier = require('./src/crashnotifier.js');
 
 // This is triggered by systemd with the crashed unit name as argument
-function main() {
+async function main() {
     if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <unitName>');
 
-    var unitName = process.argv[2];
+    const unitName = process.argv[2];
     console.log('Started crash notifier for', unitName);
 
     // eventlog api needs the db
-    database.initialize(function (error) {
-        if (error) return console.error('Cannot connect to database. Unable to send crash log.', error);
+    await database.initialize();
 
-        crashNotifier.sendFailureLogs(unitName, function (error) {
-            if (error) console.error(error);
-
-            process.exit();
-        });
-    });
+    await crashNotifier.sendFailureLogs(unitName);
 }
 
 main();

migrations/20210204181904-settings-change-ovh-url.js

@@ -0,0 +1,10 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    /* this contained an invalid migration of OVH URLs from s3 subdomain to storage subdomain. See https://forum.cloudron.io/topic/4584/issue-with-backups-listings-and-saving-backup-config-in-6-2  */
+    callback();
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210209223121-settings-add-registry-config-provider.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT value FROM settings WHERE name="registry_config"', function (error, results) {
+        if (error || results.length === 0) return callback(error);
+
+        var registryConfig = JSON.parse(results[0].value);
+        if (!registryConfig.provider) registryConfig.provider = 'other';
+
+        db.runSql('UPDATE settings SET value=? WHERE name="registry_config"', [ JSON.stringify(registryConfig) ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210315194623-tokens-add-lastUsedTime.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE tokens ADD COLUMN lastUsedTime TIMESTAMP NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE tokens DROP COLUMN lastUsedTime', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210317014101-apps-add-enableMailbox.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN enableMailbox BOOLEAN DEFAULT 1', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN enableMailbox', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20210415030645-mailboxes-add-active.js

@@ -0,0 +1,17 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes ADD COLUMN active BOOLEAN DEFAULT 1', function (error) {
+        if (error) return callback(error);
+
+        callback();
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN active', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20210429194328-users-add-avatar.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    path = require('path');
+
+const AVATAR_DIR = '/home/yellowtent/boxdata/profileicons';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN avatar MEDIUMBLOB', function (error) {
+        if (error) return callback(error);
+
+        fs.readdir(AVATAR_DIR, function (error, filenames) {
+            if (error && error.code === 'ENOENT') return callback();
+            if (error) return callback(error);
+
+            async.eachSeries(filenames, function (filename, iteratorCallback) {
+                const avatar = fs.readFileSync(path.join(AVATAR_DIR, filename));
+                const userId = filename;
+
+                db.runSql('UPDATE users SET avatar=? WHERE id=?', [ avatar, userId ], iteratorCallback);
+            }, function (error) {
+                if (error) return callback(error);
+
+                fs.rmdir(AVATAR_DIR, { recursive: true }, callback);
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN avatar', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20210429224354-settings-add-valueBlob.js

@@ -0,0 +1,20 @@
+'use strict';
+
+const fs = require('fs');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE settings ADD COLUMN valueBlob MEDIUMBLOB', function (error) {
+        if (error) return callback(error);
+
+        fs.readFile('/home/yellowtent/boxdata/avatar.png', function (error, avatar) {
+            if (error && error.code === 'ENOENT') return callback();
+            if (error) return callback(error);
+
+            db.runSql('INSERT INTO settings (name, valueBlob) VALUES (?, ?)', [ 'cloudron_avatar', avatar ], callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210430120936-users-add-loginLocationsJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN loginLocationsJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN loginLocationsJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210430200947-apps-add-icon.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    path = require('path');
+
+const APPICONS_DIR = '/home/yellowtent/boxdata/appicons';
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN icon MEDIUMBLOB'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN appStoreIcon MEDIUMBLOB'),
+        function migrateIcons(next) {
+            fs.readdir(APPICONS_DIR, function (error, filenames) {
+                if (error && error.code === 'ENOENT') return next();
+                if (error) return next(error);
+
+                async.eachSeries(filenames, function (filename, iteratorCallback) {
+                    const icon = fs.readFileSync(path.join(APPICONS_DIR, filename));
+                    const appId = filename.split('.')[0];
+
+                    if (filename.endsWith('.user.png')) {
+                        db.runSql('UPDATE apps SET icon=? WHERE id=?', [ icon, appId ], iteratorCallback);
+                    } else {
+                        db.runSql('UPDATE apps SET appStoreIcon=? WHERE id=?', [ icon, appId ], iteratorCallback);
+                    }
+                }, function (error) {
+                    if (error) return next(error);
+
+                    fs.rmdir(APPICONS_DIR, { recursive: true }, next);
+                });
+            });
+        }
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN icon'),
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN appStoreIcon'),
+    ], callback);
+};

migrations/20210430230901-apps-alter-ts-update.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210501044940-blobs-add-table.js

@@ -0,0 +1,20 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    const cmd = 'CREATE TABLE blobs(' +
+      'id VARCHAR(128) NOT NULL UNIQUE,' +
+      'value MEDIUMBLOB,' +
+      'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE blobs', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210503182308-blobs-migrate-secrets.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    safe = require('safetydance');
+
+const BOX_DATA_DIR = '/home/yellowtent/boxdata';
+const PLATFORM_DATA_DIR = '/home/yellowtent/platformdata';
+
+exports.up = function (db, callback) {
+    let funcs = [];
+
+    const acmeKey = safe.fs.readFileSync(`${BOX_DATA_DIR}/acme/acme.key`);
+    if (acmeKey) {
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'acme_account_key', acmeKey ]));
+        funcs.push(fs.rmdir.bind(fs, `${BOX_DATA_DIR}/acme`, { recursive: true }));
+    }
+    const dhparams = safe.fs.readFileSync(`${BOX_DATA_DIR}/dhparams.pem`);
+    if (dhparams) {
+        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/dhparams.pem`, dhparams);
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'dhparams', dhparams ]));
+        // leave the dhparms here for the moment because startup code regenerates box nginx config and reloads nginx. at that point,
+        // nginx config of apps has not been re-generated yet and the reload fails. post 6.3, this file can be removed in start.sh
+        // funcs.push(fs.unlink.bind(fs, `${BOX_DATA_DIR}/dhparams.pem`));
+    }
+    const turnSecret = safe.fs.readFileSync(`${BOX_DATA_DIR}/addon-turn-secret`);
+    if (turnSecret) {
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'addon_turn_secret', turnSecret ]));
+        funcs.push(fs.unlink.bind(fs, `${BOX_DATA_DIR}/addon-turn-secret`));
+    }
+
+    // sftp keys get moved to platformdata in start.sh
+    const sftpPublicKey = safe.fs.readFileSync(`${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key.pub`);
+    const sftpPrivateKey = safe.fs.readFileSync(`${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key`);
+    if (sftpPublicKey) {
+        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/sftp/ssh/ssh_host_rsa_key.pub`, sftpPublicKey);
+        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/sftp/ssh/ssh_host_rsa_key`, sftpPrivateKey);
+        safe.fs.chmodSync(`${PLATFORM_DATA_DIR}/sftp/ssh/ssh_host_rsa_key`, 0o600);
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'sftp_public_key', sftpPublicKey ]));
+        funcs.push(db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?)', [ 'sftp_private_key', sftpPrivateKey ]));
+        funcs.push(fs.rmdir.bind(fs, `${BOX_DATA_DIR}/sftp`, { recursive: true }));
+    }
+
+    async.series(funcs, callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210504221634-settings-migrate-firewall.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    safe = require('safetydance');
+
+const BOX_DATA_DIR = '/home/yellowtent/boxdata';
+const PLATFORM_DATA_DIR = '/home/yellowtent/platformdata';
+
+exports.up = function (db, callback) {
+    if (!fs.existsSync(`${BOX_DATA_DIR}/firewall`)) return callback();
+
+    const ports = safe.fs.readFileSync(`${BOX_DATA_DIR}/firewall/ports.json`);
+    if (ports) {
+        safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/firewall/ports.json`, ports);
+    }
+
+    const blocklist = safe.fs.readFileSync(`${BOX_DATA_DIR}/firewall/blocklist.txt`);
+    async.series([
+        (next) => {
+            if (!blocklist) return next();
+            db.runSql('INSERT INTO settings (name, valueBlob) VALUES (?, ?)', [ 'firewall_blocklist', blocklist ], next);
+        },
+        fs.writeFile.bind(fs, `${PLATFORM_DATA_DIR}/firewall/blocklist.txt`, blocklist || ''),
+        fs.rmdir.bind(fs, `${BOX_DATA_DIR}/firewall`, { recursive: true })
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210504230054-domains-add-fallbackCertificateJson.js

@@ -0,0 +1,38 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+const CERTS_DIR = '/home/yellowtent/boxdata/certs',
+    PLATFORM_CERTS_DIR = '/home/yellowtent/platformdata/nginx/cert';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE domains ADD COLUMN fallbackCertificateJson MEDIUMTEXT', function (error) {
+        if (error) return callback(error);
+
+        db.all('SELECT * FROM domains', [ ], function (error, domains) {
+            if (error) return callback(error);
+
+            async.eachSeries(domains, function (domain, iteratorDone) {
+                // b94dbf5fa33a6d68d784571721ff44348c2d88aa seems to have moved certs from platformdata to boxdata
+                let cert = safe.fs.readFileSync(`${CERTS_DIR}/${domain.domain}.host.cert`, 'utf8');
+                let key = safe.fs.readFileSync(`${CERTS_DIR}/${domain.domain}.host.key`, 'utf8');
+
+                if (!cert) {
+                    cert = safe.fs.readFileSync(`${PLATFORM_CERTS_DIR}/${domain.domain}.host.cert`, 'utf8');
+                    key = safe.fs.readFileSync(`${PLATFORM_CERTS_DIR}/${domain.domain}.host.key`, 'utf8');
+                }
+
+                const fallbackCertificate = { cert, key };
+
+                db.runSql('UPDATE domains SET fallbackCertificateJson=? WHERE domain=?', [ JSON.stringify(fallbackCertificate), domain.domain ], iteratorDone);
+            }, callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.run(db, 'ALTER TABLE domains DROP COLUMN fallbackCertificateJson')
+    ], callback);
+};

migrations/20210505165936-subdomains-add-certificateJson.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    safe = require('safetydance');
+
+const CERTS_DIR = '/home/yellowtent/boxdata/certs';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE subdomains ADD COLUMN certificateJson MEDIUMTEXT', function (error) {
+        if (error) return callback(error);
+
+        db.all('SELECT * FROM subdomains', [ ], function (error, subdomains) {
+            if (error) return callback(error);
+
+            async.eachSeries(subdomains, function (subdomain, iteratorDone) {
+                const cert = safe.fs.readFileSync(`${CERTS_DIR}/${subdomain.subdomain}.${subdomain.domain}.user.cert`, 'utf8');
+                const key = safe.fs.readFileSync(`${CERTS_DIR}/${subdomain.subdomain}.${subdomain.domain}.user.key`, 'utf8');
+
+                if (!cert || !key) return iteratorDone();
+
+                const certificate = { cert, key };
+
+                db.runSql('UPDATE subdomains SET certificateJson=? WHERE domain=? AND subdomain=?', [ JSON.stringify(certificate), subdomain.domain, subdomain.subdomain ], iteratorDone);
+            }, callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.run(db, 'ALTER TABLE subdomains DROP COLUMN certificateJson')
+    ], callback);
+};

migrations/20210505223829-blobs-migrate-certs.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const async = require('async'),
+    child_process = require('child_process'),
+    fs = require('fs'),
+    path = require('path'),
+    safe = require('safetydance');
+
+const OLD_CERTS_DIR = '/home/yellowtent/boxdata/certs';
+const NEW_CERTS_DIR = '/home/yellowtent/platformdata/nginx/cert';
+
+exports.up = function(db, callback) {
+    fs.readdir(OLD_CERTS_DIR, function (error, filenames) {
+        if (error && error.code === 'ENOENT') return callback();
+        if (error) return callback(error);
+
+        filenames = filenames.filter(f => f.endsWith('.key') && !f.endsWith('.host.key') && !f.endsWith('.user.key')); // ignore fallback and user keys
+
+        async.eachSeries(filenames, function (filename, iteratorCallback) {
+            const privateKeyFile = filename;
+            const privateKey = fs.readFileSync(path.join(OLD_CERTS_DIR, filename));
+            const certificateFile = filename.replace(/\.key$/, '.cert');
+            const certificate = safe.fs.readFileSync(path.join(OLD_CERTS_DIR, certificateFile));
+            if (!certificate) {
+                console.log(`${certificateFile} is missing. skipping migration`);
+                return iteratorCallback();
+            }
+            const csrFile = filename.replace(/\.key$/, '.csr');
+            const csr = safe.fs.readFileSync(path.join(OLD_CERTS_DIR, csrFile));
+            if (!csr) {
+                console.log(`${csrFile} is missing. skipping migration`);
+                return iteratorCallback();
+            }
+
+            async.series([
+                db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', `cert-${privateKeyFile}`, privateKey),
+                db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', `cert-${certificateFile}`, certificate),
+                db.runSql.bind(db, 'INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', `cert-${csrFile}`, csr),
+            ], iteratorCallback);
+        }, function (error) {
+            if (error) return callback(error);
+
+            child_process.execSync(`cp ${OLD_CERTS_DIR}/* ${NEW_CERTS_DIR}`); // this way we copy the non-migrated ones like .host, .user etc as well
+            fs.rmdir(OLD_CERTS_DIR, { recursive: true }, callback);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
+

migrations/20210512210953-volumes-add-mountType-and-mountOptionsJson.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE volumes ADD COLUMN mountType VARCHAR(16) DEFAULT "noop"'),
+        db.runSql.bind(db, 'ALTER TABLE volumes ADD COLUMN mountOptionsJson TEXT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE volumes DROP COLUMN mountType'),
+        db.runSql.bind(db, 'ALTER TABLE volumes DROP COLUMN mountOptionsJson')
+    ], callback);
+};

migrations/20210517135636-add-order-by-indexes.js

@@ -0,0 +1,21 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE backups ADD INDEX creationTime_index (creationTime)'),
+        db.runSql.bind(db, 'ALTER TABLE eventlog ADD INDEX creationTime_index (creationTime)'),
+        db.runSql.bind(db, 'ALTER TABLE notifications ADD INDEX creationTime_index (creationTime)'),
+        db.runSql.bind(db, 'ALTER TABLE tasks ADD INDEX creationTime_index (creationTime)'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE backups DROP INDEX creationTime_index'),
+        db.runSql.bind(db, 'ALTER TABLE eventlog DROP INDEX creationTime_index'),
+        db.runSql.bind(db, 'ALTER TABLE notifications DROP INDEX creationTime_index'),
+        db.runSql.bind(db, 'ALTER TABLE tasks DROP INDEX creationTime_index'),
+    ], callback);
+};

migrations/20210517141957-users-rename-createdAt-to-creationTime.js

@@ -0,0 +1,33 @@
+'use strict';
+
+const async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
+        if (error) return callback(error);
+
+        db.runSql('ALTER TABLE users ADD INDEX creationTime_index (creationTime)', function (error) {
+            if (error) return callback(error);
+
+            db.all('SELECT id, createdAt FROM users', function (error, results) {
+                if (error) return callback(error);
+
+                async.eachSeries(results, function (r, iteratorDone) {
+                    const creationTime = new Date(r.createdAt);
+                    db.runSql('UPDATE users SET creationTime=? WHERE id=?', [ creationTime, r.id ], iteratorDone);
+                }, function (error) {
+                    if (error) return callback(error);
+
+                    db.runSql('ALTER TABLE users DROP COLUMN createdAt', callback);
+                });
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN creationTime', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210517194116-backups-provider-mountpoint.js

@@ -0,0 +1,27 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
+        if (error || results.length === 0) return callback(error);
+
+        const backupConfig = JSON.parse(results[0].value);
+        if (backupConfig.provider === 'sshfs' || backupConfig.provider === 'cifs' || backupConfig.provider === 'nfs' || backupConfig.externalDisk) {
+            backupConfig.chown = backupConfig.provider === 'nfs' || backupConfig.provider === 'sshfs' || backupConfig.externalDisk;
+            backupConfig.preserveAttributes = !!backupConfig.externalDisk;
+            backupConfig.provider = 'mountpoint';
+            if (backupConfig.externalDisk) {
+                backupConfig.mountPoint = backupConfig.backupFolder;
+                backupConfig.prefix = '';
+                delete backupConfig.backupFolder;
+                delete backupConfig.externalDisk;
+            }
+            db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [JSON.stringify(backupConfig)], callback);
+        } else {
+            callback();
+        }
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210528205138-notifications-drop-userId.js

@@ -0,0 +1,13 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE notifications DROP COLUMN userId', function (error) {
+        if (error) return callback(error);
+
+        db.runSql('DELETE FROM notifications', callback); // just clear notifications table
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE notifications ADD COLUMN userId VARCHAR(128) NOT NULL', callback);
+};

migrations/20210625061443-volumes-change-noop-to-mountpoint.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM volumes', function (error, volumes) {
+        if (error || volumes.length === 0) return callback(error);
+
+        async.eachSeries(volumes, function (volume, iteratorDone) {
+            if (volume.mountType !== 'noop') return iteratorDone();
+
+            let mountType;
+            if (safe.child_process.execSync(`mountpoint -q -- ${volume.hostPath}`)) {
+                mountType = 'mountpoint';
+            } else {
+                mountType = 'filesystem';
+            }
+            db.runSql('UPDATE volumes SET mountType=? WHERE id=?', [ mountType, volume.id ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210707154724-users-migrate-avatar.js

@@ -0,0 +1,13 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('UPDATE users SET avatar="gravatar" WHERE avatar IS NULL', function (error) {
+        if (error) return callback(error);
+
+        db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB NOT NULL', callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY avatar MEDIUMBLOB', callback);
+};

migrations/20210729191138-domains-update-wellKnown-matrix-client.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const async = require('async'),
+    safe = require('safetydance');
+
+exports.up = function(db, callback) {
+    db.all('SELECT * from domains', [], function (error, results) {
+        if (error) return callback(error);
+
+        async.eachSeries(results, function (r, iteratorDone) {
+            if (!r.wellKnownJson) return iteratorDone();
+
+            const wellKnown = safe.JSON.parse(r.wellKnownJson);
+            if (!wellKnown || !wellKnown['matrix/server']) return iteratorDone();
+            const matrixHostname = JSON.parse(wellKnown['matrix/server'])['m.server'];
+
+            wellKnown['matrix/client'] = JSON.stringify({
+                'm.homeserver': {
+                    'base_url': 'https://' + matrixHostname
+                }
+            });
+
+            db.runSql('UPDATE domains SET wellKnownJson=? WHERE domain=?', [ JSON.stringify(wellKnown), r.domain ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20210809203855-appAddonConfigs-alter-value.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appAddonConfigs MODIFY value TEXT NOT NULL', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appAddonConfigs MODIFY value VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210820172800-users-alter-loginLocationsJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY loginLocationsJson MEDIUMTEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY loginLocationsJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20210921022843-apps-add-operatorsJson.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN operatorsJson TEXT', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN operatorsJson', callback);
+};

migrations/20210927211658-apps-add-crontab.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN crontab TEXT', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN crontab', callback);
+};

migrations/20211001095256-users-add-inviteToken.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN inviteToken VARCHAR(128) DEFAULT ""', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN inviteToken', callback);
+};

migrations/20211001160903-apps-add-inbox.js

@@ -0,0 +1,19 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN enableInbox BOOLEAN DEFAULT 0'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxName VARCHAR(128)'),
+        db.runSql.bind(db, 'ALTER TABLE apps ADD COLUMN inboxDomain VARCHAR(128)'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN enableInbox'),
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxName'),
+        db.runSql.bind(db, 'ALTER TABLE apps DROP COLUMN inboxDomain'),
+    ], callback);
+};

migrations/20211006200150-domains-ensure-fallbackCertificate.js

@@ -0,0 +1,33 @@
+'use strict';
+
+const async = require('async'),
+    reverseProxy = require('../src/reverseproxy.js'),
+    safe = require('safetydance');
+
+const NGINX_CERT_DIR = '/home/yellowtent/platformdata/nginx/cert';
+
+// ensure fallbackCertificate of domains are present in database and the cert dir. it seems a bad migration lost them.
+// https://forum.cloudron.io/topic/5683/data-argument-must-be-of-type-received-null-error-during-restore-process
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM domains', [ ], function (error, domains) {
+        if (error) return callback(error);
+
+        async.eachSeries(domains, async function (domain, iteratorDone) {
+            let fallbackCertificate = safe.JSON.parse(domain.fallbackCertificateJson);
+            if (!fallbackCertificate || !fallbackCertificate.cert || !fallbackCertificate.key) {
+                let error;
+                [error, fallbackCertificate] = await safe(reverseProxy.generateFallbackCertificate(domain.domain));
+                if (error) return iteratorDone(error);
+            }
+
+            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.cert`, fallbackCertificate.cert, 'utf8')) return iteratorDone(safe.error);
+            if (!safe.fs.writeFileSync(`${NGINX_CERT_DIR}/${domain.domain}.host.key`, fallbackCertificate.key, 'utf8')) return iteratorDone(safe.error);
+
+            db.runSql('UPDATE domains SET fallbackCertificateJson=? WHERE domain=?', [ JSON.stringify(fallbackCertificate), domain.domain ], iteratorDone);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20211008171126-mailboxes-add-enablePop3.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes ADD COLUMN enablePop3 BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN enablePop3', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20211012020255-mail-add-dkimKeyJson.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const async = require('async'),
+    fs = require('fs'),
+    path = require('path'),
+    safe = require('safetydance');
+
+const MAIL_DATA_DIR = '/home/yellowtent/boxdata/mail';
+const DKIM_DIR = `${MAIL_DATA_DIR}/dkim`;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mail ADD COLUMN dkimKeyJson MEDIUMTEXT', function (error) {
+        if (error) return callback(error);
+
+        fs.readdir(DKIM_DIR, function (error, filenames) {
+            if (error && error.code === 'ENOENT') return callback();
+            if (error) return callback(error);
+
+            async.eachSeries(filenames, function (filename, iteratorCallback) {
+                const domain = filename;
+                const publicKey = safe.fs.readFileSync(path.join(DKIM_DIR, domain, 'public'), 'utf8');
+                const privateKey = safe.fs.readFileSync(path.join(DKIM_DIR, domain, 'private'), 'utf8');
+                if (!publicKey || !privateKey) return iteratorCallback();
+
+                const dkimKey = {
+                    publicKey,
+                    privateKey
+                };
+
+                db.runSql('UPDATE mail SET dkimKeyJson=? WHERE domain=?', [ JSON.stringify(dkimKey), domain ], iteratorCallback);
+            }, function (error) {
+                if (error) return callback(error);
+
+                fs.rmdir(DKIM_DIR, { recursive: true }, callback);
+            });
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.run(db, 'ALTER TABLE mail DROP COLUMN dkimKeyJson')
+    ], callback);
+};

migrations/schema.sql

@@ -6,7 +6,7 @@
 #### Strict mode is enabled
 #### VARCHAR - stored as part of table row (use for strings)
 #### TEXT - stored offline from table row (use for strings)
-#### BLOB - stored offline from table row (use for binary data)
+#### BLOB (64KB), MEDIUMBLOB (16MB), LONGBLOB (4GB) - stored offline from table row (use for binary data)
 #### https://dev.mysql.com/doc/refman/5.0/en/storage-requirements.html
 #### Times are stored in the database in UTC. And precision is seconds
 
@@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS users(
     email VARCHAR(254) NOT NULL UNIQUE,
     password VARCHAR(1024) NOT NULL,
     salt VARCHAR(512) NOT NULL,
-    createdAt VARCHAR(512) NOT NULL,
+    creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     displayName VARCHAR(512) DEFAULT "",
     fallbackEmail VARCHAR(512) DEFAULT "",
@@ -28,10 +28,14 @@ CREATE TABLE IF NOT EXISTS users(
     twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
     source VARCHAR(128) DEFAULT "",
     role VARCHAR(32),
+    inviteToken VARCHAR(128) DEFAULT "",
     resetToken VARCHAR(128) DEFAULT "",
     resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     active BOOLEAN DEFAULT 1,
+    avatar MEDIUMBLOB NOT NULL,
+    loginLocationsJson MEDIUMTEXT, // { locations: [{ ip, userAgent, city, country, ts }] }
 
+    INDEX creationTime_index (creationTime),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS userGroups(
@@ -55,6 +59,7 @@ CREATE TABLE IF NOT EXISTS tokens(
     clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
     expires BIGINT NOT NULL, // FIXME: make this a timestamp
+    lastUsedTime TIMESTAMP NULL,
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS apps(
@@ -78,8 +83,12 @@ CREATE TABLE IF NOT EXISTS apps(
     reverseProxyConfigJson TEXT, // { robotsTxt, csp }
     enableBackup BOOLEAN DEFAULT 1, // misnomer: controls automatic daily backups
     enableAutomaticUpdate BOOLEAN DEFAULT 1,
+    enableMailbox BOOLEAN DEFAULT 1, // whether sendmail addon is enabled
     mailboxName VARCHAR(128), // mailbox of this app
     mailboxDomain VARCHAR(128), // mailbox domain of this apps
+    enableInbox BOOLEAN DEFAULT 0, // whether recvmail addon is enabled
+    inboxName VARCHAR(128), // mailbox of this app
+    inboxDomain VARCHAR(128), // mailbox domain of this apps
     label VARCHAR(128),     // display name
     tagsJson VARCHAR(2048), // array of tags
     dataDir VARCHAR(256) UNIQUE,
@@ -87,6 +96,8 @@ CREATE TABLE IF NOT EXISTS apps(
     errorJson TEXT,
     servicesConfigJson TEXT, // app services configuration
     containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair'
+    appStoreIcon MEDIUMBLOB,
+    icon MEDIUMBLOB,
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
@@ -103,13 +114,14 @@ CREATE TABLE IF NOT EXISTS appPortBindings(
 CREATE TABLE IF NOT EXISTS settings(
     name VARCHAR(128) NOT NULL UNIQUE,
     value TEXT,
+    valueBlob MEDIUMBLOB,
     PRIMARY KEY(name));
 
 CREATE TABLE IF NOT EXISTS appAddonConfigs(
     appId VARCHAR(128) NOT NULL,
     addonId VARCHAR(32) NOT NULL,
     name VARCHAR(128) NOT NULL,
-    value VARCHAR(512) NOT NULL,
+    value TEXT NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id));
 
 CREATE TABLE IF NOT EXISTS appEnvVars(
@@ -131,6 +143,7 @@ CREATE TABLE IF NOT EXISTS backups(
     format VARCHAR(16) DEFAULT "tgz",
     preserveSecs INTEGER DEFAULT 0,
 
+    INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
 CREATE TABLE IF NOT EXISTS eventlog(
@@ -140,6 +153,7 @@ CREATE TABLE IF NOT EXISTS eventlog(
     data TEXT, /* free flowing json based on action */
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
 
+    INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
 CREATE TABLE IF NOT EXISTS domains(
@@ -150,6 +164,8 @@ CREATE TABLE IF NOT EXISTS domains(
     tlsConfigJson TEXT, /* JSON containing the tls provider config */
     wellKnownJson TEXT, /* JSON containing well known docs for this domain */
 
+    fallbackCertificateJson MEDIUMTEXT,
+
     PRIMARY KEY (domain))
 
     /* the default db collation is utf8mb4_unicode_ci but for the app table domain constraint we have to use the old one */
@@ -164,6 +180,7 @@ CREATE TABLE IF NOT EXISTS mail(
     relayJson TEXT,
     bannerJson TEXT,
 
+    dkimKeyJson MEDIUMTEXT,
     dkimSelector VARCHAR(128) NOT NULL DEFAULT "cloudron",
 
     FOREIGN KEY(domain) REFERENCES domains(domain),
@@ -189,6 +206,8 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     membersOnly BOOLEAN DEFAULT false,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     domain VARCHAR(128),
+    active BOOLEAN DEFAULT 1,
+    enablePop3 BOOLEAN DEFAULT 0,
 
     FOREIGN KEY(domain) REFERENCES mail(domain),
     FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
@@ -200,6 +219,8 @@ CREATE TABLE IF NOT EXISTS subdomains(
     subdomain VARCHAR(128) NOT NULL,
     type VARCHAR(128) NOT NULL, /* primary or redirect */
 
+    certificateJson MEDIUMTEXT,
+
     FOREIGN KEY(domain) REFERENCES domains(domain),
     FOREIGN KEY(appId) REFERENCES apps(id),
     UNIQUE (subdomain, domain));
@@ -207,23 +228,27 @@ CREATE TABLE IF NOT EXISTS subdomains(
 CREATE TABLE IF NOT EXISTS tasks(
     id int NOT NULL AUTO_INCREMENT,
     type VARCHAR(32) NOT NULL,
+    argsJson TEXT,
     percent INTEGER DEFAULT 0,
     message TEXT,
     errorJson TEXT,
     resultJson TEXT,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+    INDEX creationTime_index (creationTime),
     PRIMARY KEY (id));
 
 CREATE TABLE IF NOT EXISTS notifications(
     id int NOT NULL AUTO_INCREMENT,
-    userId VARCHAR(128) NOT NULL,
     eventId VARCHAR(128), // reference to eventlog. can be null
     title VARCHAR(512) NOT NULL,
     message TEXT,
     acknowledged BOOLEAN DEFAULT false,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    UNIQUE KEY appPasswords_name_appId_identifier (name, userId, identifier),
+
+    INDEX creationTime_index (creationTime),
+    FOREIGN KEY(eventId) REFERENCES eventlog(id),
     PRIMARY KEY (id)
 );
 
@@ -234,6 +259,7 @@ CREATE TABLE IF NOT EXISTS appPasswords(
     identifier VARCHAR(128) NOT NULL, // resourceId: app id or mail or webadmin
     hashedPassword VARCHAR(1024) NOT NULL,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE KEY appPasswords_name_appId_identifier (name, userId, identifier)
     FOREIGN KEY(userId) REFERENCES users(id),
 
     PRIMARY KEY (id)
@@ -244,6 +270,8 @@ CREATE TABLE IF NOT EXISTS volumes(
     name VARCHAR(256) NOT NULL UNIQUE,
     hostPath VARCHAR(1024) NOT NULL UNIQUE,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    mountType VARCHAR(16) DEFAULT "noop",
+    mountOptionsJson TEXT,
     PRIMARY KEY (id)
 );
 
@@ -255,4 +283,9 @@ CREATE TABLE IF NOT EXISTS appMounts(
     FOREIGN KEY(appId) REFERENCES apps(id),
     FOREIGN KEY(volumeId) REFERENCES volumes(id));
 
+CREATE TABLE IF NOT EXISTS blobs(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    value TEXT,
+    PRIMARY KEY(id));
+
 CHARACTER SET utf8 COLLATE utf8_bin;

package.json

@@ -10,20 +10,17 @@
     "type": "git",
     "url": "https://git.cloudron.io/cloudron/box.git"
   },
-  "engines": {
-    "node": ">=4.0.0 <=4.1.1"
-  },
   "dependencies": {
-    "@google-cloud/dns": "^1.2.9",
-    "@google-cloud/storage": "^2.5.0",
+    "@google-cloud/dns": "^2.2.0",
+    "@google-cloud/storage": "^5.8.5",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
-    "async": "^2.6.3",
-    "aws-sdk": "^2.828.0",
+    "async": "^3.2.0",
+    "aws-sdk": "^2.936.0",
     "basic-auth": "^2.0.1",
     "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.10.1",
+    "cloudron-manifestformat": "^5.10.2",
     "connect": "^3.7.0",
-    "connect-lastmile": "^2.0.0",
+    "connect-lastmile": "^2.1.1",
     "connect-timeout": "^1.9.0",
     "cookie-parser": "^1.4.5",
     "cookie-session": "^1.4.0",
@@ -31,59 +28,59 @@
     "db-migrate": "^0.11.12",
     "db-migrate-mysql": "^2.1.2",
     "debug": "^4.3.1",
-    "dockerode": "^2.5.8",
-    "ejs": "^2.6.1",
+    "delay": "^5.0.0",
+    "dockerode": "^3.3.0",
+    "ejs": "^3.1.6",
     "ejs-cli": "^2.2.1",
     "express": "^4.17.1",
-    "ipaddr.js": "^2.0.0",
-    "js-yaml": "^3.14.0",
-    "json": "^9.0.6",
+    "ipaddr.js": "^2.0.1",
+    "js-yaml": "^4.1.0",
+    "json": "^11.0.0",
     "jsonwebtoken": "^8.5.1",
-    "ldapjs": "^2.2.3",
-    "lodash": "^4.17.20",
+    "ldapjs": "^2.3.0",
+    "lodash": "^4.17.21",
     "lodash.chunk": "^4.2.0",
-    "mime": "^2.5.0",
+    "mime": "^2.5.2",
     "moment": "^2.29.1",
-    "moment-timezone": "^0.5.32",
+    "moment-timezone": "^0.5.33",
     "morgan": "^1.10.0",
     "multiparty": "^4.2.2",
-    "mustache-express": "^1.3.0",
+    "mustache-express": "^1.3.1",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.4.17",
+    "nodemailer": "^6.6.2",
     "nodemailer-smtp-transport": "^2.7.4",
     "once": "^1.4.0",
-    "pretty-bytes": "^5.5.0",
+    "pretty-bytes": "^5.6.0",
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
     "qrcode": "^1.4.4",
-    "readdirp": "^3.5.0",
+    "readdirp": "^3.6.0",
     "request": "^2.88.2",
-    "rimraf": "^2.6.3",
     "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^1.1.1",
-    "semver": "^6.1.1",
-    "showdown": "^1.9.1",
+    "safetydance": "^2.2.0",
+    "semver": "^7.3.5",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^5.3.1",
+    "superagent": "^6.1.0",
     "supererror": "^0.7.2",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
     "tar-stream": "^2.2.0",
     "tldjs": "^2.3.1",
-    "underscore": "^1.12.0",
-    "uuid": "^3.4.0",
-    "validator": "^11.0.0",
-    "ws": "^7.4.2",
+    "ua-parser-js": "^0.7.28",
+    "underscore": "^1.13.1",
+    "uuid": "^8.3.2",
+    "validator": "^13.6.0",
+    "ws": "^7.5.1",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
     "hock": "^1.4.1",
     "js2xmlparser": "^4.0.1",
-    "mocha": "^6.2.3",
+    "mocha": "^9.0.1",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
-    "nock": "^10.0.6",
-    "node-sass": "^4.14.1",
+    "nock": "^13.1.0",
+    "node-sass": "^6.0.1",
     "recursive-readdir": "^2.2.2"
   },
   "scripts": {

runTests

@@ -22,8 +22,8 @@ fi
 mkdir -p ${DATA_DIR}
 cd ${DATA_DIR}
 mkdir -p appsdata
-mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com boxdata/sftp/ssh
-mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks
+mkdir -p boxdata/box boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com
+mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall platformdata/update
 sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test
 
 # translations
@@ -34,12 +34,15 @@ cp -r ${source_dir}/../dashboard/dist/translation/* box/dashboard/dist/translati
 echo "=> Generating a localhost selfsigned cert"
 openssl req -x509 -newkey rsa:2048 -keyout platformdata/nginx/cert/host.key -out platformdata/nginx/cert/host.cert -days 3650 -subj '/CN=localhost' -nodes -config <(cat /etc/ssl/openssl.cnf <(printf "\n[SAN]\nsubjectAltName=DNS:*.localhost"))
 
-# generate legacy key format for sftp
-ssh-keygen -m PEM -t rsa -f boxdata/sftp/ssh/ssh_host_rsa_key -q -N ""
-
-# clear out any containers
-echo "=> Delete all docker containers first"
-docker ps -qa | xargs --no-run-if-empty docker rm -f
+# clear out any containers if FAST is unset
+if [[ -z ${FAST+x} ]]; then
+    echo "=> Delete all docker containers first"
+    docker ps -qa --filter "label=isCloudronManaged" | xargs --no-run-if-empty docker rm -f
+    docker rm -f mysql-server
+    echo "==> To skip this run with: FAST=1 ./runTests"
+else
+    echo "==> WARNING!! Skipping docker container cleanup, the database might not be pristine!"
+fi
 
 # create docker network (while the infra code does this, most tests skip infra setup)
 docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron || true
@@ -60,6 +63,12 @@ while ! mysqladmin ping -h"${MYSQL_IP}" --silent; do
     sleep 1
 done
 
+echo "=> Ensure local base image"
+docker pull cloudron/base:3.0.0@sha256:455c70428723e3a823198c57472785437eb6eab082e79b3ff04ea584faf46e92
+
+echo "=> Create iptables blocklist"
+sudo ipset create cloudron_blocklist hash:net || true
+
 echo "=> Starting cloudron-syslog"
 cloudron-syslog --logdir "${DATA_DIR}/platformdata/logs/" &
 

scripts/cloudron-setup

@@ -2,6 +2,12 @@
 
 set -eu -o pipefail
 
+function exitHandler() {
+    rm -f /etc/update-motd.d/91-cloudron-install-in-progress
+}
+
+trap exitHandler EXIT
+
 # change this to a hash when we make a upgrade release
 readonly LOG_FILE="/var/log/cloudron-setup.log"
 readonly MINIMUM_DISK_SIZE_GB="18" # this is the size of "/" and required to fit in docker images 18 is a safe bet for different reporting on 20GB min
@@ -35,7 +41,13 @@ if [[ "${disk_size_gb}" -lt "${MINIMUM_DISK_SIZE_GB}" ]]; then
     exit 1
 fi
 
-if systemctl -q is-active box; then
+if [[ "$(uname -m)" != "x86_64" ]]; then
+    echo "Error: Cloudron only supports amd64/x86_64"
+    exit 1
+fi
+
+# do not use is-active in case box service is down and user attempts to re-install
+if systemctl cat box.service >/dev/null 2>&1; then
     echo "Error: Cloudron is already installed. To reinstall, start afresh"
     exit 1
 fi
@@ -92,6 +104,31 @@ if [[ "${ubuntu_version}" != "16.04" && "${ubuntu_version}" != "18.04" && "${ubu
     exit 1
 fi
 
+if which nginx >/dev/null || which docker >/dev/null || which node > /dev/null; then
+    echo "Error: Some packages like nginx/docker/nodejs are already installed. Cloudron requires specific versions of these packages and will install them as part of it's installation. Please start with a fresh Ubuntu install and run this script again." > /dev/stderr
+    exit 1
+fi
+
+# Install MOTD file for stack script style installations. this is removed by the trap exit handler. Heredoc quotes prevents parameter expansion
+cat > /etc/update-motd.d/91-cloudron-install-in-progress <<'EOF'
+#!/bin/bash
+
+printf "**********************************************************************\n\n"
+
+printf "\t\t\tWELCOME TO CLOUDRON\n"
+printf "\t\t\t-------------------\n"
+
+printf '\n\e[1;32m%-6s\e[m\n\n' "Cloudron is installing. Run 'tail -f /var/log/cloudron-setup.log' to view progress."
+
+printf "Cloudron overview - https://docs.cloudron.io/ \n"
+printf "Cloudron setup - https://docs.cloudron.io/installation/#setup \n"
+
+printf "\nFor help and more information, visit https://forum.cloudron.io\n\n"
+
+printf "**********************************************************************\n"
+EOF
+chmod +x /etc/update-motd.d/91-cloudron-install-in-progress
+
 # Can only write after we have confirmed script has root access
 echo "Running cloudron-setup with args : $@" > "${LOG_FILE}"
 
@@ -193,7 +230,7 @@ if [[ "${rebootServer}" == "true" ]]; then
     read -p "The server has to be rebooted to apply all the settings. Reboot now ? [Y/n] " yn
     yn=${yn:-y}
     case $yn in
-        [Yy]* ) systemctl reboot;;
+        [Yy]* ) exitHandler; systemctl reboot;;
         * ) exit;;
     esac
 fi

scripts/cloudron-support

@@ -37,12 +37,11 @@ while true; do
         # fall through
         ;&
     --owner-login)
-        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY createdAt LIMIT 1" 2>/dev/null)
+        admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' AND username IS NOT NULL ORDER BY creationTime LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
-        ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
-        printf '{"%s":"%s"}\n' "${admin_username}" "${admin_password}" > "${ghost_file}"
-        chown yellowtent:yellowtent "${ghost_file}" && chmod o-r,g-r "${ghost_file}"
-        echo "Login as ${admin_username} / ${admin_password} . This password may only be used once. ${ghost_file} will be automatically removed after use."
+        dashboard_domain=$(mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" 2>/dev/null)
+        mysql -NB -uroot -ppassword -e "INSERT INTO box.settings (name, value) VALUES ('ghosts_config', '{\"${admin_username}\":\"${admin_password}\"}') ON DUPLICATE KEY UPDATE name='ghosts_config', value='{\"${admin_username}\":\"${admin_password}\"}'" 2>/dev/null
+        echo "Login at https://${dashboard_domain} as ${admin_username} / ${admin_password} . This password may only be used once."
         exit 0
         ;;
     --) break;;
@@ -73,6 +72,9 @@ echo -n "Generating Cloudron Support stats..."
 # clear file
 rm -rf $OUT
 
+echo -e $LINE"DASHBOARD DOMAIN"$LINE >> $OUT
+mysql -NB -uroot -ppassword -e "SELECT value FROM box.settings WHERE name='admin_fqdn'" &>> $OUT 2>/dev/null || true
+
 echo -e $LINE"PROVIDER"$LINE >> $OUT
 cat /etc/cloudron/PROVIDER &>> $OUT || true
 
@@ -99,7 +101,7 @@ systemctl status --lines=100 box mysql unbound cloudron-syslog nginx collectd do
 echo -e $LINE"Box logs"$LINE >> $OUT
 tail -n 100 /home/yellowtent/platformdata/logs/box.log &>> $OUT
 
-echo -e $LINE"Firewall chains"$LINE >> $OUT
+echo -e $LINE"Interface Info"$LINE >> $OUT
 ip addr &>> $OUT
 
 echo -e $LINE"Firewall chains"$LINE >> $OUT

scripts/createReleaseTarball

@@ -41,8 +41,8 @@ if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if [[ "$(node --version)" != "v10.18.1" ]]; then
-    echo "This script requires node 10.18.1"
+if [[ "$(node --version)" != "v14.17.6" ]]; then
+    echo "This script requires node 14.17.6"
     exit 1
 fi
 

scripts/installer.sh

@@ -11,6 +11,52 @@ if [[ ${EUID} -ne 0 ]]; then
     exit 1
 fi
 
+function log() {
+  echo -e "$(date +'%Y-%m-%dT%H:%M:%S')" "==> installer: $1"
+}
+
+apt_ready="no"
+function prepare_apt_once() {
+    [[ "${apt_ready}" == "yes" ]] && return
+
+    log "Making sure apt is in a good state"
+
+    log "Waiting for all dpkg tasks to finish..."
+    while fuser /var/lib/dpkg/lock; do
+        sleep 1
+    done
+
+    # it's unclear what needs to be run first or whether both these command should be run. so keep trying both
+    for count in {1..3}; do
+        # alternative to apt-install -y --fix-missing ?
+        if ! dpkg --force-confold --configure -a; then
+            log "dpkg reconfigure failed (try $count)"
+            dpkg_configure="no"
+        else
+            dpkg_configure="yes"
+        fi
+
+        if ! apt update -y; then
+            log "apt update failed (try $count)"
+            apt_update="no"
+        else
+            apt_update="yes"
+        fi
+
+        [[ "${dpkg_configure}" == "yes" && "${apt_update}" == "yes" ]] && break
+
+        sleep 1
+    done
+
+    apt_ready="yes"
+
+    if [[ "${dpkg_configure}" == "yes" && "${apt_update}" == "yes" ]]; then
+        log "apt is ready"
+    else
+        log "apt is not ready but proceeding anyway"
+    fi
+}
+
 readonly user=yellowtent
 readonly box_src_dir=/home/${user}/box
 
@@ -21,36 +67,23 @@ readonly box_src_tmp_dir="$(realpath ${script_dir}/..)"
 readonly ubuntu_version=$(lsb_release -rs)
 readonly ubuntu_codename=$(lsb_release -cs)
 
-readonly is_update=$(systemctl is-active box && echo "yes" || echo "no")
+readonly is_update=$(systemctl is-active -q box && echo "yes" || echo "no")
 
-echo "==> installer: Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION) <=="
+log "Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION)"
 
-echo "==> installer: updating docker"
+log "updating docker"
 
-if [[ $(docker version --format {{.Client.Version}}) != "19.03.12" ]]; then
+readonly docker_version=20.10.7
+if [[ $(docker version --format {{.Client.Version}}) != "${docker_version}" ]]; then
     # there are 3 packages for docker - containerd, CLI and the daemon
-    curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.13-2_amd64.deb" -o /tmp/containerd.deb
-    curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
-    curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_19.03.12~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.4.6-1_amd64.deb" -o /tmp/containerd.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
+    $curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_${docker_version}~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
 
-    echo "==> installer: Waiting for all dpkg tasks to finish..."
-    while fuser /var/lib/dpkg/lock; do
-        sleep 1
-    done
-
-    while ! dpkg --force-confold --configure -a; do
-        echo "==> installer: Failed to fix packages. Retry"
-        sleep 1
-    done
-
-    # the latest docker might need newer packages
-    while ! apt update -y; do
-        echo "==> installer: Failed to update packages. Retry"
-        sleep 1
-    done
+    prepare_apt_once
 
     while ! apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb; do
-        echo "==> installer: Failed to install docker. Retry"
+        log "Failed to install docker. Retry"
         sleep 1
     done
 
@@ -59,24 +92,36 @@ fi
 
 readonly nginx_version=$(nginx -v 2>&1)
 if [[ "${nginx_version}" != *"1.18."* ]]; then
-    echo "==> installer: installing nginx 1.18"
-    curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
+    log "installing nginx 1.18"
+    $curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-2~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
+
+    prepare_apt_once
+
     # apt install with install deps (as opposed to dpkg -i)
     apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
     rm /tmp/nginx.deb
 fi
 
-# Cloudron 6 on ubuntu 20 installed recommended packages of collectd -> libinotify -> gnome->shell
-apt remove -y gnome-shell || true
-apt -y autoremove || true
+if ! which mount.nfs; then
+    log "installing nfs-common"
+    prepare_apt_once
+    apt install -y nfs-common
+fi
 
-echo "==> installer: updating node"
-if [[ "$(node --version)" != "v10.18.1" ]]; then
-    mkdir -p /usr/local/node-10.18.1
-    $curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
-    ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
-    ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
-    rm -rf /usr/local/node-10.15.1
+if ! which sshfs; then
+    log "installing sshfs"
+    prepare_apt_once
+    apt install -y sshfs
+fi
+
+log "updating node"
+readonly node_version=14.17.6
+if [[ "$(node --version)" != "v${node_version}" ]]; then
+    mkdir -p /usr/local/node-${node_version}
+    $curl -sL https://nodejs.org/dist/v${node_version}/node-v${node_version}-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-${node_version}
+    ln -sf /usr/local/node-${node_version}/bin/node /usr/bin/node
+    ln -sf /usr/local/node-${node_version}/bin/npm /usr/bin/npm
+    rm -rf /usr/local/node-14.15.4
 fi
 
 # this is here (and not in updater.js) because rebuild requires the above node
@@ -87,31 +132,31 @@ for try in `seq 1 10`; do
     # however by default npm drops privileges for npm rebuild
     # https://docs.npmjs.com/misc/config#unsafe-perm
     if cd "${box_src_tmp_dir}" && npm rebuild --unsafe-perm; then break; fi
-    echo "==> installer: Failed to rebuild, trying again"
+    log "Failed to rebuild, trying again"
     sleep 5
 done
 
 if [[ ${try} -eq 10 ]]; then
-    echo "==> installer: npm rebuild failed, giving up"
+    log "npm rebuild failed, giving up"
     exit 4
 fi
 
-echo "==> installer: downloading new addon images"
+log "downloading new addon images"
 images=$(node -e "var i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
-echo -e "\tPulling docker images: ${images}"
+log "\tPulling docker images: ${images}"
 for image in ${images}; do
-    if ! docker pull "${image}"; then           # this pulls the image using the sha256
-        echo "==> installer: Could not pull ${image}"
-        exit 5
-    fi
-    if ! docker pull "${image%@sha256:*}"; then  # this will tag the image for readability
-        echo "==> installer: Could not pull ${image%@sha256:*}"
-        exit 6
-   fi
+    while ! docker pull "${image}"; do           # this pulls the image using the sha256
+        log "Could not pull ${image}"
+        sleep 5
+    done
+    while ! docker pull "${image%@sha256:*}"; do  # this will tag the image for readability
+        log "Could not pull ${image%@sha256:*}"
+        sleep 5
+   done
 done
 
-echo "==> installer: update cloudron-syslog"
+log "update cloudron-syslog"
 CLOUDRON_SYSLOG_DIR=/usr/local/cloudron-syslog
 CLOUDRON_SYSLOG="${CLOUDRON_SYSLOG_DIR}/bin/cloudron-syslog"
 CLOUDRON_SYSLOG_VERSION="1.0.3"
@@ -119,7 +164,7 @@ while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLO
     rm -rf "${CLOUDRON_SYSLOG_DIR}"
     mkdir -p "${CLOUDRON_SYSLOG_DIR}"
     if npm install --unsafe-perm -g --prefix "${CLOUDRON_SYSLOG_DIR}" cloudron-syslog@${CLOUDRON_SYSLOG_VERSION}; then break; fi
-    echo "===> installer: Failed to install cloudron-syslog, trying again"
+    log "Failed to install cloudron-syslog, trying again"
     sleep 5
 done
 
@@ -128,17 +173,17 @@ if ! id "${user}" 2>/dev/null; then
 fi
 
 if [[ "${is_update}" == "yes" ]]; then
-    echo "==> installer: stop box service for update"
+    log "stop box service for update"
     ${box_src_dir}/setup/stop.sh
 fi
 
 # ensure we are not inside the source directory, which we will remove now
 cd /root
 
-echo "==> installer: switching the box code"
+log "switching the box code"
 rm -rf "${box_src_dir}"
 mv "${box_src_tmp_dir}" "${box_src_dir}"
 chown -R "${user}:${user}" "${box_src_dir}"
 
-echo "==> installer: calling box setup script"
+log "calling box setup script"
 "${box_src_dir}/setup/start.sh"

setup/start.sh

@@ -5,14 +5,19 @@ set -eu -o pipefail
 # This script is run after the box code is switched. This means that this script
 # should pretty much always succeed. No network logic/download code here.
 
-echo "==> Cloudron Start"
+function log() {
+  echo -e "$(date +'%Y-%m-%dT%H:%M:%S')" "==> start: $1"
+}
+
+log "Cloudron Start"
 
 readonly USER="yellowtent"
 readonly HOME_DIR="/home/${USER}"
 readonly BOX_SRC_DIR="${HOME_DIR}/box"
-readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata" # platform data
-readonly APPS_DATA_DIR="${HOME_DIR}/appsdata" # app data
-readonly BOX_DATA_DIR="${HOME_DIR}/boxdata" # box data
+readonly PLATFORM_DATA_DIR="${HOME_DIR}/platformdata"
+readonly APPS_DATA_DIR="${HOME_DIR}/appsdata"
+readonly BOX_DATA_DIR="${HOME_DIR}/boxdata/box"
+readonly MAIL_DATA_DIR="${HOME_DIR}/boxdata/mail"
 
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly json="$(realpath ${script_dir}/../node_modules/.bin/json)"
@@ -26,7 +31,7 @@ if ! getent group media; then
     addgroup --gid 500 --system media
 fi
 
-echo "==> Configuring docker"
+log "Configuring docker"
 cp "${script_dir}/start/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
 systemctl enable apparmor
 systemctl restart apparmor
@@ -37,16 +42,18 @@ docker network create --subnet=172.18.0.0/16 --ip-range=172.18.0.0/20 cloudron |
 
 mkdir -p "${BOX_DATA_DIR}"
 mkdir -p "${APPS_DATA_DIR}"
+mkdir -p "${MAIL_DATA_DIR}"
 
 # keep these in sync with paths.js
-echo "==> Ensuring directories"
+log "Ensuring directories"
 
 mkdir -p "${PLATFORM_DATA_DIR}/graphite"
 mkdir -p "${PLATFORM_DATA_DIR}/mysql"
 mkdir -p "${PLATFORM_DATA_DIR}/postgresql"
 mkdir -p "${PLATFORM_DATA_DIR}/mongodb"
 mkdir -p "${PLATFORM_DATA_DIR}/redis"
-mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner"
+mkdir -p "${PLATFORM_DATA_DIR}/addons/mail/banner" \
+         "${PLATFORM_DATA_DIR}/addons/mail/dkim"
 mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
 mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
 mkdir -p "${PLATFORM_DATA_DIR}/acme"
@@ -57,21 +64,14 @@ mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
          "${PLATFORM_DATA_DIR}/logs/crash" \
          "${PLATFORM_DATA_DIR}/logs/collectd"
 mkdir -p "${PLATFORM_DATA_DIR}/update"
-
-mkdir -p "${BOX_DATA_DIR}/appicons"
-mkdir -p "${BOX_DATA_DIR}/firewall"
-mkdir -p "${BOX_DATA_DIR}/profileicons"
-mkdir -p "${BOX_DATA_DIR}/certs"
-mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
-mkdir -p "${BOX_DATA_DIR}/mail/dkim"
-mkdir -p "${BOX_DATA_DIR}/well-known" # .well-known documents
-mkdir -p "${BOX_DATA_DIR}/sftp/ssh" # sftp keys
+mkdir -p "${PLATFORM_DATA_DIR}/sftp/ssh" # sftp keys
+mkdir -p "${PLATFORM_DATA_DIR}/firewall"
 
 # ensure backups folder exists and is writeable
 mkdir -p /var/backups
 chmod 777 /var/backups
 
-echo "==> Configuring journald"
+log "Configuring journald"
 sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
     -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
     -i /etc/systemd/journald.conf
@@ -81,18 +81,19 @@ sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
 sed -e "s/^WatchdogSec=.*$/WatchdogSec=3min/" \
     -i /lib/systemd/system/systemd-journald.service
 
-# Give user access to system logs
-usermod -a -G systemd-journal ${USER}
-mkdir -p /var/log/journal  # in some images, this directory is not created making system log to /run/systemd instead
-chown root:systemd-journal /var/log/journal
+usermod -a -G systemd-journal ${USER} # Give user access to system logs
+if [[ ! -d /var/log/journal ]]; then # in some images, this directory is not created making system log to /run/systemd instead
+    mkdir -p /var/log/journal
+    chown root:systemd-journal /var/log/journal
+    chmod g+s /var/log/journal  # sticky bit for group propagation
+fi
 systemctl daemon-reload
 systemctl restart systemd-journald
-setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
 
 # Give user access to nginx logs (uses adm group)
 usermod -a -G adm ${USER}
 
-echo "==> Setting up unbound"
+log "Setting up unbound"
 # DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org)
 # We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
 # We listen on 0.0.0.0 because there is no way control ordering of docker (which creates the 172.18.0.0/16) and unbound
@@ -102,9 +103,10 @@ cp -f "${script_dir}/start/unbound.conf" /etc/unbound/unbound.conf.d/cloudron-ne
 # update the root anchor after a out-of-disk-space situation (see #269)
 unbound-anchor -a /var/lib/unbound/root.key
 
-echo "==> Adding systemd services"
+log "Adding systemd services"
 cp -r "${script_dir}/start/systemd/." /etc/systemd/system/
 [[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/MemoryMax/MemoryLimit/g' -i /etc/systemd/system/box.service
+[[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/Type=notify/Type=simple/g' -i /etc/systemd/system/unbound.service
 systemctl daemon-reload
 systemctl enable --now cloudron-syslog
 systemctl enable unbound
@@ -124,11 +126,11 @@ systemctl restart unbound
 # ensure cloudron-syslog runs
 systemctl restart cloudron-syslog
 
-echo "==> Configuring sudoers"
+log "Configuring sudoers"
 rm -f /etc/sudoers.d/${USER}
 cp "${script_dir}/start/sudoers" /etc/sudoers.d/${USER}
 
-echo "==> Configuring collectd"
+log "Configuring collectd"
 rm -rf /etc/collectd /var/log/collectd.log
 ln -sfF "${PLATFORM_DATA_DIR}/collectd" /etc/collectd
 cp "${script_dir}/start/collectd/collectd.conf" "${PLATFORM_DATA_DIR}/collectd/collectd.conf"
@@ -140,7 +142,7 @@ if [[ "${ubuntu_version}" == "20.04" ]]; then
 fi
 systemctl restart collectd
 
-echo "==> Configuring logrotate"
+log "Configuring logrotate"
 if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then
     echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf
 fi
@@ -150,10 +152,10 @@ cp "${script_dir}/start/logrotate/"* "${PLATFORM_DATA_DIR}/logrotate.d/"
 # logrotate files have to be owned by root, this is here to fixup existing installations where we were resetting the owner to yellowtent
 chown root:root "${PLATFORM_DATA_DIR}/logrotate.d/"
 
-echo "==> Adding motd message for admins"
+log "Adding motd message for admins"
 cp "${script_dir}/start/cloudron-motd" /etc/update-motd.d/92-cloudron
 
-echo "==> Configuring nginx"
+log "Configuring nginx"
 # link nginx config to system config
 unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
 ln -s "${PLATFORM_DATA_DIR}/nginx" /etc/nginx
@@ -181,18 +183,26 @@ if [[ ! -f /etc/mysql/mysql.cnf ]] || ! diff -q "${script_dir}/start/mysql.cnf"
     cp "${script_dir}/start/mysql.cnf" /etc/mysql/mysql.cnf
     while true; do
         if ! systemctl list-jobs | grep mysql; then break; fi
-        echo "Waiting for mysql jobs..."
+        log "Waiting for mysql jobs..."
         sleep 1
     done
-    while true; do
-        if systemctl restart mysql; then break; fi
-        echo "Restarting MySql again after sometime since this fails randomly"
+    log "Stopping mysql"
+    systemctl stop mysql
+    while mysqladmin ping 2>/dev/null; do
+        log "Waiting for mysql to stop..."
         sleep 1
     done
-else
-    systemctl start mysql
 fi
 
+# the start/stop of mysql is separate to make sure it got reloaded with latest config and it's up and running before we start the new box code
+# when using 'system restart mysql', it seems to restart much later and the box code loses connection during platform startup (dangerous!)
+log "Starting mysql"
+systemctl start mysql
+while ! mysqladmin ping 2>/dev/null; do
+    log "Waiting for mysql to start..."
+    sleep 1
+done
+
 readonly mysql_root_password="password"
 mysqladmin -u root -ppassword password password # reset default root password
 if [[ "${ubuntu_version}" == "20.04" ]]; then
@@ -203,50 +213,31 @@ mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'
 
 # set HOME explicity, because it's not set when the installer calls it. this is done because
 # paths.js uses this env var and some of the migrate code requires box code
-echo "==> Migrating data"
+log "Migrating data"
 cd "${BOX_SRC_DIR}"
 if ! HOME=${HOME_DIR} BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@127.0.0.1/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up; then
-    echo "DB migration failed"
+    log "DB migration failed"
     exit 1
 fi
 
 rm -f /etc/cloudron/cloudron.conf
 
-if [[ ! -f "${BOX_DATA_DIR}/dhparams.pem" ]]; then
-    echo "==> Generating dhparams (takes forever)"
-    openssl dhparam -out "${BOX_DATA_DIR}/dhparams.pem" 2048
-    cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem"
-else
-    cp "${BOX_DATA_DIR}/dhparams.pem" "${PLATFORM_DATA_DIR}/addons/mail/dhparams.pem"
-fi
-
-if [[ ! -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" ]]; then
-    # the key format in Ubuntu 20 changed, so we create keys in legacy format. for older ubuntu, just re-use the host keys
-    # see https://github.com/proftpd/proftpd/issues/793
-    if [[ "${ubuntu_version}" == "20.04" ]]; then
-        ssh-keygen -m PEM -t rsa -f "${BOX_DATA_DIR}/sftp/ssh/ssh_host_rsa_key" -q -N ""
-    else
-        cp /etc/ssh/ssh_host_rsa_key* ${BOX_DATA_DIR}/sftp/ssh
-    fi
-fi
-
-echo "==> Changing ownership"
+log "Changing ownership"
+# note, change ownership after db migrate. this allow db migrate to move files around as root and then we can fix it up here
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
 chown -R "${USER}" /etc/cloudron
-chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update"
+chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup" "${PLATFORM_DATA_DIR}/logs" "${PLATFORM_DATA_DIR}/update" "${PLATFORM_DATA_DIR}/sftp" "${PLATFORM_DATA_DIR}/firewall"
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 chown "${USER}:${USER}" "${APPS_DATA_DIR}"
 
-# do not chown the boxdata/mail directory; dovecot gets upset
-chown "${USER}:${USER}" "${BOX_DATA_DIR}"
-find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${BOX_DATA_DIR}/mail" -exec chown -R "${USER}:${USER}" {} \;
-chown "${USER}:${USER}" "${BOX_DATA_DIR}/mail"
-chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys
+chown "${USER}:${USER}" -R "${BOX_DATA_DIR}"
+# do not chown the boxdata/mail directory entirely; dovecot gets upset
+chown "${USER}:${USER}" "${MAIL_DATA_DIR}"
 
-echo "==> Starting Cloudron"
+log "Starting Cloudron"
 systemctl start box
 
 sleep 2 # give systemd sometime to start the processes
 
-echo "==> Almost done"
+log "Almost done"

setup/start/cloudron-firewall.sh

@@ -18,16 +18,22 @@ fi
 
 # allow related and establisted connections
 iptables -t filter -A CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
-iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,25,80,202,443 -j ACCEPT # 202 is the alternate ssh port
+iptables -t filter -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,80,202,443 -j ACCEPT # 202 is the alternate ssh port
 
-# whitelist any user ports
-ports_json="/home/yellowtent/boxdata/firewall/ports.json"
+# whitelist any user ports. we used to use --dports but it has a 15 port limit (XT_MULTI_PORTS)
+ports_json="/home/yellowtent/platformdata/firewall/ports.json"
 if allowed_tcp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_tcp_ports.join(','))" 2>/dev/null); then
-    [[ -n "${allowed_tcp_ports}" ]] && iptables -A CLOUDRON -p tcp -m tcp -m multiport --dports "${allowed_tcp_ports}" -j ACCEPT
+    IFS=',' arr=(${allowed_tcp_ports})
+    for p in "${arr[@]}"; do
+        iptables -A CLOUDRON -p tcp -m tcp --dport "${p}" -j ACCEPT
+    done
 fi
 
 if allowed_udp_ports=$(node -e "console.log(JSON.parse(fs.readFileSync('${ports_json}', 'utf8')).allowed_udp_ports.join(','))" 2>/dev/null); then
-    [[ -n "${allowed_tcp_ports}" ]] && iptables -A CLOUDRON -p udp -m udp -m multiport --dports "${allowed_tcp_ports}" -j ACCEPT
+    IFS=',' arr=(${allowed_udp_ports})
+    for p in "${arr[@]}"; do
+        iptables -A CLOUDRON -p udp -m udp --dport "${p}" -j ACCEPT
+    done
 fi
 
 # turn and stun service
@@ -64,7 +70,7 @@ for port in 80 443; do
     iptables -A CLOUDRON_RATELIMIT -p tcp --syn --dport ${port} -m connlimit --connlimit-above 5000 -j CLOUDRON_RATELIMIT_LOG
 done
 
-# ssh smtp ssh msa imap sieve
+# ssh
 for port in 22 202; do
     iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --set --name "public-${port}"
     iptables -A CLOUDRON_RATELIMIT -p tcp --dport ${port} -m state --state NEW -m recent --update --name "public-${port}" --seconds 10 --hitcount 5 -j CLOUDRON_RATELIMIT_LOG
@@ -75,8 +81,8 @@ for port in 2525 4190 9993; do
     iptables -A CLOUDRON_RATELIMIT -p tcp --syn ! -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 50 -j CLOUDRON_RATELIMIT_LOG
 done
 
-# msa, ldap, imap, sieve
-for port in 2525 3002 4190 9993; do
+# msa, ldap, imap, sieve, pop3
+for port in 2525 3002 4190 9993 9995; do
     iptables -A CLOUDRON_RATELIMIT -p tcp --syn -s 172.18.0.0/16 -d 172.18.0.0/16 --dport ${port} -m connlimit --connlimit-above 500 -j CLOUDRON_RATELIMIT_LOG
 done
 
@@ -92,3 +98,5 @@ fi
 # Workaround issue where Docker insists on adding itself first in FORWARD table
 iptables -D FORWARD -j CLOUDRON_RATELIMIT || true
 iptables -I FORWARD 1 -j CLOUDRON_RATELIMIT
+
+echo "==> Setting up firewall done"

setup/start/cloudron-motd

@@ -1,8 +1,10 @@
 #!/bin/bash
 
+[[ -f /etc/update-motd.d/91-cloudron-install-in-progress ]] && exit
+
 printf "**********************************************************************\n\n"
 
-if [[ -z "$(ls -A /home/yellowtent/boxdata/mail/dkim)" ]]; then
+if [[ -z "$(ls -A /home/yellowtent/platformdata/addons/mail/dkim)" ]]; then
     if [[ -f /tmp/.cloudron-motd-cache ]]; then
         ip=$(cat /tmp/.cloudron-motd-cache)
     elif ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then

setup/start/collectd/collectd.conf

@@ -164,7 +164,9 @@ LoadPlugin swap
 #LoadPlugin vmem
 #LoadPlugin vserver
 #LoadPlugin wireless
-LoadPlugin write_graphite
+<LoadPlugin write_graphite>
+   FlushInterval 20
+</LoadPlugin>
 #LoadPlugin write_http
 #LoadPlugin write_riemann
 

setup/start/collectd/df.py

@@ -34,4 +34,5 @@ def read():
         val.dispatch(values=[used], type_instance='used')
 
 collectd.register_init(init)
+# see Interval setting in collectd.conf for polling interval
 collectd.register_read(read)

setup/start/collectd/du.py

@@ -6,19 +6,26 @@ PATHS = [] # { name, dir, exclude }
 # there is a pattern in carbon/storage-schemas.conf which stores values every 12h for a year
 INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this
 
+# we used to pass the INTERVAL as a parameter to register_read. however, collectd write_graphite
+# takes a bit to load (tcp connection) and drops the du data. this then means that we have to wait
+# for INTERVAL secs for du data. instead, we just cache the value for INTERVAL instead
+CACHE = dict()
+CACHE_TIME = 0
+
 def du(pathinfo):
     # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
-    cmd = 'timeout 1800 du -DsB1 "{}"'.format(pathinfo['dir'])
+    dirname = pathinfo['dir']
+    cmd = 'timeout 1800 du -DsB1 "{}"'.format(dirname)
     if pathinfo['exclude'] != '':
         cmd += ' --exclude "{}"'.format(pathinfo['exclude'])
 
     collectd.info('computing size with command: %s' % cmd);
     try:
         size = subprocess.check_output(cmd, shell=True).split()[0].decode('utf-8')
-        collectd.info('\tsize of %s is %s (time: %i)' % (pathinfo['dir'], size, int(time.time())))
+        collectd.info('\tsize of %s is %s (time: %i)' % (dirname, size, int(time.time())))
         return size
     except Exception as e:
-        collectd.info('\terror getting the size of %s: %s' % (pathinfo['dir'], str(e)))
+        collectd.info('\terror getting the size of %s: %s' % (dirname, str(e)))
         return 0
 
 def parseSize(size):
@@ -64,19 +71,35 @@ def init():
     collectd.info('custom du plugin initialized with %s %s' % (PATHS, sys.version))
 
 def read():
+    global CACHE, CACHE_TIME
+
+    # read from cache if < 12 hours
+    read_cache = (time.time() - CACHE_TIME) < INTERVAL
+
+    if not read_cache:
+        CACHE_TIME = time.time()
+
     for pathinfo in PATHS:
-        size = du(pathinfo)
+        dirname = pathinfo['dir']
+        if read_cache and dirname in CACHE:
+            size = CACHE[dirname]
+        else:
+            size = du(pathinfo)
+            CACHE[dirname] = size
 
         # type comes from https://github.com/collectd/collectd/blob/master/src/types.db
         val = collectd.Values(type='capacity', plugin='du', plugin_instance=pathinfo['name'])
         val.dispatch(values=[size], type_instance='usage')
 
-    size = dockerSize()
+    if read_cache and 'docker' in CACHE:
+        size = CACHE['docker']
+    else:
+        size = dockerSize()
+        CACHE['docker'] = size
+
     val = collectd.Values(type='capacity', plugin='du', plugin_instance='docker')
     val.dispatch(values=[size], type_instance='usage')
 
-
-
 collectd.register_init(init)
 collectd.register_config(configure)
-collectd.register_read(read, INTERVAL)
+collectd.register_read(read)

setup/start/mysql.cnf

@@ -4,7 +4,7 @@
 # http://bugs.mysql.com/bug.php?id=68514
 [mysqld]
 performance_schema=OFF
-max_connections=50
+max_connections=200
 # on ec2, without this we get a sporadic connection drop when doing the initial migration
 max_allowed_packet=64M
 
@@ -15,6 +15,15 @@ collation-server = utf8mb4_unicode_ci
 # set timezone to UTC
 default_time_zone='+00:00'
 
+# disable bin logs. they are only useful in replication mode
+skip-log-bin
+
+# this is used when creating an index using ALTER command
+innodb_sort_buffer_size=2097152
+
+# this is a per session sort (ORDER BY) variable for non-indexed fields
+sort_buffer_size = 4M
+
 [mysqldump]
 quick
 quote-names

setup/start/sudoers

@@ -13,9 +13,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/mkdirvolume.sh
 Defaults!/home/yellowtent/box/src/scripts/rmaddondir.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmaddondir.sh
 
-Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadnginx.sh
-
 Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 
@@ -25,9 +22,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/configurecollec
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 
-Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh
-
 Defaults!/home/yellowtent/box/src/scripts/update.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/update.sh
 
@@ -44,11 +38,8 @@ yellowtent ALL=(root) NOPASSWD:SETENV: /home/yellowtent/box/src/scripts/backupup
 Defaults!/home/yellowtent/box/src/scripts/restart.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restart.sh
 
-Defaults!/home/yellowtent/box/src/scripts/restartdocker.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartdocker.sh
-
-Defaults!/home/yellowtent/box/src/scripts/restartunbound.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartunbound.sh
+Defaults!/home/yellowtent/box/src/scripts/restartservice.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartservice.sh
 
 Defaults!/home/yellowtent/box/src/scripts/rmmailbox.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmailbox.sh
@@ -62,3 +53,11 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/stoptask.sh
 Defaults!/home/yellowtent/box/src/scripts/setblocklist.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setblocklist.sh
 
+Defaults!/home/yellowtent/box/src/scripts/addmount.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/addmount.sh
+
+Defaults!/home/yellowtent/box/src/scripts/rmmount.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmount.sh
+
+Defaults!/home/yellowtent/box/src/scripts/remountmount.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/remountmount.sh

setup/start/systemd/box.service

@@ -1,8 +1,6 @@
 [Unit]
 Description=Cloudron Admin
 OnFailure=crashnotifier@%n.service
-; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
-BindsTo=systemd-journald.service
 After=mysql.service nginx.service
 ; As cloudron-resize-fs is a one-shot, the Wants= automatically ensures that the service *finishes*
 Wants=cloudron-resize-fs.service
@@ -15,7 +13,8 @@ Type=idle
 WorkingDirectory=/home/yellowtent/box
 Restart=always
 ExecStart=/home/yellowtent/box/box.js
-Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production"
+; we run commands like df which will parse properly only with correct locale
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production" "LC_ALL=C"
 ; kill apptask processes as well
 KillMode=control-group
 ; Do not kill this process on OOM. Children inherit this score. Do not set it to -1000 so that MemoryMax can keep working

setup/start/systemd/cloudron-firewall.service

@@ -5,6 +5,7 @@ PartOf=docker.service
 
 [Service]
 Type=oneshot
+Environment="BOX_ENV=cloudron"
 ExecStart="/home/yellowtent/box/setup/start/cloudron-firewall.sh"
 RemainAfterExit=yes
 

setup/start/systemd/unbound.service

@@ -2,13 +2,17 @@
 
 [Unit]
 Description=Unbound DNS Resolver
-After=network.target docker.service
+After=network-online.target docker.service
+Before=nss-lookup.target
+Wants=network-online.target nss-lookup.target
 
 [Service]
 PIDFile=/run/unbound.pid
 ExecStart=/usr/sbin/unbound -d
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
+# On ubuntu 16, this doesn't work for some reason
+Type=notify
 
 [Install]
 WantedBy=multi-user.target

src/accesscontrol.js

@@ -1,29 +1,26 @@
 'use strict';
 
 exports = module.exports = {
-    verifyToken: verifyToken
+    verifyToken
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('./boxerror.js'),
-    tokendb = require('./tokendb.js'),
+    safe = require('safetydance'),
+    tokens = require('./tokens.js'),
     users = require('./users.js');
 
-function verifyToken(accessToken, callback) {
+async function verifyToken(accessToken) {
     assert.strictEqual(typeof accessToken, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    tokendb.getByAccessToken(accessToken, function (error, token) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-        if (error) return callback(error);
+    const token = await tokens.getByAccessToken(accessToken);
+    if (!token) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'No such token');
 
-        users.get(token.identifier, function (error, user) {
-            if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (error) return callback(error);
+    const user = await users.get(token.identifier);
+    if (!user) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not found');
+    if (!user.active) throw new BoxError(BoxError.INVALID_CREDENTIALS, 'User not active');
 
-            if (!user.active) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+    await safe(tokens.update(token.id, { lastUsedTime: new Date() })); // ignore any error
 
-            callback(null, user);
-        });
-    });
+    return user;
 }

src/acme2.js

@@ -0,0 +1,533 @@
+'use strict';
+
+exports = module.exports = {
+    getCertificate,
+
+    // testing
+    _name: 'acme',
+    _getChallengeSubdomain: getChallengeSubdomain
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    crypto = require('crypto'),
+    debug = require('debug')('box:cert/acme2'),
+    dns = require('./dns.js'),
+    fs = require('fs'),
+    os = require('os'),
+    path = require('path'),
+    promiseRetry = require('./promise-retry.js'),
+    superagent = require('superagent'),
+    safe = require('safetydance'),
+    _ = require('underscore');
+
+const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
+    CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory';
+
+// http://jose.readthedocs.org/en/latest/
+// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
+// https://community.letsencrypt.org/t/list-of-client-implementations/2103
+
+function Acme2(options) {
+    assert.strictEqual(typeof options, 'object');
+
+    this.accountKeyPem = options.accountKeyPem; // Buffer
+    this.email = options.email;
+    this.keyId = null;
+    this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
+    this.directory = {};
+    this.performHttpAuthorization = !!options.performHttpAuthorization;
+    this.wildcard = !!options.wildcard;
+}
+
+// urlsafe base64 encoding (jose)
+function urlBase64Encode(string) {
+    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
+}
+
+function b64(str) {
+    var buf = Buffer.isBuffer(str) ? str : Buffer.from(str);
+    return urlBase64Encode(buf.toString('base64'));
+}
+
+function getModulus(pem) {
+    assert(Buffer.isBuffer(pem));
+
+    var stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
+    if (!stdout) return null;
+    var match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
+    if (!match) return null;
+    return Buffer.from(match[1], 'hex');
+}
+
+Acme2.prototype.sendSignedRequest = async function (url, payload) {
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof payload, 'string');
+
+    assert(Buffer.isBuffer(this.accountKeyPem));
+
+    const that = this;
+    let header = {
+        url: url,
+        alg: 'RS256'
+    };
+
+    // keyId is null when registering account
+    if (this.keyId) {
+        header.kid = this.keyId;
+    } else {
+        header.jwk = {
+            e: b64(Buffer.from([0x01, 0x00, 0x01])), // exponent - 65537
+            kty: 'RSA',
+            n: b64(getModulus(this.accountKeyPem))
+        };
+    }
+
+    const payload64 = b64(payload);
+
+    let [error, response] = await safe(superagent.get(this.directory.newNonce).timeout(30000).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`);
+    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code when fetching nonce : ${response.status}`);
+
+    const nonce = response.headers['Replay-Nonce'.toLowerCase()];
+    if (!nonce) throw new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response');
+
+    debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
+
+    const protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
+
+    const signer = crypto.createSign('RSA-SHA256');
+    signer.update(protected64 + '.' + payload64, 'utf8');
+    const signature64 = urlBase64Encode(signer.sign(that.accountKeyPem, 'base64'));
+
+    const data = {
+        protected: protected64,
+        payload: payload64,
+        signature: signature64
+    };
+
+    [error, response] = await safe(superagent.post(url).send(data).set('Content-Type', 'application/jose+json').set('User-Agent', 'acme-cloudron').timeout(30000).ok(() => true));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`);
+
+    return response;
+};
+
+// https://tools.ietf.org/html/rfc8555#section-6.3
+Acme2.prototype.postAsGet = async function (url) {
+    return await this.sendSignedRequest(url, '');
+};
+
+Acme2.prototype.updateContact = async function (registrationUri) {
+    assert.strictEqual(typeof registrationUri, 'string');
+
+    debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
+
+    // https://github.com/ietf-wg-acme/acme/issues/30
+    const payload = {
+        contact: [ 'mailto:' + this.email ]
+    };
+
+    const result = await this.sendSignedRequest(registrationUri, JSON.stringify(payload));
+    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
+
+    debug(`updateContact: contact of user updated to ${this.email}`);
+};
+
+Acme2.prototype.registerUser = async function () {
+    const payload = {
+        termsOfServiceAgreed: true
+    };
+
+    debug('registerUser: registering user');
+
+    const result = await this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload));
+    // 200 if already exists. 201 for new accounts
+    if (result.status !== 200 && result.status !== 201) return new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.status} ${JSON.stringify(result.body)}`);
+
+    debug(`registerUser: user registered keyid: ${result.headers.location}`);
+
+    this.keyId = result.headers.location;
+
+    await this.updateContact(result.headers.location);
+};
+
+Acme2.prototype.newOrder = async function (domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    const payload = {
+        identifiers: [{
+            type: 'dns',
+            value: domain
+        }]
+    };
+
+    debug(`newOrder: ${domain}`);
+
+    const result = await this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload));
+    if (result.status === 403) throw new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`);
+    if (result.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+
+    debug('newOrder: created order %s %j', domain, result.body);
+
+    const order = result.body, orderUrl = result.headers.location;
+
+    if (!Array.isArray(order.authorizations)) throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid authorizations in order');
+    if (typeof order.finalize !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid finalize in order');
+    if (typeof orderUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, 'invalid order location in order header');
+
+    return { order, orderUrl };
+};
+
+Acme2.prototype.waitForOrder = async function (orderUrl) {
+    assert.strictEqual(typeof orderUrl, 'string');
+
+    debug(`waitForOrder: ${orderUrl}`);
+
+    return await promiseRetry({ times: 15, interval: 20000 }, async () => {
+        debug('waitForOrder: getting status');
+
+        const result = await this.postAsGet(orderUrl);
+        if (result.status !== 200) {
+            debug(`waitForOrder: invalid response code getting uri ${result.status}`);
+            throw new BoxError(BoxError.EXTERNAL_ERROR, `Bad response code: ${result.status}`);
+        }
+
+        debug('waitForOrder: status is "%s %j', result.body.status, result.body);
+
+        if (result.body.status === 'pending' || result.body.status === 'processing') throw new BoxError(BoxError.TRY_AGAIN, `Request is in ${result.body.status} state`);
+        else if (result.body.status === 'valid' && result.body.certificate) return result.body.certificate;
+        else throw new BoxError(BoxError.EXTERNAL_ERROR, `Unexpected status or invalid response: ${JSON.stringify(result.body)}`);
+    });
+};
+
+Acme2.prototype.getKeyAuthorization = function (token) {
+    assert(Buffer.isBuffer(this.accountKeyPem));
+
+    let jwk = {
+        e: b64(Buffer.from([0x01, 0x00, 0x01])), // Exponent - 65537
+        kty: 'RSA',
+        n: b64(getModulus(this.accountKeyPem))
+    };
+
+    let shasum = crypto.createHash('sha256');
+    shasum.update(JSON.stringify(jwk));
+    let thumbprint = urlBase64Encode(shasum.digest('base64'));
+    return token + '.' + thumbprint;
+};
+
+Acme2.prototype.notifyChallengeReady = async function (challenge) {
+    assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token }
+
+    debug('notifyChallengeReady: %s was met', challenge.url);
+
+    const keyAuthorization = this.getKeyAuthorization(challenge.token);
+
+    const payload = {
+        resource: 'challenge',
+        keyAuthorization: keyAuthorization
+    };
+
+    const result = await this.sendSignedRequest(challenge.url, JSON.stringify(payload));
+    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+};
+
+Acme2.prototype.waitForChallenge = async function (challenge) {
+    assert.strictEqual(typeof challenge, 'object');
+
+    debug('waitingForChallenge: %j', challenge);
+
+    await promiseRetry({ times: 15, interval: 20000 }, async () => {
+        debug('waitingForChallenge: getting status');
+
+        const result = await this.postAsGet(challenge.url);
+        if (result.status !== 200) {
+            debug(`waitForChallenge: invalid response code getting uri ${result.status}`);
+            throw new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode);
+        }
+
+        debug(`waitForChallenge: status is "${result.body.status}" "${JSON.stringify(result.body)}"`);
+
+        if (result.body.status === 'pending') throw new BoxError(BoxError.TRY_AGAIN);
+        else if (result.body.status === 'valid') return;
+        else throw new BoxError(BoxError.EXTERNAL_ERROR, `Unexpected status: ${result.body.status}`);
+    });
+};
+
+// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
+Acme2.prototype.signCertificate = async function (domain, finalizationUrl, csrDer) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof finalizationUrl, 'string');
+    assert(Buffer.isBuffer(csrDer));
+
+    const payload = {
+        csr: b64(csrDer)
+    };
+
+    debug('signCertificate: sending sign request');
+
+    const result = await this.sendSignedRequest(finalizationUrl, JSON.stringify(payload));
+    // 429 means we reached the cert limit for this domain
+    if (result.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.status} ${JSON.stringify(result.body)}`);
+};
+
+Acme2.prototype.createKeyAndCsr = async function (hostname, keyFilePath, csrFilePath) {
+    assert.strictEqual(typeof hostname, 'string');
+
+    if (safe.fs.existsSync(keyFilePath)) {
+        debug('createKeyAndCsr: reuse the key for renewal at %s', keyFilePath);
+    } else {
+        let key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
+        if (!key) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+        if (!safe.fs.writeFileSync(keyFilePath, key)) throw new BoxError(BoxError.FS_ERROR, safe.error);
+
+        debug('createKeyAndCsr: key file saved at %s', keyFilePath);
+    }
+
+    const [error, tmpdir] = await safe(fs.promises.mkdtemp(path.join(os.tmpdir(), 'acme-')));
+    if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating temporary directory for openssl config: ${error.message}`);
+
+    // OCSP must-staple is currently disabled because nginx does not provide staple on the first request (https://forum.cloudron.io/topic/4917/ocsp-stapling-for-tls-ssl/)
+    // ' -addext "tlsfeature = status_request"'; // this adds OCSP must-staple
+    // we used to use -addext to the CLI to add these but that arg doesn't work on Ubuntu 16.04
+    // empty distinguished_name section is required for Ubuntu 16 openssl
+    const conf = '[req]\nreq_extensions = v3_req\ndistinguished_name = req_distinguished_name\n'
+        + '[req_distinguished_name]\n\n'
+        + '[v3_req]\nbasicConstraints = CA:FALSE\nkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\nsubjectAltName = @alt_names\n'
+        + `[alt_names]\nDNS.1 = ${hostname}\n`;
+
+    const opensslConfigFile = path.join(tmpdir, 'openssl.conf');
+    if (!safe.fs.writeFileSync(opensslConfigFile, conf)) throw new BoxError(BoxError.FS_ERROR, `Failed to write openssl config: ${safe.error.message}`);
+
+    // while we pass the CN anyways, subjectAltName takes precedence
+    const csrDer = safe.child_process.execSync(`openssl req -new -key ${keyFilePath} -outform DER -subj /CN=${hostname} -config ${opensslConfigFile}`);
+    if (!csrDer) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+    if (!safe.fs.writeFileSync(csrFilePath, csrDer)) throw new BoxError(BoxError.FS_ERROR, safe.error); // bookkeeping. inspect with openssl req -text -noout -in hostname.csr -inform der
+
+    await safe(fs.promises.rmdir(tmpdir, { recursive: true }));
+
+    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFilePath);
+
+    return csrDer;
+};
+
+Acme2.prototype.downloadCertificate = async function (hostname, certUrl, certFilePath) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof certUrl, 'string');
+
+    await promiseRetry({ times: 5, interval: 20000 }, async () => {
+        debug('downloadCertificate: downloading certificate');
+
+        const result = await this.postAsGet(certUrl);
+        if (result.statusCode === 202) throw new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate');
+        if (result.statusCode !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`);
+
+        const fullChainPem = result.body; // buffer
+
+        if (!safe.fs.writeFileSync(certFilePath, fullChainPem)) throw new BoxError(BoxError.FS_ERROR, safe.error);
+
+        debug(`downloadCertificate: cert file for ${hostname} saved at ${certFilePath}`);
+    });
+};
+
+Acme2.prototype.prepareHttpChallenge = async function (hostname, domain, authorization, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof authorization, 'object');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
+
+    debug('prepareHttpChallenge: challenges: %j', authorization);
+    let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
+    if (httpChallenges.length === 0) throw new BoxError(BoxError.EXTERNAL_ERROR, 'no http challenges');
+    let challenge = httpChallenges[0];
+
+    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
+
+    let keyAuthorization = this.getKeyAuthorization(challenge.token);
+
+    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(acmeChallengesDir, challenge.token));
+
+    if (!safe.fs.writeFileSync(path.join(acmeChallengesDir, challenge.token), keyAuthorization)) throw new BoxError(BoxError.FS_ERROR, `Error writing challenge: ${safe.error.message}`);
+
+    return challenge;
+};
+
+Acme2.prototype.cleanupHttpChallenge = async function (hostname, domain, challenge, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
+
+    debug('cleanupHttpChallenge: unlinking %s', path.join(acmeChallengesDir, challenge.token));
+
+    if (!safe.fs.unlinkSync(path.join(acmeChallengesDir, challenge.token))) throw new BoxError(BoxError.FS_ERROR, `Error unlinking challenge: ${safe.error.message}`);
+};
+
+function getChallengeSubdomain(hostname, domain) {
+    let challengeSubdomain;
+
+    if (hostname === domain) {
+        challengeSubdomain = '_acme-challenge';
+    } else if (hostname.includes('*')) { // wildcard
+        let subdomain = hostname.slice(0, -domain.length - 1);
+        challengeSubdomain = subdomain ? subdomain.replace('*', '_acme-challenge') : '_acme-challenge';
+    } else {
+        challengeSubdomain = '_acme-challenge.' + hostname.slice(0, -domain.length - 1);
+    }
+
+    debug(`getChallengeSubdomain: challenge subdomain for hostname ${hostname} at domain ${domain} is ${challengeSubdomain}`);
+
+    return challengeSubdomain;
+}
+
+Acme2.prototype.prepareDnsChallenge = async function (hostname, domain, authorization) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof authorization, 'object');
+
+    debug('prepareDnsChallenge: challenges: %j', authorization);
+    const dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
+    if (dnsChallenges.length === 0) throw new BoxError(BoxError.EXTERNAL_ERROR, 'no dns challenges');
+    const challenge = dnsChallenges[0];
+
+    const keyAuthorization = this.getKeyAuthorization(challenge.token);
+    const shasum = crypto.createHash('sha256');
+    shasum.update(keyAuthorization);
+
+    const txtValue = urlBase64Encode(shasum.digest('base64'));
+    const challengeSubdomain = getChallengeSubdomain(hostname, domain);
+
+    debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
+
+    await dns.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
+
+    await dns.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 });
+
+    return challenge;
+};
+
+Acme2.prototype.cleanupDnsChallenge = async function (hostname, domain, challenge) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof challenge, 'object');
+
+    const keyAuthorization = this.getKeyAuthorization(challenge.token);
+    let shasum = crypto.createHash('sha256');
+    shasum.update(keyAuthorization);
+
+    const txtValue = urlBase64Encode(shasum.digest('base64'));
+    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
+
+    debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
+
+    await dns.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ]);
+};
+
+Acme2.prototype.prepareChallenge = async function (hostname, domain, authorizationUrl, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof authorizationUrl, 'string');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
+
+    debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
+
+    const response = await this.postAsGet(authorizationUrl);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code getting authorization : ${response.status}`);
+
+    const authorization = response.body;
+
+    if (this.performHttpAuthorization) {
+        return await this.prepareHttpChallenge(hostname, domain, authorization, acmeChallengesDir);
+    } else {
+        return await this.prepareDnsChallenge(hostname, domain, authorization);
+    }
+};
+
+Acme2.prototype.cleanupChallenge = async function (hostname, domain, challenge, acmeChallengesDir) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof acmeChallengesDir, 'string');
+
+    debug(`cleanupChallenge: http: ${this.performHttpAuthorization}`);
+
+    if (this.performHttpAuthorization) {
+        await this.cleanupHttpChallenge(hostname, domain, challenge, acmeChallengesDir);
+    } else {
+        await this.cleanupDnsChallenge(hostname, domain, challenge);
+    }
+};
+
+Acme2.prototype.acmeFlow = async function (hostname, domain, paths) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof paths, 'object');
+
+    const { certFilePath, keyFilePath, csrFilePath, acmeChallengesDir } = paths;
+
+    await this.registerUser();
+    const { order, orderUrl } = await this.newOrder(hostname);
+
+    for (let i = 0; i < order.authorizations.length; i++) {
+        const authorizationUrl = order.authorizations[i];
+        debug(`acmeFlow: authorizing ${authorizationUrl}`);
+
+        const challenge = await this.prepareChallenge(hostname, domain, authorizationUrl, acmeChallengesDir);
+        await this.notifyChallengeReady(challenge);
+        await this.waitForChallenge(challenge);
+        const csrDer = await this.createKeyAndCsr(hostname, keyFilePath, csrFilePath);
+        await this.signCertificate(hostname, order.finalize, csrDer);
+        const certUrl = await this.waitForOrder(orderUrl);
+        await this.downloadCertificate(hostname, certUrl, certFilePath);
+
+        try {
+            await this.cleanupChallenge(hostname, domain, challenge, acmeChallengesDir);
+        } catch (cleanupError) {
+            debug('acmeFlow: ignoring error when cleaning up challenge:', cleanupError);
+        }
+    }
+};
+
+Acme2.prototype.loadDirectory = async function () {
+    await promiseRetry({ times: 3, interval: 20000 }, async () => {
+        const response = await superagent.get(this.caDirectory).timeout(30000).ok(() => true);
+
+        if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response code when fetching directory : ${response.status}`);
+
+        if (typeof response.body.newNonce !== 'string' ||
+            typeof response.body.newOrder !== 'string' ||
+            typeof response.body.newAccount !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`);
+
+        this.directory = response.body;
+    });
+};
+
+Acme2.prototype.getCertificate = async function (vhost, domain, paths) {
+    assert.strictEqual(typeof vhost, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof paths, 'object');
+
+    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
+
+    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
+        vhost = dns.makeWildcard(vhost);
+        debug(`getCertificate: will get wildcard cert for ${vhost}`);
+    }
+
+    await this.loadDirectory();
+    await this.acmeFlow(vhost, domain, paths);
+};
+
+async function getCertificate(vhost, domain, paths, options) {
+    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof paths, 'object');
+    assert.strictEqual(typeof options, 'object');
+
+    let attempt = 1;
+    await promiseRetry({ times: 3, interval: 0 }, async function () {
+        debug(`getCertificate: attempt ${attempt++}`);
+
+        const acme = new Acme2(options || { });
+        return await acme.getCertificate(vhost, domain, paths);
+    });
+}

src/addonconfigs.js

@@ -0,0 +1,81 @@
+'use strict';
+
+exports = module.exports =  {
+    get,
+    set,
+    unset,
+
+    getByAppId,
+    getByName,
+    unsetByAppId,
+    getAppIdByValue,
+};
+
+const assert = require('assert'),
+    database = require('./database.js');
+
+async function set(appId, addonId, env) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert(Array.isArray(env));
+
+    await unset(appId, addonId);
+    if (env.length === 0) return;
+
+    const query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
+    const args = [ ], queryArgs = [ ];
+    for (let i = 0; i < env.length; i++) {
+        args.push(appId, addonId, env[i].name, env[i].value);
+        queryArgs.push('(?, ?, ?, ?)');
+    }
+
+    await database.query(query + queryArgs.join(','), args);
+}
+
+async function unset(appId, addonId) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+
+    await database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
+}
+
+async function unsetByAppId(appId) {
+    assert.strictEqual(typeof appId, 'string');
+
+    await database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ]);
+}
+
+async function get(appId, addonId) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+
+    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ]);
+    return results;
+}
+
+async function getByAppId(appId) {
+    assert.strictEqual(typeof appId, 'string');
+
+    const results = await database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ]);
+    return results;
+}
+
+async function getAppIdByValue(addonId, namePattern, value) {
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+    assert.strictEqual(typeof value, 'string');
+
+    const results = await database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ]);
+    if (results.length === 0) return null;
+    return results[0].appId;
+}
+
+async function getByName(appId, addonId, namePattern) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof namePattern, 'string');
+
+    const results = await database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ]);
+    if (results.length === 0) return null;
+    return results[0].value;
+}

src/appdb.js

@@ -1,573 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    get,
-    add,
-    exists,
-    del,
-    update,
-    getAll,
-    getPortBindings,
-    delPortBinding,
-
-    setAddonConfig,
-    getAddonConfig,
-    getAddonConfigByAppId,
-    getAddonConfigByName,
-    unsetAddonConfig,
-    unsetAddonConfigByAppId,
-    getAppIdByAddonConfigValue,
-    getByIpAddress,
-
-    setHealth,
-    setTask,
-    getAppStoreIds,
-
-    // subdomain table types
-    SUBDOMAIN_TYPE_PRIMARY: 'primary',
-    SUBDOMAIN_TYPE_REDIRECT: 'redirect',
-    SUBDOMAIN_TYPE_ALIAS: 'alias',
-
-    _clear: clear
-};
-
-var assert = require('assert'),
-    async = require('async'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js'),
-    safe = require('safetydance'),
-    util = require('util');
-
-const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
-    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
-    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson',
-    'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp',
-    'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
-    'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(',');
-
-const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(',');
-
-function postProcess(result) {
-    assert.strictEqual(typeof result, 'object');
-
-    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
-    result.manifest = safe.JSON.parse(result.manifestJson);
-    delete result.manifestJson;
-
-    assert(result.tagsJson === null || typeof result.tagsJson === 'string');
-    result.tags = safe.JSON.parse(result.tagsJson) || [];
-    delete result.tagsJson;
-
-    assert(result.reverseProxyConfigJson === null || typeof result.reverseProxyConfigJson === 'string');
-    result.reverseProxyConfig = safe.JSON.parse(result.reverseProxyConfigJson) || {};
-    delete result.reverseProxyConfigJson;
-
-    assert(result.hostPorts === null || typeof result.hostPorts === 'string');
-    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');
-
-    result.portBindings = { };
-    let hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
-    let environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
-    let portTypes = result.portTypes === null ? [ ] : result.portTypes.split(',');
-
-    delete result.hostPorts;
-    delete result.environmentVariables;
-    delete result.portTypes;
-
-    for (let i = 0; i < environmentVariables.length; i++) {
-        result.portBindings[environmentVariables[i]] = { hostPort: parseInt(hostPorts[i], 10), type: portTypes[i] };
-    }
-
-    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
-    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
-    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
-    delete result.accessRestrictionJson;
-
-    result.sso = !!result.sso; // make it bool
-    result.enableBackup = !!result.enableBackup; // make it bool
-    result.enableAutomaticUpdate = !!result.enableAutomaticUpdate; // make it bool
-    result.proxyAuth = !!result.proxyAuth;
-
-    assert(result.debugModeJson === null || typeof result.debugModeJson === 'string');
-    result.debugMode = safe.JSON.parse(result.debugModeJson);
-    delete result.debugModeJson;
-
-    assert(result.servicesConfigJson === null || typeof result.servicesConfigJson === 'string');
-    result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
-    delete result.servicesConfigJson;
-
-    let subdomains = JSON.parse(result.subdomains), domains = JSON.parse(result.domains), subdomainTypes = JSON.parse(result.subdomainTypes);
-    delete result.subdomains;
-    delete result.domains;
-    delete result.subdomainTypes;
-
-    result.alternateDomains = [];
-    result.aliasDomains = [];
-    for (let i = 0; i < subdomainTypes.length; i++) {
-        if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_PRIMARY) {
-            result.location = subdomains[i];
-            result.domain = domains[i];
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_REDIRECT) {
-            result.alternateDomains.push({ domain: domains[i], subdomain: subdomains[i] });
-        } else if (subdomainTypes[i] === exports.SUBDOMAIN_TYPE_ALIAS) {
-            result.aliasDomains.push({ domain: domains[i], subdomain: subdomains[i] });
-        }
-    }
-
-    let envNames = JSON.parse(result.envNames), envValues = JSON.parse(result.envValues);
-    delete result.envNames;
-    delete result.envValues;
-    result.env = {};
-    for (let i = 0; i < envNames.length; i++) { // NOTE: envNames is [ null ] when env of an app is empty
-        if (envNames[i]) result.env[envNames[i]] = envValues[i];
-    }
-
-    let volumeIds = JSON.parse(result.volumeIds);
-    delete result.volumeIds;
-    let volumeReadOnlys = JSON.parse(result.volumeReadOnlys);
-    delete result.volumeReadOnlys;
-
-    result.mounts = volumeIds[0] === null ? [] : volumeIds.map((v, idx) => { return { volumeId: v, readOnly: !!volumeReadOnlys[idx] }; }); // NOTE: volumeIds is [null] when volumes of an app is empty
-
-    result.error = safe.JSON.parse(result.errorJson);
-    delete result.errorJson;
-
-    result.taskId = result.taskId ? String(result.taskId) : null;
-}
-
-// each query simply join apps table with another table by id. we then join the full result together
-const PB_QUERY = 'SELECT id, GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes FROM apps LEFT JOIN appPortBindings ON apps.id = appPortBindings.appId GROUP BY apps.id';
-const ENV_QUERY = 'SELECT id, JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues FROM apps LEFT JOIN appEnvVars ON apps.id = appEnvVars.appId GROUP BY apps.id';
-const SUBDOMAIN_QUERY = 'SELECT id, JSON_ARRAYAGG(subdomains.subdomain) AS subdomains, JSON_ARRAYAGG(subdomains.domain) AS domains, JSON_ARRAYAGG(subdomains.type) AS subdomainTypes FROM apps LEFT JOIN subdomains ON apps.id = subdomains.appId GROUP BY apps.id';
-const MOUNTS_QUERY = 'SELECT id, JSON_ARRAYAGG(appMounts.volumeId) AS volumeIds, JSON_ARRAYAGG(appMounts.readOnly) AS volumeReadOnlys FROM apps LEFT JOIN appMounts ON apps.id = appMounts.appId GROUP BY apps.id';
-const APPS_QUERY = `SELECT ${APPS_FIELDS_PREFIXED}, hostPorts, environmentVariables, portTypes, envNames, envValues, subdomains, domains, subdomainTypes, volumeIds, volumeReadOnlys FROM apps`
-    + ` LEFT JOIN (${PB_QUERY}) AS q1 on q1.id = apps.id`
-    + ` LEFT JOIN (${ENV_QUERY}) AS q2 on q2.id = apps.id`
-    + ` LEFT JOIN (${SUBDOMAIN_QUERY}) AS q3 on q3.id = apps.id`
-    + ` LEFT JOIN (${MOUNTS_QUERY}) AS q4 on q4.id = apps.id`;
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`${APPS_QUERY} WHERE apps.id = ?`, [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
-function getByIpAddress(ip, callback) {
-    assert.strictEqual(typeof ip, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`${APPS_QUERY} WHERE apps.containerIp = ?`, [ ip ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
-function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`${APPS_QUERY} ORDER BY apps.id`, [ ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        results.forEach(postProcess);
-
-        callback(null, results);
-    });
-}
-
-function add(id, appStoreId, manifest, location, domain, portBindings, data, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appStoreId, 'string');
-    assert(manifest && typeof manifest === 'object');
-    assert.strictEqual(typeof manifest.version, 'string');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert(data && typeof data === 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    portBindings = portBindings || { };
-
-    var manifestJson = JSON.stringify(manifest);
-
-    const accessRestriction = data.accessRestriction || null;
-    const accessRestrictionJson = JSON.stringify(accessRestriction);
-    const memoryLimit = data.memoryLimit || 0;
-    const cpuShares = data.cpuShares || 512;
-    const installationState = data.installationState;
-    const runState = data.runState;
-    const sso = 'sso' in data ? data.sso : null;
-    const debugModeJson = data.debugMode ? JSON.stringify(data.debugMode) : null;
-    const env = data.env || {};
-    const label = data.label || null;
-    const tagsJson = data.tags ? JSON.stringify(data.tags) : null;
-    const mailboxName = data.mailboxName || null;
-    const mailboxDomain = data.mailboxDomain || null;
-    const reverseProxyConfigJson = data.reverseProxyConfig ? JSON.stringify(data.reverseProxyConfig) : null;
-
-    var queries = [];
-
-    queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares, '
-            + 'sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson) '
-            + ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, installationState, runState, accessRestrictionJson, memoryLimit, cpuShares,
-            sso, debugModeJson, mailboxName, mailboxDomain, label, tagsJson, reverseProxyConfigJson ]
-    });
-
-    queries.push({
-        query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-        args: [ id, domain, location, exports.SUBDOMAIN_TYPE_PRIMARY ]
-    });
-
-    Object.keys(portBindings).forEach(function (env) {
-        queries.push({
-            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, type, appId) VALUES (?, ?, ?, ?)',
-            args: [ env, portBindings[env].hostPort, portBindings[env].type, id ]
-        });
-    });
-
-    Object.keys(env).forEach(function (name) {
-        queries.push({
-            query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
-            args: [ id, name, env[name] ]
-        });
-    });
-
-    if (data.alternateDomains) {
-        data.alternateDomains.forEach(function (d) {
-            queries.push({
-                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]
-            });
-        });
-    }
-
-    if (data.aliasDomains) {
-        data.aliasDomains.forEach(function (d) {
-            queries.push({
-                query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)',
-                args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]
-            });
-        });
-    }
-
-    database.transaction(queries, function (error) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
-        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND, 'no such domain'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function exists(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        return callback(null, result.length !== 0);
-    });
-}
-
-function getPortBindings(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        var portBindings = { };
-        for (var i = 0; i < results.length; i++) {
-            portBindings[results[i].environmentVariable] = { hostPort: results[i].hostPort, type: results[i].type };
-        }
-
-        callback(null, portBindings);
-    });
-}
-
-function delPortBinding(hostPort, type, callback) {
-    assert.strictEqual(typeof hostPort, 'number');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM appPortBindings WHERE hostPort=? AND type=?', [ hostPort, type ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var queries = [
-        { query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] },
-        { query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
-    ];
-
-    database.transaction(queries, function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[5].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        database.query.bind(null, 'DELETE FROM subdomains'),
-        database.query.bind(null, 'DELETE FROM appPortBindings'),
-        database.query.bind(null, 'DELETE FROM appAddonConfigs'),
-        database.query.bind(null, 'DELETE FROM appEnvVars'),
-        database.query.bind(null, 'DELETE FROM apps')
-    ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        return callback(null);
-    });
-}
-
-function update(id, app, callback) {
-    updateWithConstraints(id, app, '', callback);
-}
-
-function updateWithConstraints(id, app, constraints, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof constraints, 'string');
-    assert.strictEqual(typeof callback, 'function');
-    assert(!('portBindings' in app) || typeof app.portBindings === 'object');
-    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
-    assert(!('alternateDomains' in app) || Array.isArray(app.alternateDomains));
-    assert(!('aliasDomains' in app) || Array.isArray(app.aliasDomains));
-    assert(!('tags' in app) || Array.isArray(app.tags));
-    assert(!('env' in app) || typeof app.env === 'object');
-
-    var queries = [ ];
-
-    if ('portBindings' in app) {
-        var portBindings = app.portBindings || { };
-        // replace entries by app id
-        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
-        Object.keys(portBindings).forEach(function (env) {
-            var values = [ portBindings[env].hostPort, portBindings[env].type, env, id ];
-            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, type, environmentVariable, appId) VALUES(?, ?, ?, ?)', args: values });
-        });
-    }
-
-    if ('env' in app) {
-        queries.push({ query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] });
-
-        Object.keys(app.env).forEach(function (name) {
-            queries.push({
-                query: 'INSERT INTO appEnvVars (appId, name, value) VALUES (?, ?, ?)',
-                args: [ id, name, app.env[name] ]
-            });
-        });
-    }
-
-    if ('location' in app && 'domain' in app) { // must be updated together as they are unique together
-        queries.push({ query: 'DELETE FROM subdomains WHERE appId = ?', args: [ id ]}); // all locations of an app must be updated together
-        queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, app.domain, app.location, exports.SUBDOMAIN_TYPE_PRIMARY ]});
-
-        if ('alternateDomains' in app) {
-            app.alternateDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_REDIRECT ]});
-            });
-        }
-
-        if ('aliasDomains' in app) {
-            app.aliasDomains.forEach(function (d) {
-                queries.push({ query: 'INSERT INTO subdomains (appId, domain, subdomain, type) VALUES (?, ?, ?, ?)', args: [ id, d.domain, d.subdomain, exports.SUBDOMAIN_TYPE_ALIAS ]});
-            });
-        }
-    }
-
-    if ('mounts' in app) {
-        queries.push({ query: 'DELETE FROM appMounts WHERE appId = ?', args: [ id ]});
-        app.mounts.forEach(function (m) {
-            queries.push({ query: 'INSERT INTO appMounts (appId, volumeId, readOnly) VALUES (?, ?, ?)', args: [ id, m.volumeId, m.readOnly ]});
-        });
-    }
-
-    var fields = [ ], values = [ ];
-    for (var p in app) {
-        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig') {
-            fields.push(`${p}Json = ?`);
-            values.push(JSON.stringify(app[p]));
-        } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'aliasDomains' && p !== 'env' && p !== 'mounts') {
-            fields.push(p + ' = ?');
-            values.push(app[p]);
-        }
-    }
-
-    if (values.length !== 0) {
-        values.push(id);
-        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
-    }
-
-    database.transaction(queries, function (error, results) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[results.length - 1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        return callback(null);
-    });
-}
-
-function setHealth(appId, health, healthTime, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof health, 'string');
-    assert(util.isDate(healthTime));
-    assert.strictEqual(typeof callback, 'function');
-
-    var values = { health, healthTime };
-
-    updateWithConstraints(appId, values, '', callback);
-}
-
-function setTask(appId, values, options, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof values, 'object');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!options.requireNullTaskId)  return updateWithConstraints(appId, values, '', callback);
-
-    if (options.requiredState === null) {
-        updateWithConstraints(appId, values, 'AND taskId IS NULL', callback);
-    } else {
-        updateWithConstraints(appId, values, `AND taskId IS NULL AND installationState = "${options.requiredState}"`, callback);
-    }
-}
-
-function getAppStoreIds(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT id, appStoreId FROM apps', function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function setAddonConfig(appId, addonId, env, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert(util.isArray(env));
-    assert.strictEqual(typeof callback, 'function');
-
-    unsetAddonConfig(appId, addonId, function (error) {
-        if (error) return callback(error);
-
-        if (env.length === 0) return callback(null);
-
-        var query = 'INSERT INTO appAddonConfigs(appId, addonId, name, value) VALUES ';
-        var args = [ ], queryArgs = [ ];
-        for (var i = 0; i < env.length; i++) {
-            args.push(appId, addonId, env[i].name, env[i].value);
-            queryArgs.push('(?, ?, ?, ?)');
-        }
-
-        database.query(query + queryArgs.join(','), args, function (error) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            return callback(null);
-        });
-    });
-}
-
-function unsetAddonConfig(appId, addonId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function unsetAddonConfigByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function getAddonConfig(appId, addonId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAddonConfigByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT name, value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results);
-    });
-}
-
-function getAppIdByAddonConfigValue(addonId, namePattern, value, callback) {
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-    assert.strictEqual(typeof value, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT appId FROM appAddonConfigs WHERE addonId = ? AND name LIKE ? AND value = ?', [ addonId, namePattern, value ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null, results[0].appId);
-    });
-}
-
-function getAddonConfigByName(appId, addonId, namePattern, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof namePattern, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ? AND name LIKE ?', [ appId, addonId, namePattern ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found'));
-
-        callback(null, results[0].value);
-    });
-}

src/apphealthmonitor.js

@@ -1,200 +1,181 @@
 'use strict';
 
-var appdb = require('./appdb.js'),
-    apps = require('./apps.js'),
+const apps = require('./apps.js'),
     assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
     BoxError = require('./boxerror.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:apphealthmonitor'),
     docker = require('./docker.js'),
     eventlog = require('./eventlog.js'),
     safe = require('safetydance'),
-    superagent = require('superagent'),
-    util = require('util');
+    superagent = require('superagent');
 
 exports = module.exports = {
     run
 };
 
-const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
-const UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes
+const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes
 
-const OOM_EVENT_LIMIT = 60 * 60 * 1000; // 60 minutes
-const gStartTime = new Date(); // time when apphealthmonitor was started
+const OOM_EVENT_LIMIT = 60 * 60 * 1000; // will only raise 1 oom event every hour
+let gStartTime = null; // time when apphealthmonitor was started
 let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago
 
-function debugApp(app) {
-    assert(typeof app === 'object');
-
-    debug(app.fqdn + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)) + ' - ' + app.id);
-}
-
-function setHealth(app, health, callback) {
+async function setHealth(app, health) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof health, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    let now = new Date(), curHealth = app.health;
+    // app starts out with null health
+    // if it became healthy, we update immediately. this is required for ui to say "running" etc
+    // if it became unhealthy/error/dead, wait for a threshold before updating db
+
+    const now = new Date(), lastHealth = app.health;
     let healthTime = gStartTime > app.healthTime ? gStartTime : app.healthTime; // on box restart, clamp value to start time
 
     if (health === apps.HEALTH_HEALTHY) {
         healthTime = now;
-        if (curHealth && curHealth !== apps.HEALTH_HEALTHY) { // app starts out with null health
-            debugApp(app, 'app switched from %s to healthy', curHealth);
+        if (lastHealth && lastHealth !== apps.HEALTH_HEALTHY) { // app starts out with null health
+            debug(`setHealth: ${app.id} (${app.fqdn}) switched from ${lastHealth} to healthy`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, auditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, AuditSource.HEALTH_MONITOR, { app: app });
         }
     } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) {
-        if (curHealth === apps.HEALTH_HEALTHY) {
-            debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000));
+        if (lastHealth === apps.HEALTH_HEALTHY) {
+            debug(`setHealth: marking ${app.id} (${app.fqdn}) as unhealthy since not seen for more than ${UNHEALTHY_THRESHOLD/(60 * 1000)} minutes`);
 
             // do not send mails for dev apps
-            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, auditSource.HEALTH_MONITOR, { app: app });
+            if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, AuditSource.HEALTH_MONITOR, { app: app });
         }
     } else {
-        debugApp(app, 'waiting for %s seconds to update the app health', (UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000);
-        return callback(null);
+        debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`);
+        return;
     }
 
-    appdb.setHealth(app.id, health, healthTime, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(null); // app uninstalled?
-        if (error) return callback(error);
+    const [error] = await safe(apps.setHealth(app.id, health, healthTime));
+    if (error && error.reason === BoxError.NOT_FOUND) return; // app uninstalled?
+    if (error) throw error;
 
-        app.health = health;
-
-        callback(null);
-    });
+    app.health = health;
+    app.healthTime = healthTime;
 }
 
 
 // callback is called with error for fatal errors and not if health check failed
-function checkAppHealth(app, callback) {
+async function checkAppHealth(app, options) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
+    assert.strictEqual(typeof options, 'object');
 
-    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) {
-        return callback(null);
-    }
+    if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) return;
 
     const manifest = app.manifest;
 
-    docker.inspect(app.containerId, function (error, data) {
-        if (error || !data || !data.State) return setHealth(app, apps.HEALTH_ERROR, callback);
-        if (data.State.Running !== true) return setHealth(app, apps.HEALTH_DEAD, callback);
+    const [error, data] = await safe(docker.inspect(app.containerId));
+    if (error || !data || !data.State) return await setHealth(app, apps.HEALTH_ERROR);
+    if (data.State.Running !== true) return await setHealth(app, apps.HEALTH_DEAD);
 
-        // non-appstore apps may not have healthCheckPath
-        if (!manifest.healthCheckPath) return setHealth(app, apps.HEALTH_HEALTHY, callback);
+    // non-appstore apps may not have healthCheckPath
+    if (!manifest.healthCheckPath) return await setHealth(app, apps.HEALTH_HEALTHY);
 
-        const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
-        superagent
-            .get(healthCheckUrl)
-            .set('Host', app.fqdn) // required for some apache configs with rewrite rules
-            .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
-            .redirects(0)
-            .timeout(HEALTHCHECK_INTERVAL)
-            .end(function (error, res) {
-                if (error && !error.response) {
-                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
-                } else if (res.statusCode >= 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
-                    setHealth(app, apps.HEALTH_UNHEALTHY, callback);
-                } else {
-                    setHealth(app, apps.HEALTH_HEALTHY, callback);
-                }
-            });
-    });
+    const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`;
+    const [healthCheckError, response] = await safe(superagent
+        .get(healthCheckUrl)
+        .set('Host', app.fqdn) // required for some apache configs with rewrite rules
+        .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio)
+        .redirects(0)
+        .ok(() => true)
+        .timeout(options.timeout * 1000));
+
+    if (healthCheckError) {
+        await setHealth(app, apps.HEALTH_UNHEALTHY);
+    } else if (response.status > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites)
+        await setHealth(app, apps.HEALTH_UNHEALTHY);
+    } else {
+        await setHealth(app, apps.HEALTH_HEALTHY);
+    }
 }
 
-function getContainerInfo(containerId, callback) {
-    docker.inspect(containerId, function (error, result) {
-        if (error) return callback(error);
+async function getContainerInfo(containerId) {
+    const result = await docker.inspect(containerId);
 
-        const appId = safe.query(result, 'Config.Labels.appId', null);
+    const appId = safe.query(result, 'Config.Labels.appId', null);
+    if (appId) return { app: await apps.get(appId) }; // don't get by container id as this can be an exec container
 
-        if (!appId) return callback(null, null /* app */, { name: result.Name }); // addon
-
-        apps.get(appId, callback); // don't get by container id as this can be an exec container
-    });
+    if (result.Name.startsWith('/redis-')) {
+        return { app: await apps.get(result.Name.slice('/redis-'.length)), addonName: 'redis' };
+    } else {
+        return { addonName: result.Name.slice(1) }; // addon . Name has a '/' in the beginning for some reason
+    }
 }
 
 /*
     OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:2.0.0 /bin/bash
-        apt-get update && apt-get install stress
+        docker run -ti -m 100M cloudron/base:3.0.0 /bin/bash
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
-function processDockerEvents(intervalSecs, callback) {
-    assert.strictEqual(typeof intervalSecs, 'number');
-    assert.strictEqual(typeof callback, 'function');
+async function processDockerEvents(options) {
+    assert.strictEqual(typeof options, 'object');
 
-    const since = ((new Date().getTime() / 1000) - intervalSecs).toFixed(0);
+    const since = ((new Date().getTime() / 1000) - options.intervalSecs).toFixed(0);
     const until = ((new Date().getTime() / 1000) - 1).toFixed(0);
 
-    docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
-        if (error) return callback(error);
+    const stream = await docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) });
+    stream.setEncoding('utf8');
+    stream.on('data', async function (data) { // this is actually ldjson, we only process the first line for now
+        const event = safe.JSON.parse(data);
+        if (!event) return;
+        const containerId = String(event.id);
 
-        stream.setEncoding('utf8');
-        stream.on('data', function (data) {
-            const event = JSON.parse(data);
-            const containerId = String(event.id);
+        const [error, info] = await safe(getContainerInfo(containerId));
+        const program = error ? containerId : (info.addonName || info.app.fqdn);
+        const now = Date.now();
 
-            getContainerInfo(containerId, function (error, app, addon) {
-                const program = error ? containerId : (app ? app.fqdn : addon.name);
-                const now = Date.now();
-                const notifyUser = !(app && app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
+        // do not send mails for dev apps
+        const notifyUser = !(info.app && info.app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
 
-                debug('OOM %s notifyUser: %s. lastOomTime: %s (now: %s)', program, notifyUser, gLastOomMailTime, now);
+        debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
 
-                // do not send mails for dev apps
-                if (notifyUser) {
-                    // app can be null for addon containers
-                    eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event: event, containerId: containerId, addon: addon || null, app: app || null });
+        if (notifyUser) {
+            await eventlog.add(eventlog.ACTION_APP_OOM, AuditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null });
 
-                    gLastOomMailTime = now;
-                }
-            });
-        });
-
-        stream.on('error', function (error) {
-            debug('Error reading docker events', error);
-            callback();
-        });
-
-        stream.on('end', callback);
-
-        // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
-        setTimeout(stream.destroy.bind(stream), 3000); // https://github.com/apocas/dockerode/issues/179
+            gLastOomMailTime = now;
+        }
     });
+
+    stream.on('error', function (error) {
+        debug('Error reading docker events', error);
+    });
+
+    stream.on('end', function () {
+        // debug('Event stream ended');
+    });
+
+    // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time)
+    setTimeout(stream.destroy.bind(stream), options.timeout); // https://github.com/apocas/dockerode/issues/179
 }
 
-function processApp(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function processApp(options) {
+    assert.strictEqual(typeof options, 'object');
 
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(error);
+    const allApps = await apps.list();
 
-        async.each(allApps, checkAppHealth, function (error) {
-            const alive = allApps
-                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
+    const healthChecks = allApps.map((app) => checkAppHealth(app, options)); // start healthcheck in parallel
 
-            debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.` + (error ? ` ${error.reason}` : ''));
+    await Promise.allSettled(healthChecks); // wait for all promises to finish
 
-            callback(null);
-        });
-    });
+    const alive = allApps
+        .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
+
+    debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.`);
 }
 
-function run(intervalSecs, callback) {
+async function run(intervalSecs) {
     assert.strictEqual(typeof intervalSecs, 'number');
-    assert.strictEqual(typeof callback, 'function');
 
-    async.series([
-        processApp, // this is first because docker.getEvents seems to get 'stuck' sometimes
-        processDockerEvents.bind(null, intervalSecs)
-    ], function (error) {
-        if (error) debug(`run: could not check app health. ${error.message}`);
+    if (constants.TEST) return;
 
-        callback();
-    });
+    if (!gStartTime) gStartTime = new Date();
+
+    await processApp({ timeout: (intervalSecs - 3) * 1000 });
+    await processDockerEvents({ intervalSecs, timeout: 3000 });
 }

src/apppasswords.js

@@ -0,0 +1,88 @@
+'use strict';
+
+exports = module.exports = {
+    get,
+    add,
+    list,
+    del,
+
+    removePrivateFields
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    crypto = require('crypto'),
+    database = require('./database.js'),
+    hat = require('./hat.js'),
+    safe = require('safetydance'),
+    uuid = require('uuid'),
+    _ = require('underscore');
+
+const APP_PASSWORD_FIELDS = [ 'id', 'name', 'userId', 'identifier', 'hashedPassword', 'creationTime' ].join(',');
+
+function validateAppPasswordName(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'name must be atleast 1 char');
+    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'name too long');
+
+    return null;
+}
+
+function removePrivateFields(appPassword) {
+    return _.pick(appPassword, 'id', 'name', 'userId', 'identifier', 'creationTime');
+}
+
+async function get(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE id = ?', [ id ]);
+    if (result.length === 0) return null;
+    return result[0];
+}
+
+async function add(userId, identifier, name) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof name, 'string');
+
+    let error = validateAppPasswordName(name);
+    if (error) throw error;
+
+    if (identifier.length < 1) throw new BoxError(BoxError.BAD_FIELD, 'identifier must be atleast 1 char');
+
+    const password = hat(16 * 4);
+    const hashedPassword = crypto.createHash('sha256').update(password).digest('base64');
+
+    const appPassword = {
+        id: 'uid-' + uuid.v4(),
+        name,
+        userId,
+        identifier,
+        password,
+        hashedPassword
+    };
+
+    const query = 'INSERT INTO appPasswords (id, userId, identifier, name, hashedPassword) VALUES (?, ?, ?, ?, ?)';
+    const args = [ appPassword.id, appPassword.userId, appPassword.identifier, appPassword.name, appPassword.hashedPassword ];
+
+    [error] = await safe(database.query(query, args));
+    if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('appPasswords_name_userId_identifier') !== -1) throw new BoxError(BoxError.ALREADY_EXISTS, 'name/app combination already exists');
+    if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.indexOf('userId')) throw new BoxError(BoxError.NOT_FOUND, 'user not found');
+    if (error) throw error;
+
+    return { id: appPassword.id, password: appPassword.password };
+}
+
+async function list(userId) {
+    assert.strictEqual(typeof userId, 'string');
+
+    return await database.query('SELECT ' + APP_PASSWORD_FIELDS + ' FROM appPasswords WHERE userId = ?', [ userId ]);
+}
+
+async function del(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query('DELETE FROM appPasswords WHERE id = ?', [ id ]);
+    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'password not found');
+}

src/appstore.js

@@ -7,15 +7,13 @@ exports = module.exports = {
     getApp,
     getAppVersion,
 
-    trackBeginSetup,
-    trackFinishedSetup,
-
     registerWithLoginCredentials,
+    updateCloudron,
 
     purchaseApp,
     unpurchaseApp,
 
-    getUserToken,
+    createUserToken,
     getSubscription,
     isFreePlan,
 
@@ -25,9 +23,8 @@ exports = module.exports = {
     createTicket
 };
 
-var apps = require('./apps.js'),
+const apps = require('./apps.js'),
     assert = require('assert'),
-    async = require('async'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:appstore'),
@@ -76,99 +73,86 @@ function isAppAllowed(appstoreId, listingConfig) {
     return true;
 }
 
-function getCloudronToken(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    settings.getCloudronToken(function (error, token) {
-        if (error) return callback(error);
-        if (!token) return callback(new BoxError(BoxError.LICENSE_ERROR, 'Missing token'));
-
-        callback(null, token);
-    });
-}
-
-function login(email, password, totpToken, callback) {
+async function login(email, password, totpToken) {
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof totpToken, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var data = {
-        email: email,
-        password: password,
-        totpToken: totpToken
-    };
+    const data = { email, password, totpToken };
 
     const url = settings.apiServerOrigin() + '/api/v1/login';
-    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${result.statusCode}`));
+    const [error, response] = await safe(superagent.post(url)
+        .send(data)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-        callback(null, result.body); // { userId, accessToken }
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `login status code: ${response.status}`);
+
+    return response.body; // { userId, accessToken }
 }
 
-function registerUser(email, password, callback) {
+async function registerUser(email, password) {
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof password, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var data = {
-        email: email,
-        password: password,
-    };
+    const data = { email, password };
 
     const url = settings.apiServerOrigin() + '/api/v1/register_user';
-    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${result.statusCode}`));
+    const [error, response] = await safe(superagent.post(url)
+        .send(data)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-        callback(null);
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+
+    if (response.status === 409) throw new BoxError(BoxError.ALREADY_EXISTS, error.message);
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${response.status}`);
 }
 
-function getUserToken(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function createUserToken() {
+    if (settings.isDemo()) throw new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode');
 
-    if (settings.isDemo()) return callback(new BoxError(BoxError.BAD_FIELD, 'Not allowed in demo mode'));
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
 
-        const url = `${settings.apiServerOrigin()}/api/v1/user_token`;
+    const [error, response] = await safe(superagent.post(url)
+        .send({})
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-        superagent.post(url).send({}).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${result.status}`));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `getUserToken status code: ${response.status}`);
 
-            callback(null, result.body.accessToken);
-        });
-    });
+    return response.body.accessToken;
 }
 
-function getSubscription(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function getSubscription() {
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const url = settings.apiServerOrigin() + '/api/v1/subscription';
 
-        const url = settings.apiServerOrigin() + '/api/v1/subscription';
-        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR));
-            if (result.statusCode === 502) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`));
-            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`));
+    const [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            // update the features cache
-            gFeatures = result.body.features;
-            safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR);
+    if (response.status === 502) throw new BoxError(BoxError.EXTERNAL_ERROR, `Stripe error: ${error.message}`);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error: ${error.message}`);
 
-            callback(null, result.body);
-        });
-    });
+    // update the features cache
+    gFeatures = response.body.features;
+    safe.fs.writeFileSync(paths.FEATURES_INFO_FILE, JSON.stringify(gFeatures), 'utf8');
+
+    return response.body;
 }
 
 function isFreePlan(subscription) {
@@ -176,226 +160,212 @@ function isFreePlan(subscription) {
 }
 
 // See app.js install it will create a db record first but remove it again if appstore purchase fails
-function purchaseApp(data, callback) {
+async function purchaseApp(data) {
     assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId, appId }
     assert(data.appstoreId || data.manifestId);
     assert.strictEqual(typeof data.appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps`;
+    const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps`;
 
-        superagent.post(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); // appstoreId does not exist
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 402) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            // 200 if already purchased, 201 is newly purchased
-            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', result.status, result.body)));
+    const [error, response] = await safe(superagent.post(url)
+        .send(data)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            callback(null);
-        });
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND); // appstoreId does not exist
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 402) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    // 200 if already purchased, 201 is newly purchased
+    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App purchase failed. %s %j', response.status, response.body));
 }
 
-function unpurchaseApp(appId, data, callback) {
+async function unpurchaseApp(appId, data) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof data, 'object'); // { appstoreId, manifestId }
     assert(data.appstoreId || data.manifestId);
-    assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-        const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
+    const url = `${settings.apiServerOrigin()}/api/v1/cloudronapps/${appId}`;
 
-        superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 404) return callback(null);   // was never purchased
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode !== 201 && result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
+    let [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            superagent.del(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
-                if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', result.status, result.body)));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 404) return;   // was never purchased
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 201 && response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
 
-                callback(null);
-            });
-        });
-    });
+    [error, response] = await safe(superagent.del(url)
+        .send(data)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
+
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status !== 204) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App unpurchase failed. %s %j', response.status, response.body));
 }
 
-function getBoxUpdate(options, callback) {
+async function getBoxUpdate(options) {
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-        const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
+    const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
 
-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            automatic: options.automatic
-        };
+    const query = {
+        accessToken: token,
+        boxVersion: constants.VERSION,
+        automatic: options.automatic
+    };
 
-        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 204) return callback(null, null); // no update
-            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+    const [error, response] = await safe(superagent.get(url)
+        .query(query)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            var updateInfo = result.body;
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status === 204) return; // no update
+    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-            if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', result.statusCode, result.text)));
-            }
+    const updateInfo = response.body;
 
-            // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
-            if (!updateInfo.version || typeof updateInfo.version !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', result.statusCode, result.text)));
-            if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', result.statusCode, result.text)));
+    if (!semver.valid(updateInfo.version) || semver.gt(constants.VERSION, updateInfo.version)) {
+        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Update version invalid or is a downgrade: %s %s', response.status, response.text));
+    }
 
-            callback(null, updateInfo);
-        });
-    });
+    // updateInfo: { version, changelog, sourceTarballUrl, sourceTarballSigUrl, boxVersionsUrl, boxVersionsSigUrl }
+    if (!updateInfo.version || typeof updateInfo.version !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
+    if (!updateInfo.changelog || !Array.isArray(updateInfo.changelog)) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad version): %s %s', response.status, response.text));
+    if (!updateInfo.sourceTarballUrl || typeof updateInfo.sourceTarballUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballUrl): %s %s', response.status, response.text));
+    if (!updateInfo.sourceTarballSigUrl || typeof updateInfo.sourceTarballSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad sourceTarballSigUrl): %s %s', response.status, response.text));
+    if (!updateInfo.boxVersionsUrl || typeof updateInfo.boxVersionsUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsUrl): %s %s', response.status, response.text));
+    if (!updateInfo.boxVersionsSigUrl || typeof updateInfo.boxVersionsSigUrl !== 'string') throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response (bad boxVersionsSigUrl): %s %s', response.status, response.text));
+
+    return updateInfo;
 }
 
-function getAppUpdate(app, options, callback) {
+async function getAppUpdate(app, options) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-        const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            appId: app.appStoreId,
-            appVersion: app.manifest.version,
-            automatic: options.automatic
-        };
+    const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
+    const query = {
+        accessToken: token,
+        boxVersion: constants.VERSION,
+        appId: app.appStoreId,
+        appVersion: app.manifest.version,
+        automatic: options.automatic
+    };
 
-        superagent.get(url).query(query).timeout(30 * 1000).end(function (error, result) {
-            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
-            if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-            if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-            if (result.statusCode === 204) return callback(null); // no update
-            if (result.statusCode !== 200 || !result.body) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+    const [error, response] = await safe(superagent.get(url)
+        .query(query)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-            const updateInfo = result.body;
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status === 204) return; // no update
+    if (response.status !== 200 || !response.body) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-            // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
-            const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
+    const updateInfo = response.body;
 
-            // do some sanity checks
-            if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
-                debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
-                return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', result.statusCode, result.text)));
-            }
+    // for the appstore, x.y.z is the same as x.y.z-0 but in semver, x.y.z > x.y.z-0
+    const curAppVersion = semver.prerelease(app.manifest.version) ? app.manifest.version : `${app.manifest.version}-0`;
 
-            updateInfo.unstable = !!updateInfo.unstable;
+    // do some sanity checks
+    if (!safe.query(updateInfo, 'manifest.version') || semver.gt(curAppVersion, safe.query(updateInfo, 'manifest.version'))) {
+        debug('Skipping malformed update of app %s version: %s. got %j', app.id, curAppVersion, updateInfo);
+        throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', response.status, response.text));
+    }
 
-            // { id, creationDate, manifest, unstable }
-            callback(null, updateInfo);
-        });
-    });
+    updateInfo.unstable = !!updateInfo.unstable;
+
+    // { id, creationDate, manifest, unstable }
+    return updateInfo;
 }
 
-function registerCloudron(data, callback) {
+async function registerCloudron(data) {
     assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     const url = `${settings.apiServerOrigin()}/api/v1/register_cloudron`;
 
-    superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${result.statusCode} ${error.message}`));
+    const [error, response] = await safe(superagent.post(url)
+        .send(data)
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-        // cloudronId, token, licenseKey
-        if (!result.body.cloudronId) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id'));
-        if (!result.body.cloudronToken) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token'));
-        if (!result.body.licenseKey) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no license'));
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${response.statusCode} ${error.message}`);
 
-        async.series([
-            settings.setCloudronId.bind(null, result.body.cloudronId),
-            settings.setCloudronToken.bind(null, result.body.cloudronToken),
-            settings.setLicenseKey.bind(null, result.body.licenseKey),
-        ], function (error) {
-            if (error) return callback(error);
+    // cloudronId, token, licenseKey
+    if (!response.body.cloudronId) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id');
+    if (!response.body.cloudronToken) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no token');
+    if (!response.body.licenseKey) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no license');
 
-            debug(`registerCloudron: Cloudron registered with id ${result.body.cloudronId}`);
+    await settings.setCloudronId(response.body.cloudronId);
+    await settings.setCloudronToken(response.body.cloudronToken);
+    await settings.setLicenseKey(response.body.licenseKey);
 
-            callback();
-        });
-    });
+    debug(`registerCloudron: Cloudron registered with id ${response.body.cloudronId}`);
 }
 
-// This works without a Cloudron token as this Cloudron was not yet registered
-let gBeginSetupAlreadyTracked = false;
-function trackBeginSetup() {
-    // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
-    if (gBeginSetupAlreadyTracked) return;
-    gBeginSetupAlreadyTracked = true;
+async function updateCloudron(data) {
+    assert.strictEqual(typeof data, 'object');
 
-    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    superagent.post(url).send({}).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return debug(`trackBeginSetup: ${error.message}`);
-        if (result.statusCode !== 200) return debug(`trackBeginSetup: ${result.statusCode} ${error.message}`);
-    });
+    const url = `${settings.apiServerOrigin()}/api/v1/update_cloudron`;
+    const query = {
+        accessToken: token
+    };
+
+    const [error, response] = await safe(superagent.post(url)
+        .query(query)
+        .send(data)
+        .timeout(30 * 1000)
+        .ok(() => true));
+
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
+
+    debug(`updateCloudron: Cloudron updated with data ${JSON.stringify(data)}`);
 }
 
-// This works without a Cloudron token as this Cloudron was not yet registered
-function trackFinishedSetup(domain) {
-    assert.strictEqual(typeof domain, 'string');
-
-    const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;
-
-    superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return debug(`trackFinishedSetup: ${error.message}`);
-        if (result.statusCode !== 200) return debug(`trackFinishedSetup: ${result.statusCode} ${error.message}`);
-    });
-}
-
-function registerWithLoginCredentials(options, callback) {
+async function registerWithLoginCredentials(options) {
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    function maybeSignup(done) {
-        if (!options.signup) return done();
+    const token = await settings.getCloudronToken();
+    if (token) throw new BoxError(BoxError.CONFLICT, 'Cloudron is already registered');
 
-        registerUser(options.email, options.password, done);
-    }
+    if (options.signup) await registerUser(options.email, options.password);
 
-    getCloudronToken(function (error, token) {
-        if (token) return callback(new BoxError(BoxError.CONFLICT, 'Cloudron is already registered'));
-
-        maybeSignup(function (error) {
-            if (error) return callback(error);
-
-            login(options.email, options.password, options.totpToken || '', function (error, result) {
-                if (error) return callback(error);
-
-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
-            });
-        });
-    });
+    const result = await login(options.email, options.password, options.totpToken || '');
+    await registerCloudron({ domain: settings.dashboardDomain(), accessToken: result.accessToken, version: constants.VERSION });
 }
 
-function createTicket(info, auditSource, callback) {
+async function createTicket(info, auditSource) {
     assert.strictEqual(typeof info, 'object');
     assert.strictEqual(typeof info.email, 'string');
     assert.strictEqual(typeof info.displayName, 'string');
@@ -403,128 +373,99 @@ function createTicket(info, auditSource, callback) {
     assert.strictEqual(typeof info.subject, 'string');
     assert.strictEqual(typeof info.description, 'string');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    function collectAppInfoIfNeeded(callback) {
-        if (!info.appId) return callback();
-        apps.get(info.appId, callback);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
+
+    if (info.enableSshSupport) {
+        await safe(support.enableRemoteSupport(true, auditSource));
     }
 
-    function enableSshIfNeeded(callback) {
-        if (!info.enableSshSupport) return callback();
+    info.app = info.appId ? await apps.get(info.appId) : null;
+    info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
 
-        support.enableRemoteSupport(true, auditSource, function (error) {
-            // ensure we can at least get the ticket through
-            if (error) debug('Unable to enable SSH support.', error);
+    const request = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true);
 
-            callback();
+    // either send as JSON through body or as multipart, depending on attachments
+    if (info.app) {
+        request.field('infoJSON', JSON.stringify(info));
+
+        apps.getLocalLogfilePaths(info.app).forEach(function (filePath) {
+            const logs = safe.child_process.execSync(`tail --lines=1000 ${filePath}`);
+            if (logs) request.attach(path.basename(filePath), logs, path.basename(filePath));
         });
+    } else {
+        request.send(info);
     }
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const [error, response] = await safe(request);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 201) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-        enableSshIfNeeded(function (error) {
-            if (error) return callback(error);
+    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
 
-            collectAppInfoIfNeeded(function (error, app) {
-                if (error) return callback(error);
-                if (app) info.app = app;
-
-                info.supportEmail = constants.SUPPORT_EMAIL; // destination address for tickets
-
-                var req = superagent.post(`${settings.apiServerOrigin()}/api/v1/ticket`)
-                    .query({ accessToken: token })
-                    .timeout(30 * 1000);
-
-                // either send as JSON through body or as multipart, depending on attachments
-                if (info.app) {
-                    req.field('infoJSON', JSON.stringify(info));
-
-                    apps.getLocalLogfilePaths(info.app).forEach(function (filePath) {
-                        var logs = safe.child_process.execSync(`tail --lines=1000 ${filePath}`);
-                        if (logs) req.attach(path.basename(filePath), logs, path.basename(filePath));
-                    });
-                } else {
-                    req.send(info);
-                }
-
-                req.end(function (error, result) {
-                    if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                    if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                    if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                    if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
-
-                    eventlog.add(eventlog.ACTION_SUPPORT_TICKET, auditSource, info);
-
-                    callback(null, { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` });
-                });
-            });
-        });
-    });
+    return { message: `An email was sent to ${constants.SUPPORT_EMAIL}. We will get back shortly!` };
 }
 
-function getApps(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function getApps() {
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-    getCloudronToken(function (error, token) {
-        if (error) return callback(error);
+    const unstable = await settings.getUnstableAppsConfig();
 
-        settings.getUnstableAppsConfig(function (error, unstable) {
-            if (error) return callback(error);
+    const url = `${settings.apiServerOrigin()}/api/v1/apps`;
 
-            const url = `${settings.apiServerOrigin()}/api/v1/apps`;
-            superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable }).timeout(30 * 1000).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
-                if (!result.body.apps) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', result.statusCode, result.text)));
+    const [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token, boxVersion: constants.VERSION, unstable: unstable })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-                settings.getAppstoreListingConfig(function (error, listingConfig) {
-                    if (error) return callback(error);
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 403 || response.status === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', response.status, response.body));
+    if (!response.body.apps) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('Bad response: %s %s', response.status, response.text));
 
-                    const filteredApps = result.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
-
-                    callback(null, filteredApps);
-                });
-            });
-        });
-    });
+    const listingConfig = await settings.getAppstoreListingConfig();
+    const filteredApps = response.body.apps.filter(app => isAppAllowed(app.id, listingConfig));
+    return filteredApps;
 }
 
-function getAppVersion(appId, version, callback) {
+async function getAppVersion(appId, version) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof version, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    settings.getAppstoreListingConfig(function (error, listingConfig) {
-        if (error) return callback(error);
+    const listingConfig = await settings.getAppstoreListingConfig();
 
-        if (!isAppAllowed(appId, listingConfig)) return callback(new BoxError(BoxError.FEATURE_DISABLED));
+    if (!isAppAllowed(appId, listingConfig)) throw new BoxError(BoxError.FEATURE_DISABLED);
 
-        getCloudronToken(function (error, token) {
-            if (error) return callback(error);
+    const token = await settings.getCloudronToken();
+    if (!token) throw new BoxError(BoxError.LICENSE_ERROR, 'Missing token');
 
-            let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
-            if (version !== 'latest') url += `/versions/${version}`;
+    let url = `${settings.apiServerOrigin()}/api/v1/apps/${appId}`;
+    if (version !== 'latest') url += `/versions/${version}`;
 
-            superagent.get(url).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
-                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-                if (result.statusCode === 403 || result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
-                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
-                if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', result.status, result.body)));
+    const [error, response] = await safe(superagent.get(url)
+        .query({ accessToken: token })
+        .timeout(30 * 1000)
+        .ok(() => true));
 
-                callback(null, result.body);
-            });
-        });
-    });
+    if (error) throw new BoxError(BoxError.NETWORK_ERROR, error.message);
+    if (response.status === 403 || response.statusCode === 401) throw new BoxError(BoxError.INVALID_CREDENTIALS);
+    if (response.status === 404) throw new BoxError(BoxError.NOT_FOUND);
+    if (response.status === 422) throw new BoxError(BoxError.LICENSE_ERROR, response.body.message);
+    if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, util.format('App fetch failed. %s %j', response.status, response.body));
+
+    return response.body;
 }
 
-function getApp(appId, callback) {
+async function getApp(appId) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    getAppVersion(appId, 'latest', callback);
+    return await getAppVersion(appId, 'latest');
 }

src/apptaskmanager.js

@@ -4,7 +4,7 @@ exports = module.exports = {
     scheduleTask
 };
 
-let assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('box:apptaskmanager'),
     fs = require('fs'),
@@ -13,7 +13,6 @@ let assert = require('assert'),
     path = require('path'),
     paths = require('./paths.js'),
     scheduler = require('./scheduler.js'),
-    services = require('./services.js'),
     tasks = require('./tasks.js');
 
 let gActiveTasks = { }; // indexed by app id
@@ -21,12 +20,11 @@ let gPendingTasks = [ ];
 let gInitialized = false;
 
 const TASK_CONCURRENCY = 3;
-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 function waitText(lockOperation) {
     if (lockOperation === locker.OP_BOX_UPDATE) return 'Waiting for Cloudron to finish updating. See the Settings view';
     if (lockOperation === locker.OP_PLATFORM_START) return 'Waiting for Cloudron to initialize';
-    if (lockOperation === locker.OP_FULL_BACKUP) return 'Wait for Cloudron to finish backup. See the Backups view';
+    if (lockOperation === locker.OP_FULL_BACKUP) return 'Waiting for Cloudron to finish backup. See the Backups view';
 
     return ''; // cannot happen
 }
@@ -37,31 +35,31 @@ function initializeSync() {
 }
 
 // callback is called when task is finished
-function scheduleTask(appId, taskId, options, callback) {
+function scheduleTask(appId, taskId, options, onFinished) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof taskId, 'string');
     assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
+    assert.strictEqual(typeof onFinished, 'function');
 
     if (!gInitialized) initializeSync();
 
     if (appId in gActiveTasks) {
-        return callback(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
+        return onFinished(new BoxError(BoxError.CONFLICT, `Task for %s is already active: ${appId}`));
     }
 
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug(`Reached concurrency limit, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, callback });
+        tasks.update(taskId, { percent: 1, message: 'Waiting for other app tasks to complete' });
+        gPendingTasks.push({ appId, taskId, options, onFinished });
         return;
     }
 
-    var lockError = locker.recursiveLock(locker.OP_APPTASK);
+    const lockError = locker.recursiveLock(locker.OP_APPTASK);
 
     if (lockError) {
         debug(`Could not get lock. ${lockError.message}, queueing task id ${taskId}`);
-        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) }, NOOP_CALLBACK);
-        gPendingTasks.push({ appId, taskId, callback });
+        tasks.update(taskId, { percent: 1, message: waitText(lockError.operation) });
+        gPendingTasks.push({ appId, taskId, options, onFinished });
         return;
     }
 
@@ -74,13 +72,11 @@ function scheduleTask(appId, taskId, options, callback) {
     scheduler.suspendJobs(appId);
 
     tasks.startTask(taskId, Object.assign(options, { logFile }), function (error, result) {
-        callback(error, result);
+        onFinished(error, result);
 
         delete gActiveTasks[appId];
         locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
 
-        // post app task hooks
-        services.rebuildService('sftp', error => { if (error) debug('Unable to rebuild sftp:', error); });
         scheduler.resumeJobs(appId);
     });
 }
@@ -91,6 +87,5 @@ function startNextTask() {
     assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
     const t = gPendingTasks.shift();
-    scheduleTask(t.appId, t.taskId, t.callback);
+    scheduleTask(t.appId, t.taskId, t.options, t.onFinished);
 }
-

src/auditsource.js

@@ -1,16 +1,23 @@
 'use strict';
 
-exports = module.exports = {
-    CRON:  { userId: null, username: 'cron' },
-    HEALTH_MONITOR: { userId: null, username: 'healthmonitor' },
-    APP_TASK: { userId: null, username: 'apptask' },
-    EXTERNAL_LDAP_TASK: { userId: null, username: 'externalldap' },
-    EXTERNAL_LDAP_AUTO_CREATE: { userId: null, username: 'externalldap' },
+class AuditSource {
+    constructor(username, userId, ip) {
+        this.username = username;
+        this.userId = userId || null;
+        this.ip = ip || null;
+    }
 
-    fromRequest: fromRequest
-};
-
-function fromRequest(req) {
-    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
-    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+    static fromRequest(req) {
+        const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+        return new AuditSource(req.user?.username, req.user?.id, ip);
+    }
 }
+
+// these can be static variables but see https://stackoverflow.com/questions/60046847/eslint-does-not-allow-static-class-properties#comment122122927_60464446
+AuditSource.CRON =  new AuditSource('cron');
+AuditSource.HEALTH_MONITOR = new AuditSource('healthmonitor');
+AuditSource.EXTERNAL_LDAP_TASK = new AuditSource('externalldap');
+AuditSource.EXTERNAL_LDAP_AUTO_CREATE = new AuditSource('externalldap');
+AuditSource.APPTASK = new AuditSource('apptask');
+
+exports = module.exports = AuditSource;

src/autoconfig.xml.ejs

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<clientConfig version="1.1">
+  <emailProvider id="<%= domain %>">
+    <domain><%= domain %></domain>
+    <displayName>Cloudron Mail</displayName>
+    <displayShortName>Cloudron</displayShortName>
+    <incomingServer type="imap">
+      <hostname><%= mailFqdn %></hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <outgoingServer type="smtp">
+      <hostname><%= mailFqdn %></hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+      <addThisServer>true</addThisServer>
+    </outgoingServer>
+    <incomingServer type="pop3">
+      <hostname><%= mailFqdn %></hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <username>%EMAILADDRESS%</username>
+      <authentication>password-cleartext</authentication>
+    </incomingServer>
+    <documentation url="http://cloudron.io/email/#autodiscover">
+       <descr lang="en">Cloudron Email</descr>
+    </documentation>
+
+  </emailProvider>
+</clientConfig>
+

src/backupcleaner.js

@@ -0,0 +1,280 @@
+'use strict';
+
+exports = module.exports = {
+    run,
+
+    _applyBackupRetentionPolicy: applyBackupRetentionPolicy
+};
+
+const apps = require('./apps.js'),
+    assert = require('assert'),
+    backups = require('./backups.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:backupcleaner'),
+    moment = require('moment'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    settings = require('./settings.js'),
+    storage = require('./storage.js'),
+    util = require('util'),
+    _ = require('underscore');
+
+function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
+    assert(Array.isArray(allBackups));
+    assert.strictEqual(typeof policy, 'object');
+    assert(Array.isArray(referencedBackupIds));
+
+    const now = new Date();
+
+    for (const backup of allBackups) {
+        if (backup.state === backups.BACKUP_STATE_ERROR) {
+            backup.discardReason = 'error';
+        } else if (backup.state === backups.BACKUP_STATE_CREATING) {
+            if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
+            else backup.discardReason = 'creating-too-long';
+        } else if (referencedBackupIds.includes(backup.id)) {
+            backup.keepReason = 'reference';
+        } else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) {
+            backup.keepReason = 'preserveSecs';
+        } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) {
+            backup.keepReason = 'keepWithinSecs';
+        }
+    }
+
+    const KEEP_FORMATS = {
+        keepDaily: 'Y-M-D',
+        keepWeekly: 'Y-W',
+        keepMonthly: 'Y-M',
+        keepYearly: 'Y'
+    };
+
+    for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) {
+        if (!(format in policy)) continue;
+
+        const n = policy[format]; // we want to keep "n" backups of format
+        if (!n) continue; // disabled rule
+
+        let lastPeriod = null, keptSoFar = 0;
+        for (const backup of allBackups) {
+            if (backup.discardReason) continue; // already discarded for some reason
+            if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason
+            const period = moment(backup.creationTime).format(KEEP_FORMATS[format]);
+            if (period === lastPeriod) continue; // already kept for this period
+
+            lastPeriod = period;
+            backup.keepReason = backup.keepReason ? `${backup.keepReason}+${format}` : format;
+            if (++keptSoFar === n) break;
+        }
+    }
+
+    if (policy.keepLatest) {
+        let latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL);
+        if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest';
+    }
+
+    for (const backup of allBackups) {
+        debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
+    }
+}
+
+async function cleanupBackup(backupConfig, backup, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof backup, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
+
+    return new Promise((resolve) => {
+        function done(error) {
+            if (error) {
+                debug('cleanupBackup: error removing backup %j : %s', backup, error.message);
+                return resolve();
+            }
+
+            // prune empty directory if possible
+            storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), async function (error) {
+                if (error) debug('cleanupBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message);
+
+                const [delError] = await safe(backups.del(backup.id));
+                if (delError) debug('cleanupBackup: error removing from database', delError);
+                else debug('cleanupBackup: removed %s', backup.id);
+
+                resolve();
+            });
+        }
+
+        if (backup.format ==='tgz') {
+            progressCallback({ message: `${backup.id}: Removing ${backupFilePath}`});
+            storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, done);
+        } else {
+            const events = storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath);
+            events.on('progress', (message) => progressCallback({ message: `${backup.id}: ${message}` }));
+            events.on('done', done);
+        }
+    });
+}
+
+async function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert(Array.isArray(referencedAppBackupIds));
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    let removedAppBackupIds = [];
+
+    const allApps = await apps.list();
+    const allAppIds = allApps.map(a => a.id);
+
+    const appBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000);
+
+    // collate the backups by app id. note that the app could already have been uninstalled
+    let appBackupsById = {};
+    for (const appBackup of appBackups) {
+        if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = [];
+        appBackupsById[appBackup.identifier].push(appBackup);
+    }
+
+    // apply backup policy per app. keep latest backup only for existing apps
+    let appBackupsToRemove = [];
+    for (const appId of Object.keys(appBackupsById)) {
+        applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds);
+        appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
+    }
+
+    for (const appBackup of appBackupsToRemove) {
+        await progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
+        removedAppBackupIds.push(appBackup.id);
+        await cleanupBackup(backupConfig, appBackup, progressCallback); // never errors
+    }
+
+    debug('cleanupAppBackups: done');
+
+    return removedAppBackupIds;
+}
+
+async function cleanupBoxBackups(backupConfig, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    let referencedAppBackupIds = [], removedBoxBackupIds = [];
+
+    const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
+
+    applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */);
+
+    for (const boxBackup of boxBackups) {
+        if (boxBackup.keepReason) {
+            referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn);
+            continue;
+        }
+
+        await progressCallback({ message: `Removing box backup ${boxBackup.id}`});
+
+        removedBoxBackupIds.push(boxBackup.id);
+        await cleanupBackup(backupConfig, boxBackup, progressCallback);
+    }
+
+    debug('cleanupBoxBackups: done');
+
+    return { removedBoxBackupIds, referencedAppBackupIds };
+}
+
+async function cleanupMissingBackups(backupConfig, progressCallback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const perPage = 1000;
+    let missingBackupIds = [];
+    const backupExists = util.promisify(storage.api(backupConfig.provider).exists);
+
+    if (constants.TEST) return missingBackupIds;
+
+    let page = 1, result = [];
+    do {
+        result = await backups.list(page, perPage);
+
+        for (const backup of result) {
+            let backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
+            if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
+
+            const [existsError, exists] = await safe(backupExists(backupConfig, backupFilePath));
+            if (existsError || exists) continue;
+
+            await progressCallback({ message: `Removing missing backup ${backup.id}`});
+
+            const [delError] = await safe(backups.del(backup.id));
+            if (delError) debug(`cleanupBackup: error removing ${backup.id} from database`, delError);
+
+            missingBackupIds.push(backup.id);
+        }
+
+        ++ page;
+    } while (result.length === perPage);
+
+    debug('cleanupMissingBackups: done');
+
+    return missingBackupIds;
+}
+
+// removes the snapshots of apps that have been uninstalled
+async function cleanupSnapshots(backupConfig) {
+    assert.strictEqual(typeof backupConfig, 'object');
+
+    const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
+    const info = safe.JSON.parse(contents);
+    if (!info) return;
+
+    delete info.box;
+
+    for (const appId of Object.keys(info)) {
+        const app = await apps.get(appId);
+        if (app) continue; // app is still installed
+
+        await new Promise((resolve) => {
+            async function done(/* ignoredError */) {
+                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
+                safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`));
+
+                await safe(backups.setSnapshotInfo(appId, null /* info */), { debug });
+                debug(`cleanupSnapshots: cleaned up snapshot of app ${appId}`);
+
+                resolve();
+            }
+
+            if (info[appId].format ==='tgz') {
+                storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), done);
+            } else {
+                const events = storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format));
+                events.on('progress', function (detail) { debug(`cleanupSnapshots: ${detail}`); });
+                events.on('done', done);
+            }
+        });
+    }
+
+    debug('cleanupSnapshots: done');
+}
+
+async function run(progressCallback) {
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const backupConfig = await settings.getBackupConfig();
+
+    if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
+        debug('cleanup: keeping all backups');
+        return {};
+    }
+
+    await progressCallback({ percent: 10, message: 'Cleaning box backups' });
+    const { removedBoxBackupIds, referencedAppBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback);
+
+    await progressCallback({ percent: 40, message: 'Cleaning app backups' });
+    const removedAppBackupIds = await cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback);
+
+    await progressCallback({ percent: 70, message: 'Cleaning missing backups' });
+    const missingBackupIds = await cleanupMissingBackups(backupConfig, progressCallback);
+
+    await progressCallback({ percent: 90, message: 'Cleaning snapshots' });
+    await cleanupSnapshots(backupConfig);
+
+    return { removedBoxBackupIds, removedAppBackupIds, missingBackupIds };
+}

src/backupdb.js

@@ -1,161 +0,0 @@
-'use strict';
-
-var assert = require('assert'),
-    BoxError = require('./boxerror.js'),
-    database = require('./database.js'),
-    safe = require('safetydance'),
-    util = require('util');
-
-var BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
-
-exports = module.exports = {
-    add,
-
-    getByTypePaged,
-    getByIdentifierPaged,
-    getByIdentifierAndStatePaged,
-
-    get,
-    del,
-    update,
-
-    _clear: clear
-};
-
-function postProcess(result) {
-    assert.strictEqual(typeof result, 'object');
-
-    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
-
-    result.manifest = result.manifestJson ? safe.JSON.parse(result.manifestJson) : null;
-    delete result.manifestJson;
-}
-
-function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof state, 'string');
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, state, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function getByTypePaged(type, page, perPage, callback) {
-    assert.strictEqual(typeof type, 'string');
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ type, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function getByIdentifierPaged(identifier, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert(typeof page === 'number' && page > 0);
-    assert(typeof perPage === 'number' && perPage > 0);
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, (page-1)*perPage, perPage ], function (error, results) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            results.forEach(function (result) { postProcess(result); });
-
-            callback(null, results);
-        });
-}
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
-        [ id ], function (error, result) {
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-            if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
-
-            postProcess(result[0]);
-
-            callback(null, result[0]);
-        });
-}
-
-function add(id, data, callback) {
-    assert(data && typeof data === 'object');
-    assert.strictEqual(typeof id, 'string');
-    assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number');
-    assert.strictEqual(typeof data.packageVersion, 'string');
-    assert.strictEqual(typeof data.type, 'string');
-    assert.strictEqual(typeof data.identifier, 'string');
-    assert.strictEqual(typeof data.state, 'string');
-    assert(util.isArray(data.dependsOn));
-    assert.strictEqual(typeof data.manifest, 'object');
-    assert.strictEqual(typeof data.format, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var creationTime = data.creationTime || new Date(); // allow tests to set the time
-    var manifestJson = JSON.stringify(data.manifest);
-
-    database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ],
-        function (error) {
-            if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
-            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-            callback(null);
-        });
-}
-
-function update(id, backup, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof backup, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var fields = [ ], values = [ ];
-    for (var p in backup) {
-        fields.push(p + ' = ?');
-        values.push(backup[p]);
-    }
-    values.push(id);
-
-    database.query('UPDATE backups SET ' + fields.join(', ') + ' WHERE id = ?', values, function (error) {
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.NOT_FOUND, 'Backup not found'));
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null);
-    });
-}
-
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('TRUNCATE TABLE backups', [], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        callback(null);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        callback(null);
-    });
-}

src/blobs.js

@@ -0,0 +1,103 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    get,
+    set,
+    del,
+
+    initSecrets,
+
+    ACME_ACCOUNT_KEY: 'acme_account_key',
+    ADDON_TURN_SECRET: 'addon_turn_secret',
+    DHPARAMS: 'dhparams',
+    SFTP_PUBLIC_KEY: 'sftp_public_key',
+    SFTP_PRIVATE_KEY: 'sftp_private_key',
+
+    CERT_PREFIX: 'cert',
+
+    _clear: clear
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    constants = require('./constants.js'),
+    crypto = require('crypto'),
+    database = require('./database.js'),
+    debug = require('debug')('box:blobs'),
+    paths = require('./paths.js'),
+    safe = require('safetydance');
+
+const BLOBS_FIELDS = [ 'id', 'value' ].join(',');
+
+async function get(id) {
+    assert.strictEqual(typeof id, 'string');
+
+    const result = await database.query(`SELECT ${BLOBS_FIELDS} FROM blobs WHERE id = ?`, [ id ]);
+    if (result.length === 0) return null;
+    return result[0].value;
+}
+
+async function set(id, value) {
+    assert.strictEqual(typeof id, 'string');
+    assert(value === null || Buffer.isBuffer(value));
+
+    await database.query('INSERT INTO blobs (id, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', [ id, value ]);
+}
+
+async function del(id) {
+    await database.query('DELETE FROM blobs WHERE id=?', [ id ]);
+}
+
+async function clear() {
+    await database.query('DELETE FROM blobs');
+}
+
+async function initSecrets() {
+    let acmeAccountKey = await get(exports.ACME_ACCOUNT_KEY);
+    if (!acmeAccountKey) {
+        acmeAccountKey = safe.child_process.execSync('openssl genrsa 4096');
+        if (!acmeAccountKey) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate acme account key: ${safe.error.message}`);
+        await set(exports.ACME_ACCOUNT_KEY, acmeAccountKey);
+    }
+
+    let turnSecret = await get(exports.ADDON_TURN_SECRET);
+    if (!turnSecret) {
+        turnSecret = 'a' + crypto.randomBytes(15).toString('hex'); // prefix with a to ensure string starts with a letter
+        await set(exports.ADDON_TURN_SECRET, Buffer.from(turnSecret));
+    }
+
+    // TODO maybe skip this in tests if possible again
+    let dhparams = await get(exports.DHPARAMS);
+    if (!dhparams) {
+        debug('initSecrets: generating dhparams.pem. this takes forever');
+        if (constants.TEST) dhparams = safe.fs.readFileSync('/tmp/dhparams.pem');
+        if (!dhparams) dhparams = safe.child_process.execSync('openssl dhparam 2048');
+        if (!dhparams) throw new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+        if (constants.TEST) safe.fs.writeFileSync('/tmp/dhparams.pem', dhparams);
+        if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
+        await set(exports.DHPARAMS, dhparams);
+    } else if (!safe.fs.existsSync(paths.DHPARAMS_FILE)) {
+        if (!safe.fs.writeFileSync(paths.DHPARAMS_FILE, dhparams)) throw new BoxError(BoxError.FS_ERROR, `Could not save dhparams.pem: ${safe.error.message}`);
+    }
+
+    let sftpPrivateKey = await get(exports.SFTP_PRIVATE_KEY);
+    let sftpPublicKey = await get(exports.SFTP_PUBLIC_KEY);
+
+    if (!sftpPrivateKey || !sftpPublicKey) {
+        debug('initSecrets: generate sftp keys');
+        if (constants.TEST) {
+            safe.fs.unlinkSync(paths.SFTP_PUBLIC_KEY_FILE);
+            safe.fs.unlinkSync(paths.SFTP_PRIVATE_KEY_FILE);
+        }
+        if (!safe.child_process.execSync(`ssh-keygen -m PEM -t rsa -f "${paths.SFTP_KEYS_DIR}/ssh_host_rsa_key" -q -N ""`)) throw new BoxError(BoxError.OPENSSL_ERROR, `Could not generate sftp ssh keys: ${safe.error.message}`);
+        sftpPublicKey = safe.fs.readFileSync(paths.SFTP_PUBLIC_KEY_FILE);
+        await set(exports.SFTP_PUBLIC_KEY, sftpPublicKey);
+        sftpPrivateKey = safe.fs.readFileSync(paths.SFTP_PRIVATE_KEY_FILE);
+        await set(exports.SFTP_PRIVATE_KEY, sftpPrivateKey);
+    } else if (!safe.fs.existsSync(paths.SFTP_PUBLIC_KEY_FILE) || !safe.fs.existsSync(paths.SFTP_PRIVATE_KEY_FILE)) {
+        if (!safe.fs.writeFileSync(paths.SFTP_PUBLIC_KEY_FILE, sftpPublicKey)) throw new BoxError(BoxError.FS_ERROR, `Could not save sftp public key: ${safe.error.message}`);
+        if (!safe.fs.writeFileSync(paths.SFTP_PRIVATE_KEY_FILE, sftpPrivateKey)) throw new BoxError(BoxError.FS_ERROR, `Could not save sftp private key: ${safe.error.message}`);
+    }
+}

src/boxerror.js

@@ -9,17 +9,17 @@ const assert = require('assert'),
 
 exports = module.exports = BoxError;
 
-function BoxError(reason, errorOrMessage, details) {
+function BoxError(reason, errorOrMessage, override) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
-    assert(typeof details === 'object' || typeof details === 'undefined');
+    assert(typeof override === 'object' || typeof override === 'undefined');
 
     Error.call(this);
     Error.captureStackTrace(this, this.constructor);
 
     this.name = this.constructor.name;
     this.reason = reason;
-    this.details = details || {};
+    this.details = {};
 
     if (typeof errorOrMessage === 'undefined') {
         this.message = reason;
@@ -28,7 +28,7 @@ function BoxError(reason, errorOrMessage, details) {
     } else { // error object
         this.message = errorOrMessage.message;
         this.nestedError = errorOrMessage;
-        _.extend(this.details, errorOrMessage); // copy enumerable properies
+        _.extend(this, override); // copy enumerable properies
     }
 }
 util.inherits(BoxError, Error);
@@ -47,18 +47,20 @@ BoxError.DOCKER_ERROR = 'Docker Error';
 BoxError.EXTERNAL_ERROR = 'External Error'; // use this for external API errors
 BoxError.FEATURE_DISABLED = 'Feature Disabled';
 BoxError.FS_ERROR = 'FileSystem Error';
-BoxError.INACTIVE = 'Inactive';
+BoxError.INACTIVE = 'Inactive'; // service/volume/mount
 BoxError.INTERNAL_ERROR = 'Internal Error';
 BoxError.INVALID_CREDENTIALS = 'Invalid Credentials';
 BoxError.IPTABLES_ERROR = 'IPTables Error';
 BoxError.LICENSE_ERROR = 'License Error';
 BoxError.LOGROTATE_ERROR = 'Logrotate Error';
 BoxError.MAIL_ERROR = 'Mail Error';
+BoxError.MOUNT_ERROR = 'Mount Error';
 BoxError.NETWORK_ERROR = 'Network Error';
 BoxError.NGINX_ERROR = 'Nginx Error';
 BoxError.NOT_FOUND = 'Not found';
 BoxError.NOT_IMPLEMENTED = 'Not implemented';
 BoxError.NOT_SIGNED = 'Not Signed';
+BoxError.NOT_SUPPORTED = 'Not Supported';
 BoxError.OPENSSL_ERROR = 'OpenSSL Error';
 BoxError.PLAN_LIMIT = 'Plan Limit';
 BoxError.SPAWN_ERROR = 'Spawn Error';
@@ -84,12 +86,14 @@ BoxError.toHttpError = function (error) {
     case BoxError.ALREADY_EXISTS:
     case BoxError.BAD_STATE:
     case BoxError.CONFLICT:
+    case BoxError.NOT_SUPPORTED:
         return new HttpError(409, error);
     case BoxError.INVALID_CREDENTIALS:
         return new HttpError(412, error);
     case BoxError.EXTERNAL_ERROR:
     case BoxError.NETWORK_ERROR:
     case BoxError.FS_ERROR:
+    case BoxError.MOUNT_ERROR:
     case BoxError.MAIL_ERROR:
     case BoxError.DOCKER_ERROR:
     case BoxError.ADDONS_ERROR:

src/cert/acme2.js

@@ -1,628 +0,0 @@
-'use strict';
-
-var assert = require('assert'),
-    async = require('async'),
-    BoxError = require('../boxerror.js'),
-    crypto = require('crypto'),
-    debug = require('debug')('box:cert/acme2'),
-    domains = require('../domains.js'),
-    fs = require('fs'),
-    path = require('path'),
-    paths = require('../paths.js'),
-    request = require('request'),
-    safe = require('safetydance'),
-    util = require('util'),
-    _ = require('underscore');
-
-const CA_PROD_DIRECTORY_URL = 'https://acme-v02.api.letsencrypt.org/directory',
-    CA_STAGING_DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory';
-
-exports = module.exports = {
-    getCertificate: getCertificate,
-
-    // testing
-    _name: 'acme',
-    _getChallengeSubdomain: getChallengeSubdomain
-};
-
-// http://jose.readthedocs.org/en/latest/
-// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
-// https://community.letsencrypt.org/t/list-of-client-implementations/2103
-
-function Acme2(options) {
-    assert.strictEqual(typeof options, 'object');
-
-    this.accountKeyPem = null; // Buffer
-    this.email = options.email;
-    this.keyId = null;
-    this.caDirectory = options.prod ? CA_PROD_DIRECTORY_URL : CA_STAGING_DIRECTORY_URL;
-    this.directory = {};
-    this.performHttpAuthorization = !!options.performHttpAuthorization;
-    this.wildcard = !!options.wildcard;
-}
-
-// urlsafe base64 encoding (jose)
-function urlBase64Encode(string) {
-    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
-}
-
-function b64(str) {
-    var buf = util.isBuffer(str) ? str : Buffer.from(str);
-    return urlBase64Encode(buf.toString('base64'));
-}
-
-function getModulus(pem) {
-    assert(util.isBuffer(pem));
-
-    var stdout = safe.child_process.execSync('openssl rsa -modulus -noout', { input: pem, encoding: 'utf8' });
-    if (!stdout) return null;
-    var match = stdout.match(/Modulus=([0-9a-fA-F]+)$/m);
-    if (!match) return null;
-    return Buffer.from(match[1], 'hex');
-}
-
-Acme2.prototype.sendSignedRequest = function (url, payload, callback) {
-    assert.strictEqual(typeof url, 'string');
-    assert.strictEqual(typeof payload, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    assert(util.isBuffer(this.accountKeyPem));
-
-    const that = this;
-    let header = {
-        url: url,
-        alg: 'RS256'
-    };
-
-    // keyId is null when registering account
-    if (this.keyId) {
-        header.kid = this.keyId;
-    } else {
-        header.jwk = {
-            e: b64(Buffer.from([0x01, 0x00, 0x01])), // exponent - 65537
-            kty: 'RSA',
-            n: b64(getModulus(this.accountKeyPem))
-        };
-    }
-
-    var payload64 = b64(payload);
-
-    request.get(this.directory.newNonce, { json: true, timeout: 30000 }, function (error, response) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`));
-        if (response.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching nonce : ' + response.statusCode));
-
-        const nonce = response.headers['Replay-Nonce'.toLowerCase()];
-        if (!nonce) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'No nonce in response'));
-
-        debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
-
-        var protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
-
-        var signer = crypto.createSign('RSA-SHA256');
-        signer.update(protected64 + '.' + payload64, 'utf8');
-        var signature64 = urlBase64Encode(signer.sign(that.accountKeyPem, 'base64'));
-
-        var data = {
-            protected: protected64,
-            payload: payload64,
-            signature: signature64
-        };
-
-        request.post(url, { headers: { 'Content-Type': 'application/jose+json', 'User-Agent': 'acme-cloudron' }, body: JSON.stringify(data), timeout: 30000 }, function (error, response) {
-            if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error sending signed request: ${error.message}`)); // network error
-
-            // we don't set json: true in request because it ends up mangling the content-type
-            // we don't set json: true in request because it ends up mangling the content-type
-            if (response.headers['content-type'] === 'application/json') response.body = safe.JSON.parse(response.body);
-
-            callback(null, response);
-        });
-    });
-};
-
-// https://tools.ietf.org/html/rfc8555#section-6.3
-Acme2.prototype.postAsGet = function (url, callback) {
-    this.sendSignedRequest(url, '', callback);
-};
-
-Acme2.prototype.updateContact = function (registrationUri, callback) {
-    assert.strictEqual(typeof registrationUri, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`updateContact: registrationUri: ${registrationUri} email: ${this.email}`);
-
-    // https://github.com/ietf-wg-acme/acme/issues/30
-    const payload = {
-        contact: [ 'mailto:' + this.email ]
-    };
-
-    const that = this;
-    this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to update contact. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        debug(`updateContact: contact of user updated to ${that.email}`);
-
-        callback();
-    });
-};
-
-Acme2.prototype.registerUser = function (callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var payload = {
-        termsOfServiceAgreed: true
-    };
-
-    debug('registerUser: registering user');
-
-    var that = this;
-    this.sendSignedRequest(this.directory.newAccount, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        // 200 if already exists. 201 for new accounts
-        if (result.statusCode !== 200 && result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to register new account. Expecting 200 or 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        debug(`registerUser: user registered keyid: ${result.headers.location}`);
-
-        that.keyId = result.headers.location;
-
-        that.updateContact(result.headers.location, callback);
-    });
-};
-
-Acme2.prototype.newOrder = function (domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var payload = {
-        identifiers: [{
-            type: 'dns',
-            value: domain
-        }]
-    };
-
-    debug('newOrder: %s', domain);
-
-    this.sendSignedRequest(this.directory.newOrder, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        if (result.statusCode === 403) return callback(new BoxError(BoxError.ACCESS_DENIED, `Forbidden sending new order: ${result.body.detail}`));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to send new order. Expecting 201, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        debug('newOrder: created order %s %j', domain, result.body);
-
-        const order = result.body, orderUrl = result.headers.location;
-
-        if (!Array.isArray(order.authorizations)) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid authorizations in order'));
-        if (typeof order.finalize !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid finalize in order'));
-        if (typeof orderUrl !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'invalid order location in order header'));
-
-        callback(null, order, orderUrl);
-    });
-};
-
-Acme2.prototype.waitForOrder = function (orderUrl, callback) {
-    assert.strictEqual(typeof orderUrl, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`waitForOrder: ${orderUrl}`);
-    const that = this;
-
-    async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
-        debug('waitForOrder: getting status');
-
-        that.postAsGet(orderUrl, function (error, result) {
-            if (error) {
-                debug('waitForOrder: network error getting uri %s', orderUrl);
-                return retryCallback(error);
-            }
-            if (result.statusCode !== 200) {
-                debug('waitForOrder: invalid response code getting uri %s', result.statusCode);
-                return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
-            }
-
-            debug('waitForOrder: status is "%s %j', result.body.status, result.body);
-
-            if (result.body.status === 'pending' || result.body.status === 'processing') return retryCallback(new BoxError(BoxError.TRY_AGAIN, `Request is in ${result.body.status} state`));
-            else if (result.body.status === 'valid' && result.body.certificate) return retryCallback(null, result.body.certificate);
-            else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status or invalid response: ' + result.body));
-        });
-    }, callback);
-};
-
-Acme2.prototype.getKeyAuthorization = function (token) {
-    assert(util.isBuffer(this.accountKeyPem));
-
-    let jwk = {
-        e: b64(Buffer.from([0x01, 0x00, 0x01])), // Exponent - 65537
-        kty: 'RSA',
-        n: b64(getModulus(this.accountKeyPem))
-    };
-
-    let shasum = crypto.createHash('sha256');
-    shasum.update(JSON.stringify(jwk));
-    let thumbprint = urlBase64Encode(shasum.digest('base64'));
-    return token + '.' + thumbprint;
-};
-
-Acme2.prototype.notifyChallengeReady = function (challenge, callback) {
-    assert.strictEqual(typeof challenge, 'object'); // { type, status, url, token }
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('notifyChallengeReady: %s was met', challenge.url);
-
-    const keyAuthorization = this.getKeyAuthorization(challenge.token);
-
-    var payload = {
-        resource: 'challenge',
-        keyAuthorization: keyAuthorization
-    };
-
-    this.sendSignedRequest(challenge.url, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to notify challenge. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        callback();
-    });
-};
-
-Acme2.prototype.waitForChallenge = function (challenge, callback) {
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('waitingForChallenge: %j', challenge);
-    const that = this;
-
-    async.retry({ times: 15, interval: 20000 }, function (retryCallback) {
-        debug('waitingForChallenge: getting status');
-
-        that.postAsGet(challenge.url, function (error, result) {
-            if (error) {
-                debug('waitForChallenge: network error getting uri %s', challenge.url);
-                return retryCallback(error);
-            }
-            if (result.statusCode !== 200) {
-                debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
-                return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
-            }
-
-            debug('waitForChallenge: status is "%s" %j', result.body.status, result.body);
-
-            if (result.body.status === 'pending') return retryCallback(new BoxError(BoxError.TRY_AGAIN));
-            else if (result.body.status === 'valid') return retryCallback();
-            else return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
-        });
-    }, function retryFinished(error) {
-        // async.retry will pass 'undefined' as second arg making it unusable with async.waterfall()
-        callback(error);
-    });
-};
-
-// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
-Acme2.prototype.signCertificate = function (domain, finalizationUrl, csrDer, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof finalizationUrl, 'string');
-    assert(util.isBuffer(csrDer));
-    assert.strictEqual(typeof callback, 'function');
-
-    const payload = {
-        csr: b64(csrDer)
-    };
-
-    debug('signCertificate: sending sign request');
-
-    this.sendSignedRequest(finalizationUrl, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(error);
-        // 429 means we reached the cert limit for this domain
-        if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to sign certificate. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-        return callback(null);
-    });
-};
-
-Acme2.prototype.createKeyAndCsr = function (hostname, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var outdir = paths.APP_CERTS_DIR;
-    const certName = hostname.replace('*.', '_.');
-    var csrFile = path.join(outdir, `${certName}.csr`);
-    var privateKeyFile = path.join(outdir, `${certName}.key`);
-
-    if (safe.fs.existsSync(privateKeyFile)) {
-        // in some old releases, csr file was corrupt. so always regenerate it
-        debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
-    } else {
-        var key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
-        if (!key) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
-        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error));
-
-        debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
-    }
-
-    var csrDer = safe.child_process.execSync(`openssl req -new -key ${privateKeyFile} -outform DER -subj /CN=${hostname}`);
-    if (!csrDer) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
-    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new BoxError(BoxError.FS_ERROR, safe.error)); // bookkeeping
-
-    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
-
-    callback(null, csrDer);
-};
-
-Acme2.prototype.downloadCertificate = function (hostname, certUrl, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof certUrl, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var outdir = paths.APP_CERTS_DIR;
-    const that = this;
-
-    async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
-        debug('downloadCertificate: downloading certificate');
-
-        that.postAsGet(certUrl, function (error, result) {
-            if (error) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error when downloading certificate: ${error.message}`));
-            if (result.statusCode === 202) return retryCallback(new BoxError(BoxError.TRY_AGAIN, 'Retry downloading certificate'));
-            if (result.statusCode !== 200) return retryCallback(new BoxError(BoxError.EXTERNAL_ERROR, `Failed to get cert. Expecting 200, got ${result.statusCode} ${JSON.stringify(result.body)}`));
-
-            const fullChainPem = result.body; // buffer
-
-            const certName = hostname.replace('*.', '_.');
-            var certificateFile = path.join(outdir, `${certName}.cert`);
-            if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return retryCallback(new BoxError(BoxError.FS_ERROR, safe.error));
-
-            debug('downloadCertificate: cert file for %s saved at %s', hostname, certificateFile);
-
-            retryCallback(null);
-        });
-    }, callback);
-};
-
-Acme2.prototype.prepareHttpChallenge = function (hostname, domain, authorization, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof authorization, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('acmeFlow: challenges: %j', authorization);
-    let httpChallenges = authorization.challenges.filter(function(x) { return x.type === 'http-01'; });
-    if (httpChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no http challenges'));
-    let challenge = httpChallenges[0];
-
-    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
-
-    let keyAuthorization = this.getKeyAuthorization(challenge.token);
-
-    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
-
-    fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), keyAuthorization, function (error) {
-        if (error) return callback(new BoxError(BoxError.FS_ERROR, error));
-
-        callback(null, challenge);
-    });
-};
-
-Acme2.prototype.cleanupHttpChallenge = function (hostname, domain, challenge, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('cleanupHttpChallenge: unlinking %s', path.join(paths.ACME_CHALLENGES_DIR, challenge.token));
-
-    fs.unlink(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), callback);
-};
-
-function getChallengeSubdomain(hostname, domain) {
-    let challengeSubdomain;
-
-    if (hostname === domain) {
-        challengeSubdomain = '_acme-challenge';
-    } else if (hostname.includes('*')) { // wildcard
-        let subdomain = hostname.slice(0, -domain.length - 1);
-        challengeSubdomain = subdomain ? subdomain.replace('*', '_acme-challenge') : '_acme-challenge';
-    } else {
-        challengeSubdomain = '_acme-challenge.' + hostname.slice(0, -domain.length - 1);
-    }
-
-    debug(`getChallengeSubdomain: challenge subdomain for hostname ${hostname} at domain ${domain} is ${challengeSubdomain}`);
-
-    return challengeSubdomain;
-}
-
-Acme2.prototype.prepareDnsChallenge = function (hostname, domain, authorization, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof authorization, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('acmeFlow: challenges: %j', authorization);
-    let dnsChallenges = authorization.challenges.filter(function(x) { return x.type === 'dns-01'; });
-    if (dnsChallenges.length === 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'no dns challenges'));
-    let challenge = dnsChallenges[0];
-
-    const keyAuthorization = this.getKeyAuthorization(challenge.token);
-    let shasum = crypto.createHash('sha256');
-    shasum.update(keyAuthorization);
-
-    const txtValue = urlBase64Encode(shasum.digest('base64'));
-    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
-
-    debug(`prepareDnsChallenge: update ${challengeSubdomain} with ${txtValue}`);
-
-    domains.upsertDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-        if (error) return callback(error);
-
-        domains.waitForDnsRecord(challengeSubdomain, domain, 'TXT', txtValue, { times: 200 }, function (error) {
-            if (error) return callback(error);
-
-            callback(null, challenge);
-        });
-    });
-};
-
-Acme2.prototype.cleanupDnsChallenge = function (hostname, domain, challenge, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    const keyAuthorization = this.getKeyAuthorization(challenge.token);
-    let shasum = crypto.createHash('sha256');
-    shasum.update(keyAuthorization);
-
-    const txtValue = urlBase64Encode(shasum.digest('base64'));
-    let challengeSubdomain = getChallengeSubdomain(hostname, domain);
-
-    debug(`cleanupDnsChallenge: remove ${challengeSubdomain} with ${txtValue}`);
-
-    domains.removeDnsRecords(challengeSubdomain, domain, 'TXT', [ `"${txtValue}"` ], function (error) {
-        if (error) return callback(error);
-
-        callback(null);
-    });
-};
-
-Acme2.prototype.prepareChallenge = function (hostname, domain, authorizationUrl, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof authorizationUrl, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`prepareChallenge: http: ${this.performHttpAuthorization}`);
-
-    const that = this;
-    this.postAsGet(authorizationUrl, function (error, response) {
-        if (error) return callback(error);
-        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code getting authorization : ' + response.statusCode));
-
-        const authorization = response.body;
-
-        if (that.performHttpAuthorization) {
-            that.prepareHttpChallenge(hostname, domain, authorization, callback);
-        } else {
-            that.prepareDnsChallenge(hostname, domain, authorization, callback);
-        }
-    });
-};
-
-Acme2.prototype.cleanupChallenge = function (hostname, domain, challenge, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`cleanupChallenge: http: ${this.performHttpAuthorization}`);
-
-    if (this.performHttpAuthorization) {
-        this.cleanupHttpChallenge(hostname, domain, challenge, callback);
-    } else {
-        this.cleanupDnsChallenge(hostname, domain, challenge, callback);
-    }
-};
-
-Acme2.prototype.acmeFlow = function (hostname, domain, callback) {
-    assert.strictEqual(typeof hostname, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
-        debug('getCertificate: generating acme account key on first run');
-        this.accountKeyPem = safe.child_process.execSync('openssl genrsa 4096');
-        if (!this.accountKeyPem) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
-
-        safe.fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, this.accountKeyPem);
-    } else {
-        debug('getCertificate: using existing acme account key');
-        this.accountKeyPem = fs.readFileSync(paths.ACME_ACCOUNT_KEY_FILE);
-    }
-
-    var that = this;
-    this.registerUser(function (error) {
-        if (error) return callback(error);
-
-        that.newOrder(hostname, function (error, order, orderUrl) {
-            if (error) return callback(error);
-
-            async.eachSeries(order.authorizations, function (authorizationUrl, iteratorCallback) {
-                debug(`acmeFlow: authorizing ${authorizationUrl}`);
-
-                that.prepareChallenge(hostname, domain, authorizationUrl, function (error, challenge) {
-                    if (error) return iteratorCallback(error);
-
-                    async.waterfall([
-                        that.notifyChallengeReady.bind(that, challenge),
-                        that.waitForChallenge.bind(that, challenge),
-                        that.createKeyAndCsr.bind(that, hostname),
-                        that.signCertificate.bind(that, hostname, order.finalize),
-                        that.waitForOrder.bind(that, orderUrl),
-                        that.downloadCertificate.bind(that, hostname)
-                    ], function (error) {
-                        that.cleanupChallenge(hostname, domain, challenge, function (cleanupError) {
-                            if (cleanupError) debug('acmeFlow: ignoring error when cleaning up challenge:', cleanupError);
-
-                            iteratorCallback(error);
-                        });
-                    });
-                });
-            }, callback);
-        });
-    });
-};
-
-Acme2.prototype.getDirectory = function (callback) {
-    const that = this;
-
-    request.get(this.caDirectory, { json: true, timeout: 30000 }, function (error, response) {
-        if (error) return callback(new BoxError(BoxError.NETWORK_ERROR, `Network error getting directory: ${error.message}`));
-        if (response.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response code when fetching directory : ' + response.statusCode));
-
-        if (typeof response.body.newNonce !== 'string' ||
-            typeof response.body.newOrder !== 'string' ||
-            typeof response.body.newAccount !== 'string') return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Invalid response body : ${response.body}`));
-
-        that.directory = response.body;
-
-        callback(null);
-    });
-};
-
-Acme2.prototype.getCertificate = function (vhost, domain, callback) {
-    assert.strictEqual(typeof vhost, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug(`getCertificate: start acme flow for ${vhost} from ${this.caDirectory}`);
-
-    if (vhost !== domain && this.wildcard) { // bare domain is not part of wildcard SAN
-        vhost = domains.makeWildcard(vhost);
-        debug(`getCertificate: will get wildcard cert for ${vhost}`);
-    }
-
-    const that = this;
-    this.getDirectory(function (error) {
-        if (error) return callback(error);
-
-        that.acmeFlow(vhost, domain, function (error) {
-            if (error) return callback(error);
-
-            var outdir = paths.APP_CERTS_DIR;
-            const certName = vhost.replace('*.', '_.');
-            callback(null, path.join(outdir, `${certName}.cert`), path.join(outdir, `${certName}.key`));
-        });
-    });
-};
-
-function getCertificate(vhost, domain, options, callback) {
-    assert.strictEqual(typeof vhost, 'string'); // this can also be a wildcard domain (for alias domains)
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    let attempt = 1;
-    async.retry({ times: 3, interval: 0 }, function (retryCallback) {
-        debug(`getCertificate: attempt ${attempt++}`);
-
-        let acme = new Acme2(options || { });
-        acme.getCertificate(vhost, domain, retryCallback);
-    }, callback);
-}

src/changelog.js

@@ -5,7 +5,7 @@ let assert = require('assert'),
     path = require('path');
 
 exports = module.exports = {
-    getChanges: getChanges
+    getChanges
 };
 
 function getChanges(version) {

src/cloudron.js

@@ -10,6 +10,7 @@ exports = module.exports = {
     isRebootRequired,
 
     onActivated,
+    onRestored,
 
     setupDnsAndCert,
 
@@ -17,6 +18,7 @@ exports = module.exports = {
     setDashboardDomain,
     updateDashboardDomain,
     renewCerts,
+    syncDnsRecords,
 
     runSystemChecks
 };
@@ -24,14 +26,16 @@ exports = module.exports = {
 const apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
+    backuptask = require('./backuptask.js'),
     BoxError = require('./boxerror.js'),
     branding = require('./branding.js'),
     constants = require('./constants.js'),
     cron = require('./cron.js'),
     debug = require('debug')('box:cloudron'),
+    delay = require('delay'),
+    dns = require('./dns.js'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
@@ -51,192 +55,177 @@ const apps = require('./apps.js'),
     tasks = require('./tasks.js'),
     users = require('./users.js');
 
-var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');
+const REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');
 
-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
+async function initialize() {
+    safe(runStartupTasks(), { debug });
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    runStartupTasks();
-
-    notifyUpdate(callback);
+    await notifyUpdate();
 }
 
-function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        cron.stopJobs,
-        platform.stopAllTasks
-    ], callback);
+async function uninitialize() {
+    await cron.stopJobs();
+    await platform.stopAllTasks();
 }
 
-function onActivated(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function onActivated(options) {
+    assert.strictEqual(typeof options, 'object');
+
+    debug('onActivated: running post activation tasks');
 
     // Starting the platform after a user is available means:
     // 1. mail bounces can now be sent to the cloudron owner
     // 2. the restore code path can run without sudo (since mail/ is non-root)
-    async.series([
-        platform.start,
-        cron.startJobs,
-        function checkBackupConfiguration(done) {
-            backups.checkConfiguration(function (error, message) {
-                if (error) return done(error);
-                notifications.alert(notifications.ALERT_BACKUP_CONFIG, 'Backup configuration is unsafe', message, done);
-            });
-        },
-        // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
-        // the UI some time to query the dashboard domain in the restore code path
-        (done) => setTimeout(() => reverseProxy.writeDefaultConfig({ activated :true }, done), 30000)
-    ], callback);
+    await platform.start(options);
+    await cron.startJobs();
+
+    // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
+    // the UI some time to query the dashboard domain in the restore code path
+    await delay(30000);
+    await reverseProxy.writeDefaultConfig({ activated :true });
 }
 
-function notifyUpdate(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function onRestored(options) {
+    assert.strictEqual(typeof options, 'object');
 
+    debug('onRestored: downloading mail');
+
+    const [error, results] = await safe(backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_MAIL, backups.BACKUP_STATE_NORMAL, 1, 1));
+    if (error || results.length == 0) {
+        debug('startMail: could not find backup to restore', error, results);
+    } else {
+        debug(`startMail: downloading backup ${results[0].id}`);
+        const restoreConfig = { backupId: results[0].id, backupFormat: results[0].format };
+        // have to wait for download before starting mail container because we download as non-root user
+        await backuptask.downloadMail(restoreConfig, (progress) => debug(`startMail: ${progress.message}`));
+    }
+
+    await onActivated(options);
+}
+
+async function notifyUpdate() {
     const version = safe.fs.readFileSync(paths.VERSION_FILE, 'utf8');
-    if (version === constants.VERSION) return callback();
+    if (version === constants.VERSION) return;
 
-    eventlog.add(eventlog.ACTION_UPDATE_FINISH, auditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION }, function (error) {
-        if (error) return callback(error);
+    await eventlog.add(eventlog.ACTION_UPDATE_FINISH, AuditSource.CRON, { errorMessage: '', oldVersion: version || 'dev', newVersion: constants.VERSION });
 
-        tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }, function (error) {
-            if (error && error.reason !== BoxError.NOT_FOUND) return callback(error); // when hotfixing, task may not exist
+    const [error] = await safe(tasks.setCompletedByType(tasks.TASK_UPDATE, { error: null }));
+    if (error && error.reason !== BoxError.NOT_FOUND) throw error; // when hotfixing, task may not exist
 
-            safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
-
-            callback();
-        });
-    });
+    safe.fs.writeFileSync(paths.VERSION_FILE, constants.VERSION, 'utf8');
 }
 
 // each of these tasks can fail. we will add some routes to fix/re-run them
-function runStartupTasks() {
-    const tasks = [
-        // stop all the systemd tasks
-        platform.stopAllTasks,
+async function runStartupTasks() {
+    const tasks = [];
 
-        // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
-        function (callback) {
-            settings.getBackupConfig(function (error, backupConfig) {
-                if (error) return callback(error);
+    // stop all the systemd tasks
+    tasks.push(platform.stopAllTasks);
 
-                backups.configureCollectd(backupConfig, callback);
-            });
-        },
+    // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
+    tasks.push(async function () {
+        const backupConfig = await settings.getBackupConfig();
+        await backups.configureCollectd(backupConfig);
+    });
 
-        // always generate webadmin config since we have no versioning mechanism for the ejs
-        function (callback) {
-            if (!settings.adminDomain()) return callback();
+    // always generate webadmin config since we have no versioning mechanism for the ejs
+    tasks.push(async function () {
+        if (!settings.dashboardDomain()) return;
 
-            reverseProxy.writeDashboardConfig(settings.adminDomain(), callback);
-        },
+        await reverseProxy.writeDashboardConfig(settings.dashboardDomain());
+    });
 
+    tasks.push(async function () {
         // check activation state and start the platform
-        function (callback) {
-            users.isActivated(function (error, activated) {
-                if (error) return callback(error);
+        const activated = await users.isActivated();
 
-                // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
-                // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
-                // we remove the config as a simple security measure to not expose IP <-> domain
-                if (!activated) {
-                    debug('runStartupTasks: not activated. generating IP based redirection config');
-                    return reverseProxy.writeDefaultConfig({ activated: false }, callback);
-                }
-
-                onActivated(callback);
-            });
+        // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
+        // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
+        // we remove the config as a simple security measure to not expose IP <-> domain
+        if (!activated) {
+            debug('runStartupTasks: not activated. generating IP based redirection config');
+            return await reverseProxy.writeDefaultConfig({ activated: false });
         }
-    ];
+
+        await onActivated({});
+    });
 
     // we used to run tasks in parallel but simultaneous nginx reloads was causing issues
-    async.series(async.reflectAll(tasks), function (error, results) {
-        results.forEach((result, idx) => {
-            if (result.error) debug(`Startup task at index ${idx} failed: ${result.error.message}`);
-        });
-    });
+    for (let i = 0; i < tasks.length; i++) {
+        const [error] = await safe(tasks[i]());
+        if (error) debug(`Startup task at index ${i} failed: ${error.message}`);
+    }
 }
 
-function getConfig(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function getConfig() {
+    const release = safe.fs.readFileSync('/etc/lsb-release', 'utf-8');
+    if (release === null) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
+    const ubuntuVersion = release.match(/DISTRIB_DESCRIPTION="(.*)"/)[1];
 
-    settings.getAll(function (error, allSettings) {
-        if (error) return callback(error);
+    const allSettings = await settings.list();
 
-        // be picky about what we send out here since this is sent for 'normal' users as well
-        callback(null, {
-            apiServerOrigin: settings.apiServerOrigin(),
-            webServerOrigin: settings.webServerOrigin(),
-            adminDomain: settings.adminDomain(),
-            adminFqdn: settings.adminFqdn(),
-            mailFqdn: settings.mailFqdn(),
-            version: constants.VERSION,
-            isDemo: settings.isDemo(),
-            cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
-            footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
-            features: appstore.getFeatures(),
-            profileLocked: allSettings[settings.DIRECTORY_CONFIG_KEY].lockUserProfiles,
-            mandatory2FA: allSettings[settings.DIRECTORY_CONFIG_KEY].mandatory2FA
-        });
-    });
+    // be picky about what we send out here since this is sent for 'normal' users as well
+    return {
+        apiServerOrigin: settings.apiServerOrigin(),
+        webServerOrigin: settings.webServerOrigin(),
+        adminDomain: settings.dashboardDomain(),
+        adminFqdn: settings.dashboardFqdn(),
+        mailFqdn: settings.mailFqdn(),
+        version: constants.VERSION,
+        ubuntuVersion,
+        isDemo: settings.isDemo(),
+        cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
+        footer: branding.renderFooter(allSettings[settings.FOOTER_KEY] || constants.FOOTER),
+        features: appstore.getFeatures(),
+        profileLocked: allSettings[settings.DIRECTORY_CONFIG_KEY].lockUserProfiles,
+        mandatory2FA: allSettings[settings.DIRECTORY_CONFIG_KEY].mandatory2FA
+    };
 }
 
-function reboot(callback) {
-    notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '', function (error) {
-        if (error) debug('reboot: failed to clear reboot notification.', error);
+async function reboot() {
+    await notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '');
 
-        shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
-    });
+    const [error] = await safe(shell.promises.sudo('reboot', [ REBOOT_CMD ], {}));
+    if (error) debug('reboot: could not reboot', error);
 }
 
-function isRebootRequired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function isRebootRequired() {
     // https://serverfault.com/questions/92932/how-does-ubuntu-keep-track-of-the-system-restart-required-flag-in-motd
-    callback(null, fs.existsSync('/var/run/reboot-required'));
+    return fs.existsSync('/var/run/reboot-required');
 }
 
-// called from cron.js
-function runSystemChecks(callback) {
-    assert.strictEqual(typeof callback, 'function');
+async function runSystemChecks() {
+    debug('runSystemChecks: checking status');
 
-    async.parallel([
-        checkMailStatus,
-        checkRebootRequired
-    ], callback);
+    const checks = [
+        checkMailStatus(),
+        checkRebootRequired(),
+        checkUbuntuVersion()
+    ];
+
+    await Promise.allSettled(checks);
 }
 
-function checkMailStatus(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('checking mail status');
-
-    mail.checkConfiguration(function (error, message) {
-        if (error) return callback(error);
-
-        notifications.alert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly', message, callback);
-    });
+async function checkMailStatus() {
+    const message = await mail.checkConfiguration();
+    await notifications.alert(notifications.ALERT_MAIL_STATUS, 'Email is not configured properly', message);
 }
 
-function checkRebootRequired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('checking if reboot required');
-
-    isRebootRequired(function (error, rebootRequired) {
-        if (error) return callback(error);
-
-        notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '', callback);
-    });
+async function checkRebootRequired() {
+    const rebootRequired = await isRebootRequired();
+    await notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', rebootRequired ? 'To finish ubuntu security updates, a reboot is necessary.' : '');
 }
 
-function getLogs(unit, options, callback) {
+async function checkUbuntuVersion() {
+    const isXenial = fs.readFileSync('/etc/lsb-release', 'utf-8').includes('16.04');
+    if (!isXenial) return;
+
+    await notifications.alert(notifications.ALERT_UPDATE_UBUNTU, 'Ubuntu upgrade required', 'Ubuntu 16.04 has reached end of life and will not receive security and maintenance updates. Please follow https://docs.cloudron.io/guides/upgrade-ubuntu-18/ to upgrade to Ubuntu 18 at the earliest.');
+}
+
+async function getLogs(unit, options) {
     assert.strictEqual(typeof unit, 'string');
     assert(options && typeof options === 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     assert.strictEqual(typeof options.lines, 'number');
     assert.strictEqual(typeof options.format, 'string');
@@ -254,15 +243,15 @@ function getLogs(unit, options, callback) {
     // need to handle box.log without subdir
     if (unit === 'box') args.push(path.join(paths.LOG_DIR, 'box.log'));
     else if (unit.startsWith('crash-')) args.push(path.join(paths.CRASH_LOG_DIR, unit.slice(6) + '.log'));
-    else return callback(new BoxError(BoxError.BAD_FIELD, 'No such unit', { field: 'unit' }));
+    else throw new BoxError(BoxError.BAD_FIELD, 'No such unit', { field: 'unit' });
 
-    var cp = spawn('/usr/bin/tail', args);
+    const cp = spawn('/usr/bin/tail', args);
 
-    var transformStream = split(function mapper(line) {
+    const transformStream = split(function mapper(line) {
         if (format !== 'json') return line + '\n';
 
-        var data = line.split(' '); // logs are <ISOtimestamp> <msg>
-        var timestamp = (new Date(data[0])).getTime();
+        const data = line.split(' '); // logs are <ISOtimestamp> <msg>
+        let timestamp = (new Date(data[0])).getTime();
         if (isNaN(timestamp)) timestamp = 0;
 
         return JSON.stringify({
@@ -276,127 +265,98 @@ function getLogs(unit, options, callback) {
 
     cp.stdout.pipe(transformStream);
 
-    return callback(null, transformStream);
+    return transformStream;
 }
 
-function prepareDashboardDomain(domain, auditSource, callback) {
+async function prepareDashboardDomain(domain, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`prepareDashboardDomain: ${domain}`);
 
-    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+    if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
 
-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await domains.get(domain);
+    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');
 
-        const fqdn = domains.fqdn(constants.ADMIN_LOCATION, domainObject);
+    const fqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
-        apps.getAll(function (error, result) {
-            if (error) return callback(error);
+    const result = await apps.list();
+    if (result.some(app => app.fqdn === fqdn)) throw new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app');
 
-            const conflict = result.filter(app => app.fqdn === fqdn);
-            if (conflict.length) return callback(new BoxError(BoxError.BAD_STATE, 'Dashboard location conflicts with an existing app'));
+    const taskId = await tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.DASHBOARD_LOCATION, domain, auditSource ]);
 
-            tasks.add(tasks.TASK_SETUP_DNS_AND_CERT, [ constants.ADMIN_LOCATION, domain, auditSource ], function (error, taskId) {
-                if (error) return callback(error);
+    tasks.startTask(taskId, {});
 
-                tasks.startTask(taskId, {}, NOOP_CALLBACK);
-
-                callback(null, taskId);
-            });
-        });
-    });
+    return taskId;
 }
 
 // call this only pre activation since it won't start mail server
-function setDashboardDomain(domain, auditSource, callback) {
+async function setDashboardDomain(domain, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`setDashboardDomain: ${domain}`);
 
-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await domains.get(domain);
+    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'No such domain');
 
-        reverseProxy.writeDashboardConfig(domain, function (error) {
-            if (error) return callback(error);
+    await reverseProxy.writeDashboardConfig(domain);
+    const fqdn = dns.fqdn(constants.DASHBOARD_LOCATION, domainObject);
 
-            const fqdn = domains.fqdn(constants.ADMIN_LOCATION, domainObject);
+    await settings.setDashboardLocation(domain, fqdn);
 
-            settings.setAdminLocation(domain, fqdn, function (error) {
-                if (error) return callback(error);
+    await safe(appstore.updateCloudron({ domain }));
 
-                eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain: domain, fqdn: fqdn });
-
-                callback(null);
-            });
-        });
-    });
+    await eventlog.add(eventlog.ACTION_DASHBOARD_DOMAIN_UPDATE, auditSource, { domain, fqdn });
 }
 
 // call this only post activation because it will restart mail server
-function updateDashboardDomain(domain, auditSource, callback) {
+async function updateDashboardDomain(domain, auditSource) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
     debug(`updateDashboardDomain: ${domain}`);
 
-    if (settings.isDemo()) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode'));
+    if (settings.isDemo()) throw new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode');
 
-    setDashboardDomain(domain, auditSource, function (error) {
-        if (error) return callback(error);
+    await setDashboardDomain(domain, auditSource);
 
-        services.rebuildService('turn', NOOP_CALLBACK); // to update the realm variable
-
-        callback(null);
-    });
+    safe(services.rebuildService('turn', auditSource), { debug }); // to update the realm variable
 }
 
-function renewCerts(options, auditSource, callback) {
+async function renewCerts(options, auditSource) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    tasks.add(tasks.TASK_RENEW_CERTS, [ options, auditSource ], function (error, taskId) {
-        if (error) return callback(error);
-
-        tasks.startTask(taskId, {}, NOOP_CALLBACK);
-
-        callback(null, taskId);
-    });
+    const taskId = await tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ]);
+    tasks.startTask(taskId, {});
+    return taskId;
 }
 
-function setupDnsAndCert(subdomain, domain, auditSource, progressCallback, callback) {
+async function setupDnsAndCert(subdomain, domain, auditSource, progressCallback) {
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
 
-    domains.get(domain, function (error, domainObject) {
-        if (error) return callback(error);
+    const domainObject = await domains.get(domain);
+    const dashboardFqdn = dns.fqdn(subdomain, domainObject);
 
-        const adminFqdn = domains.fqdn(subdomain, domainObject);
+    const ip = await sysinfo.getServerIp();
 
-        sysinfo.getServerIp(function (error, ip) {
-            if (error) return callback(error);
-
-            async.series([
-                (done) => { progressCallback({ message: `Updating DNS of ${adminFqdn}` }); done(); },
-                domains.upsertDnsRecords.bind(null, subdomain, domain, 'A', [ ip ]),
-                (done) => { progressCallback({ message: `Waiting for DNS of ${adminFqdn}` }); done(); },
-                domains.waitForDnsRecord.bind(null, subdomain, domain, 'A', ip, { interval: 30000, times: 50000 }),
-                (done) => { progressCallback({ message: `Getting certificate of ${adminFqdn}` }); done(); },
-                reverseProxy.ensureCertificate.bind(null, domains.fqdn(subdomain, domainObject), domain, auditSource)
-            ], function (error) {
-                if (error) return callback(error);
-
-                callback(null);
-            });
-        });
-    });
+    progressCallback({ message: `Updating DNS of ${dashboardFqdn}` });
+    await dns.upsertDnsRecords(subdomain, domain, 'A', [ ip ]);
+    progressCallback({ message: `Waiting for DNS of ${dashboardFqdn}` });
+    await dns.waitForDnsRecord(subdomain, domain, 'A', ip, { interval: 30000, times: 50000 });
+    progressCallback({ message: `Getting certificate of ${dashboardFqdn}` });
+    await reverseProxy.ensureCertificate(dns.fqdn(subdomain, domainObject), domain, auditSource);
+}
+
+async function syncDnsRecords(options) {
+    assert.strictEqual(typeof options, 'object');
+
+    const taskId = await tasks.add(tasks.TASK_SYNC_DNS_RECORDS, [ options ]);
+    tasks.startTask(taskId, {});
+    return taskId;
 }

src/collectd.js

@@ -8,7 +8,6 @@ exports = module.exports = {
 const assert = require('assert'),
     BoxError = require('./boxerror.js'),
     debug = require('debug')('collectd'),
-    fs = require('fs'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -16,40 +15,29 @@ const assert = require('assert'),
 
 const CONFIGURE_COLLECTD_CMD = path.join(__dirname, 'scripts/configurecollectd.sh');
 
-function addProfile(name, profile, callback) {
+async function addProfile(name, profile) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof profile, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     const configFilePath = path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`);
 
     // skip restarting collectd if the profile already exists with the same contents
     const currentProfile = safe.fs.readFileSync(configFilePath, 'utf8') || '';
-    if (currentProfile === profile) return callback(null);
+    if (currentProfile === profile) return;
 
-    fs.writeFile(configFilePath, profile, function (error) {
-        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${error.message}`));
-
-        shell.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config'));
-
-            callback(null);
-        });
-    });
+    if (!safe.fs.writeFileSync(configFilePath, profile)) throw new BoxError(BoxError.FS_ERROR, `Error writing collectd config: ${safe.error.message}`);
 
+    const [error] = await safe(shell.promises.sudo('addCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'add', name ], {}));
+    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not add collectd config');
 }
 
-function removeProfile(name, callback) {
+async function removeProfile(name) {
     assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`), function (error) {
-        if (error && error.code !== 'ENOENT') debug('Error removing collectd profile', error);
+    if (!safe.fs.unlinkSync(path.join(paths.COLLECTD_APPCONFIG_DIR, `${name}.conf`))) {
+        if (safe.error.code !== 'ENOENT') debug('Error removing collectd profile', safe.error);
+    }
 
-        shell.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}, function (error) {
-            if (error) return callback(new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config'));
-
-            callback(null);
-        });
-    });
+    const [error] = await safe(shell.promises.sudo('removeCollectdProfile', [ CONFIGURE_COLLECTD_CMD, 'remove', name ], {}));
+    if (error) throw new BoxError(BoxError.COLLECTD_ERROR, 'Could not remove collectd config');
 }

src/constants.js

@@ -22,7 +22,7 @@ exports = module.exports = {
         'admins', 'users'         // ldap code uses 'users' pseudo group
     ],
 
-    ADMIN_LOCATION: 'my',
+    DASHBOARD_LOCATION: 'my',
 
     PORT: CLOUDRON ? 3000 : 5454,
     INTERNAL_SMTP_PORT: 2525, // this value comes from the mail container
@@ -32,7 +32,8 @@ exports = module.exports = {
 
     NGINX_DEFAULT_CONFIG_FILE_NAME: 'default.conf',
 
-    DEFAULT_TOKEN_EXPIRATION: 365 * 24 * 60 * 60 * 1000, // 1 year
+    DEFAULT_TOKEN_EXPIRATION_MSECS: 365 * 24 * 60 * 60 * 1000, // 1 year
+    DEFAULT_TOKEN_EXPIRATION_DAYS: 365,
 
     DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024), // see also client.js
 
@@ -48,6 +49,11 @@ exports = module.exports = {
 
     AUTOUPDATE_PATTERN_NEVER: 'never',
 
+    // the db field is a blob so we make this explicit
+    AVATAR_NONE: Buffer.from('', 'utf8'),
+    AVATAR_GRAVATAR: Buffer.from('gravatar', 'utf8'),
+    AVATAR_CUSTOM: Buffer.from('custom', 'utf8'),   // this is not used here just for reference. The field will contain a byte buffer instead of the type string
+
     SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8), // also used in dashboard client.js
 
     CLOUDRON: CLOUDRON,
@@ -57,6 +63,6 @@ exports = module.exports = {
 
     FOOTER: '&copy; %YEAR%  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
 
-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '6.0.1-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '6.3.0-test'
 };
 

src/crashnotifier.js

@@ -1,11 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    sendFailureLogs: sendFailureLogs
+    sendFailureLogs
 };
 
-var assert = require('assert'),
-    auditSource = require('./auditsource.js'),
+const assert = require('assert'),
+    AuditSource = require('./auditsource.js'),
+    child_process = require('child_process'),
     eventlog = require('./eventlog.js'),
     safe = require('safetydance'),
     path = require('path'),
@@ -17,44 +18,36 @@ const COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
 const CRASH_LOG_TIMESTAMP_OFFSET = 1000 * 60 * 60; // 60 min
 const CRASH_LOG_TIMESTAMP_FILE = '/tmp/crashlog.timestamp';
 
-function collectLogs(unitName, callback) {
+async function collectLogs(unitName) {
     assert.strictEqual(typeof unitName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
-    if (!logs) return callback(safe.error);
-
-    callback(null, logs);
+    const logs = child_process.execSync(`sudo ${COLLECT_LOGS_CMD} ${unitName}`, { encoding: 'utf8' });
+    return logs;
 }
 
-function sendFailureLogs(unitName, callback) {
+async function sendFailureLogs(unitName) {
     assert.strictEqual(typeof unitName, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // check if we already sent a mail in the last CRASH_LOG_TIME_OFFSET window
     const timestamp = safe.fs.readFileSync(CRASH_LOG_TIMESTAMP_FILE, 'utf8');
     if (timestamp && (parseInt(timestamp) + CRASH_LOG_TIMESTAMP_OFFSET) > Date.now()) {
         console.log('Crash log already sent within window');
-        return callback();
+        return;
     }
 
-    collectLogs(unitName, function (error, logs) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            logs = util.format('Failed to collect logs.', error);
-        }
+    let [error, logs] = await safe(collectLogs(unitName));
+    if (error) {
+        console.error('Failed to collect logs.', error);
+        logs = util.format('Failed to collect logs.', error);
+    }
 
-        const crashId = `${new Date().toISOString()}`;
-        console.log(`Creating crash log for ${unitName} with id ${crashId}`);
+    const crashId = `${new Date().toISOString()}`;
+    console.log(`Creating crash log for ${unitName} with id ${crashId}`);
 
-        if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, `${crashId}.log`), logs)) console.log(`Failed to stash logs to ${crashId}.log:`, safe.error);
+    if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, `${crashId}.log`), logs)) console.log(`Failed to stash logs to ${crashId}.log:`, safe.error);
 
-        eventlog.add(eventlog.ACTION_PROCESS_CRASH, auditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }, function (error) {
-            if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);
+    [error] = await safe(eventlog.add(eventlog.ACTION_PROCESS_CRASH, AuditSource.HEALTH_MONITOR, { processName: unitName, crashId: crashId }));
+    if (error) console.log(`Error sending crashlog. Logs stashed at ${crashId}.log`);
 
-            safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
-
-            callback();
-        });
-    });
+    safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
 }

src/cron.js

@@ -16,11 +16,10 @@ exports = module.exports = {
     DEFAULT_AUTOUPDATE_PATTERN,
 };
 
-var appHealthMonitor = require('./apphealthmonitor.js'),
+const appHealthMonitor = require('./apphealthmonitor.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
-    async = require('async'),
-    auditSource = require('./auditsource.js'),
+    AuditSource = require('./auditsource.js'),
     backups = require('./backups.js'),
     cloudron = require('./cloudron.js'),
     constants = require('./constants.js'),
@@ -29,13 +28,15 @@ var appHealthMonitor = require('./apphealthmonitor.js'),
     dyndns = require('./dyndns.js'),
     eventlog = require('./eventlog.js'),
     janitor = require('./janitor.js'),
+    safe = require('safetydance'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
     system = require('./system.js'),
     updater = require('./updater.js'),
-    updateChecker = require('./updatechecker.js');
+    updateChecker = require('./updatechecker.js'),
+    _ = require('underscore');
 
-var gJobs = {
+const gJobs = {
     autoUpdater: null,
     backup: null,
     updateChecker: null,
@@ -51,8 +52,6 @@ var gJobs = {
     appHealthMonitor: null
 };
 
-var NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 // cron format
 // Seconds: 0-59
 // Minutes: 0-59
@@ -61,87 +60,81 @@ var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 // Months: 0-11
 // Day of Week: 0-6
 
-function startJobs(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
+async function startJobs() {
     debug('startJobs: starting cron jobs');
 
     const randomTick = Math.floor(60*Math.random());
     gJobs.systemChecks = new CronJob({
         cronTime: '00 30 2 * * *', // once a day. if you change this interval, change the notification messages with correct duration
-        onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
+        onTick: async () => await safe(cloudron.runSystemChecks(), { debug }),
         start: true
     });
 
     gJobs.diskSpaceChecker = new CronJob({
         cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
-        onTick: () => system.checkDiskSpace(NOOP_CALLBACK),
+        onTick: async () => await safe(system.checkDiskSpace(), { debug }),
         start: true
     });
 
     // this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
     gJobs.updateCheckerJob = new CronJob({
         cronTime: `${randomTick} ${randomTick} 1,5,9,13,17,21,23 * * *`,
-        onTick: () => updateChecker.checkForUpdates({ automatic: true }, NOOP_CALLBACK),
+        onTick: async () => await safe(updateChecker.checkForUpdates({ automatic: true }), { debug }),
         start: true
     });
 
     gJobs.cleanupTokens = new CronJob({
         cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: janitor.cleanupTokens,
+        onTick: async () => await safe(janitor.cleanupTokens(), { debug }),
         start: true
     });
 
     gJobs.cleanupBackups = new CronJob({
         cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN,
-        onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        onTick: async () => await safe(backups.startCleanupTask(AuditSource.CRON), { debug }),
         start: true
     });
 
     gJobs.cleanupEventlog = new CronJob({
         cronTime: '00 */30 * * * *', // every 30 minutes
-        onTick: eventlog.cleanup,
+        onTick: async () => await safe(eventlog.cleanup({ creationTime: new Date(Date.now() - 60 * 60 * 24 * 10 * 1000) }), { debug }), // 10 days ago
         start: true
     });
 
     gJobs.dockerVolumeCleaner = new CronJob({
         cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: janitor.cleanupDockerVolumes,
+        onTick: async () => await safe(janitor.cleanupDockerVolumes(), { debug }),
         start: true
     });
 
     gJobs.schedulerSync = new CronJob({
         cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
-        onTick: scheduler.sync,
+        onTick: async () => await safe(scheduler.sync(), { debug }),
         start: true
     });
 
     gJobs.certificateRenew = new CronJob({
         cronTime: '00 00 */12 * * *', // every 12 hours
-        onTick: cloudron.renewCerts.bind(null, {}, auditSource.CRON, NOOP_CALLBACK),
+        onTick: async () => await safe(cloudron.renewCerts({}, AuditSource.CRON), { debug }),
         start: true
     });
 
     gJobs.appHealthMonitor = new CronJob({
         cronTime: '*/10 * * * * *', // every 10 seconds
-        onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK),
+        onTick: async () => await safe(appHealthMonitor.run(10), { debug }), // 10 is the max run time
         start: true
     });
 
-    settings.getAll(function (error, allSettings) {
-        if (error) return callback(error);
+    const allSettings = await settings.list();
 
-        const tz = allSettings[settings.TIME_ZONE_KEY];
-        backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
-        autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz);
-        dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
-
-        callback();
-    });
+    const tz = allSettings[settings.TIME_ZONE_KEY];
+    backupConfigChanged(allSettings[settings.BACKUP_CONFIG_KEY], tz);
+    autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY], tz);
+    dynamicDnsChanged(allSettings[settings.DYNAMIC_DNS_KEY]);
 }
 
 // eslint-disable-next-line no-unused-vars
-function handleSettingsChanged(key, value) {
+async function handleSettingsChanged(key, value) {
     assert.strictEqual(typeof key, 'string');
     // value is a variant
 
@@ -151,10 +144,8 @@ function handleSettingsChanged(key, value) {
     case settings.AUTOUPDATE_PATTERN_KEY:
     case settings.DYNAMIC_DNS_KEY:
         debug('handleSettingsChanged: recreating all jobs');
-        async.series([
-            stopJobs,
-            startJobs
-        ], NOOP_CALLBACK);
+        await stopJobs();
+        await startJobs();
         break;
     default:
         break;
@@ -171,7 +162,7 @@ function backupConfigChanged(value, tz) {
 
     gJobs.backup = new CronJob({
         cronTime: value.schedulePattern,
-        onTick: backups.startBackupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        onTick: async () => await safe(backups.startBackupTask(AuditSource.CRON), { debug }),
         start: true,
         timeZone: tz
     });
@@ -189,18 +180,21 @@ function autoupdatePatternChanged(pattern, tz) {
 
     gJobs.autoUpdater = new CronJob({
         cronTime: pattern,
-        onTick: function() {
+        onTick: async function() {
             const updateInfo = updateChecker.getUpdateInfo();
             // do box before app updates. for the off chance that the box logic fixes some app update logic issue
             if (updateInfo.box && !updateInfo.box.unstable) {
                 debug('Starting box autoupdate to %j', updateInfo.box);
-                updater.updateToLatest({ skipBackup: false }, auditSource.CRON, NOOP_CALLBACK);
+                const [error] = await safe(updater.updateToLatest({ skipBackup: false }, AuditSource.CRON));
+                if (error) debug(`Failed to box autoupdate: ${error.message}`);
                 return;
             }
 
-            if (updateInfo.apps && Object.keys(updateInfo.apps).length > 0) {
-                debug('Starting app update to %j', updateInfo.apps);
-                apps.autoupdateApps(updateInfo.apps, auditSource.CRON, NOOP_CALLBACK);
+            const appUpdateInfo = _.omit(updateInfo, 'box');
+            if (Object.keys(appUpdateInfo).length > 0) {
+                debug('Starting app update to %j', appUpdateInfo);
+                const [error] = await safe(apps.autoupdateApps(appUpdateInfo, AuditSource.CRON));
+                if (error) debug(`Failed to app autoupdate: ${error.message}`);
             } else {
                 debug('No app auto updates available');
             }
@@ -219,7 +213,7 @@ function dynamicDnsChanged(enabled) {
     if (enabled) {
         gJobs.dynamicDns = new CronJob({
             cronTime: '5 * * * * *',    // we only update the records if the ip has changed.
-            onTick: dyndns.sync.bind(null, auditSource.CRON, NOOP_CALLBACK),
+            onTick: async () => await safe(dyndns.sync(AuditSource.CRON), { debug }),
             start: true
         });
     } else {
@@ -228,14 +222,10 @@ function dynamicDnsChanged(enabled) {
     }
 }
 
-function stopJobs(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    for (var job in gJobs) {
+async function stopJobs() {
+    for (const job in gJobs) {
         if (!gJobs[job]) continue;
         gJobs[job].stop();
         gJobs[job] = null;
     }
-
-    callback();
 }

src/database.js

@@ -1,28 +1,29 @@
 'use strict';
 
 exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize,
-    query: query,
-    transaction: transaction,
+    initialize,
+    uninitialize,
+    query,
+    transaction,
 
-    importFromFile: importFromFile,
-    exportToFile: exportToFile,
+    importFromFile,
+    exportToFile,
 
     _clear: clear
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
     constants = require('./constants.js'),
     debug = require('debug')('box:database'),
     mysql = require('mysql'),
-    once = require('once'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
     util = require('util');
 
-var gConnectionPool = null;
+let gConnectionPool = null;
 
 const gDatabase = {
     hostname: '127.0.0.1',
@@ -32,10 +33,8 @@ const gDatabase = {
     name: 'box'
 };
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    if (gConnectionPool !== null) return callback(null);
+async function initialize() {
+    if (gConnectionPool !== null) return;
 
     if (constants.TEST) {
         // see setupTest script how the mysql-server is run
@@ -45,6 +44,8 @@ function initialize(callback) {
     // https://github.com/mysqljs/mysql#pool-options
     gConnectionPool  = mysql.createPool({
         connectionLimit: 5,
+        acquireTimeout: 60000,
+        connectTimeout: 60000,
         host: gDatabase.hostname,
         user: gDatabase.username,
         password: gDatabase.password,
@@ -64,89 +65,91 @@ function initialize(callback) {
         connection.query('USE ' + gDatabase.name);
         connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
     });
-
-    callback(null);
 }
 
-function uninitialize(callback) {
-    if (!gConnectionPool) return callback(null);
+async function uninitialize() {
+    if (!gConnectionPool) return;
 
-    gConnectionPool.end(callback);
+    gConnectionPool.end();
     gConnectionPool = null;
 }
 
-function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var cmd = util.format('mysql --host="%s" --user="%s" --password="%s" -Nse "SHOW TABLES" %s | grep -v "^migrations$" | while read table; do mysql --host="%s" --user="%s" --password="%s" -e "SET FOREIGN_KEY_CHECKS = 0; TRUNCATE TABLE $table" %s; done',
+async function clear() {
+    const cmd = util.format('mysql --host="%s" --user="%s" --password="%s" -Nse "SHOW TABLES" %s | grep -v "^migrations$" | while read table; do mysql --host="%s" --user="%s" --password="%s" -e "SET FOREIGN_KEY_CHECKS = 0; TRUNCATE TABLE $table" %s; done',
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name,
         gDatabase.hostname, gDatabase.username, gDatabase.password, gDatabase.name);
 
-    child_process.exec(cmd, callback);
+    await shell.promises.exec('clear_database', cmd);
 }
 
-function query() {
-    const args = Array.prototype.slice.call(arguments);
-    const callback = args[args.length - 1];
-    assert.strictEqual(typeof callback, 'function');
+async function query() {
+    assert.notStrictEqual(gConnectionPool, null);
 
-    if (constants.TEST && !gConnectionPool) return callback(new BoxError(BoxError.DATABASE_ERROR, 'database.js not initialized'));
+    return new Promise((resolve, reject) => {
+        let args = Array.prototype.slice.call(arguments);
 
-    gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
+        args.push(function queryCallback(error, result) {
+            if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+
+            resolve(result);
+        });
+
+        gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
+    });
 }
 
-function transaction(queries, callback) {
-    assert(util.isArray(queries));
-    assert.strictEqual(typeof callback, 'function');
+async function transaction(queries) {
+    assert(Array.isArray(queries));
 
-    callback = once(callback);
+    return new Promise((resolve, reject) => {
+        gConnectionPool.getConnection(function (error, connection) {
+            if (error) return reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
 
-    gConnectionPool.getConnection(function (error, connection) {
-        if (error) return callback(error);
+            const releaseConnection = (error) => {
+                connection.release();
+                reject(new BoxError(BoxError.DATABASE_ERROR, error, { code: error.code, sqlMessage: error.sqlMessage }));
+            };
 
-        const releaseConnection = (error) => { connection.release(); callback(error); };
+            connection.beginTransaction(function (error) {
+                if (error) return releaseConnection(error);
 
-        connection.beginTransaction(function (error) {
-            if (error) return releaseConnection(error);
-
-            async.mapSeries(queries, function iterator(query, done) {
-                connection.query(query.query, query.args, done);
-            }, function seriesDone(error, results) {
-                if (error) return connection.rollback(() => releaseConnection(error));
-
-                connection.commit(function (error) {
+                async.mapSeries(queries, function iterator(query, done) {
+                    connection.query(query.query, query.args, done);
+                }, function seriesDone(error, results) {
                     if (error) return connection.rollback(() => releaseConnection(error));
 
-                    connection.release();
+                    connection.commit(function (error) {
+                        if (error) return connection.rollback(() => releaseConnection(error));
 
-                    callback(null, results);
+                        connection.release();
+
+                        resolve(results);
+                    });
                 });
             });
         });
     });
 }
 
-function importFromFile(file, callback) {
+async function importFromFile(file) {
     assert.strictEqual(typeof file, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
-    var cmd = `/usr/bin/mysql -h "${gDatabase.hostname}" -u ${gDatabase.username} -p${gDatabase.password} ${gDatabase.name} < ${file}`;
+    const cmd = `/usr/bin/mysql -h "${gDatabase.hostname}" -u ${gDatabase.username} -p${gDatabase.password} ${gDatabase.name} < ${file}`;
 
-    async.series([
-        query.bind(null, 'CREATE DATABASE IF NOT EXISTS box'),
-        child_process.exec.bind(null, cmd)
-    ], callback);
+    await query('CREATE DATABASE IF NOT EXISTS box');
+    const [error] = await safe(shell.promises.exec('importFromFile', cmd));
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 }
 
-function exportToFile(file, callback) {
+async function exportToFile(file) {
     assert.strictEqual(typeof file, 'string');
-    assert.strictEqual(typeof callback, 'function');
 
     // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
     // this option must not be set in production cloudrons which still use the old mysqldump
-    const disableColStats = (constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
+    const colStats = (!constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
 
-    var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${disableColStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
+    const cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${colStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
 
-    child_process.exec(cmd, callback);
+    const [error] = await safe(shell.promises.exec('exportToFile', cmd));
+    if (error) throw new BoxError(BoxError.DATABASE_ERROR, error);
 }

src/datalayout.js

@@ -1,6 +1,6 @@
 'use strict';
 
-let assert = require('assert'),
+const assert = require('assert'),
     path = require('path');
 
 class DataLayout {

src/dns.js

@@ -0,0 +1,335 @@
+'use strict';
+
+module.exports = exports = {
+    fqdn,
+    getName,
+
+    getDnsRecords,
+    upsertDnsRecords,
+    removeDnsRecords,
+
+    waitForDnsRecord,
+
+    validateHostname,
+
+    makeWildcard,
+
+    registerLocations,
+    unregisterLocations,
+
+    checkDnsRecords,
+    syncDnsRecords,
+
+    resolve,
+
+    promises: {
+        resolve: require('util').promisify(resolve)
+    }
+};
+
+const apps = require('./apps.js'),
+    assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:dns'),
+    dns = require('dns'),
+    domains = require('./domains.js'),
+    mail = require('./mail.js'),
+    promiseRetry = require('./promise-retry.js'),
+    safe = require('safetydance'),
+    settings = require('./settings.js'),
+    sysinfo = require('./sysinfo.js'),
+    tld = require('tldjs'),
+    util = require('util'),
+    _ = require('underscore');
+
+// choose which subdomain backend we use for test purpose we use route53
+function api(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
+    switch (provider) {
+    case 'cloudflare': return require('./dns/cloudflare.js');
+    case 'route53': return require('./dns/route53.js');
+    case 'gcdns': return require('./dns/gcdns.js');
+    case 'digitalocean': return require('./dns/digitalocean.js');
+    case 'gandi': return require('./dns/gandi.js');
+    case 'godaddy': return require('./dns/godaddy.js');
+    case 'linode': return require('./dns/linode.js');
+    case 'vultr': return require('./dns/vultr.js');
+    case 'namecom': return require('./dns/namecom.js');
+    case 'namecheap': return require('./dns/namecheap.js');
+    case 'netcup': return require('./dns/netcup.js');
+    case 'noop': return require('./dns/noop.js');
+    case 'manual': return require('./dns/manual.js');
+    case 'wildcard': return require('./dns/wildcard.js');
+    default: return null;
+    }
+}
+
+function fqdn(location, domainObject) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domainObject, 'object');
+
+    return location + (location ? '.' : '') + domainObject.domain;
+}
+
+// Hostname validation comes from RFC 1123 (section 2.1)
+// Domain name validation comes from RFC 2181 (Name syntax)
+// https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
+// We are validating the validity of the location-fqdn as host name (and not dns name)
+function validateHostname(location, domainObject) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domainObject, 'object');
+
+    const hostname = fqdn(location, domainObject);
+
+    const RESERVED_LOCATIONS = [
+        constants.SMTP_LOCATION,
+        constants.IMAP_LOCATION
+    ];
+    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
+
+    if (hostname === settings.dashboardFqdn()) return new BoxError(BoxError.BAD_FIELD, location + ' is reserved', { field: 'location' });
+
+    // workaround https://github.com/oncletom/tld.js/issues/73
+    var tmp = hostname.replace('_', '-');
+    if (!tld.isValid(tmp)) return new BoxError(BoxError.BAD_FIELD, 'Hostname is not a valid domain name', { field: 'location' });
+
+    if (hostname.length > 253) return new BoxError(BoxError.BAD_FIELD, 'Hostname length exceeds 253 characters', { field: 'location' });
+
+    if (location) {
+        // label validation
+        if (location.split('.').some(function (p) { return p.length > 63 || p.length < 1; })) return new BoxError(BoxError.BAD_FIELD, 'Invalid subdomain length', { field: 'location' });
+        if (location.match(/^[A-Za-z0-9-.]+$/) === null) return new BoxError(BoxError.BAD_FIELD, 'Subdomain can only contain alphanumeric, hyphen and dot', { field: 'location' });
+        if (/^[-.]/.test(location)) return new BoxError(BoxError.BAD_FIELD, 'Subdomain cannot start or end with hyphen or dot', { field: 'location' });
+    }
+
+    return null;
+}
+
+// returns the 'name' that needs to be inserted into zone
+// eslint-disable-next-line no-unused-vars
+function getName(domain, location, type) {
+    const part = domain.domain.slice(0, -domain.zoneName.length - 1);
+
+    if (location === '') return part;
+
+    return part ? `${location}.${part}` : location;
+}
+
+function maybePromisify(func) {
+    if (util.types.isAsyncFunction(func)) return func;
+    return util.promisify(func);
+}
+
+async function getDnsRecords(location, domain, type) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof type, 'string');
+
+    const domainObject = await domains.get(domain);
+    return await maybePromisify(api(domainObject.provider).get)(domainObject, location, type);
+}
+
+async function checkDnsRecords(location, domain) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+
+    const values = await getDnsRecords(location, domain, 'A');
+
+    const ip = await sysinfo.getServerIp();
+
+    if (values.length === 0) return { needsOverwrite: false }; // does not exist
+    if (values[0] === ip) return { needsOverwrite: false }; // exists but in sync
+
+    return { needsOverwrite: true };
+}
+
+// note: for TXT records the values must be quoted
+async function upsertDnsRecords(location, domain, type, values) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    debug(`upsertDNSRecord: location ${location} on domain ${domain} of type ${type} with values ${JSON.stringify(values)}`);
+
+    const domainObject = await domains.get(domain);
+    await maybePromisify(api(domainObject.provider).upsert)(domainObject, location, type, values);
+}
+
+async function removeDnsRecords(location, domain, type, values) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(Array.isArray(values));
+
+    debug('removeDNSRecord: %s on %s type %s values', location, domain, type, values);
+
+    const domainObject = await domains.get(domain);
+    const [error] = await safe(maybePromisify(api(domainObject.provider).del)(domainObject, location, type, values));
+    if (error && error.reason !== BoxError.NOT_FOUND) throw error;
+}
+
+async function waitForDnsRecord(location, domain, type, value, options) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert(type === 'A' || type === 'TXT');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+
+    const domainObject = await domains.get(domain);
+
+    // linode DNS takes ~15mins
+    if (!options.interval) options.interval = domainObject.provider === 'linode' ? 20000 : 5000;
+
+    await maybePromisify(api(domainObject.provider).wait)(domainObject, location, type, value, options);
+}
+
+function makeWildcard(vhost) {
+    assert.strictEqual(typeof vhost, 'string');
+
+    // if the vhost is like *.example.com, this function will do nothing
+    let parts = vhost.split('.');
+    parts[0] = '*';
+    return parts.join('.');
+}
+
+async function registerLocations(locations, options, progressCallback) {
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    debug(`registerLocations: Will register ${JSON.stringify(locations)} with options ${JSON.stringify(options)}`);
+
+    const overwriteDns = options.overwriteDns || false;
+
+    const ip = await sysinfo.getServerIp();
+
+    for (const location of locations) {
+        const error = await promiseRetry({ times: 200, interval: 5000 }, async function () {
+            progressCallback({ message: `Registering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+
+            // get the current record before updating it
+            const [error, values] = await safe(getDnsRecords(location.subdomain, location.domain, 'A'));
+            if (error && error.reason === BoxError.EXTERNAL_ERROR) throw new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }); // try again
+            // give up for other errors
+            if (error && error.reason === BoxError.ACCESS_DENIED) return new BoxError(BoxError.ACCESS_DENIED, error.message, { domain: location });
+            if (error && error.reason === BoxError.NOT_FOUND) return new BoxError(BoxError.NOT_FOUND, error.message, { domain: location });
+            if (error) return new BoxError(BoxError.EXTERNAL_ERROR, error.message, location);
+
+            if (values.length !== 0 && values[0] === ip) return null; // up-to-date
+
+            // refuse to update any existing DNS record for custom domains that we did not create
+            if (values.length !== 0 && !overwriteDns) return new BoxError(BoxError.ALREADY_EXISTS, 'DNS Record already exists', { domain: location });
+
+            const [upsertError] = await safe(upsertDnsRecords(location.subdomain, location.domain, 'A', [ ip ]));
+            if (upsertError && (upsertError.reason === BoxError.BUSY || upsertError.reason === BoxError.EXTERNAL_ERROR)) {
+                progressCallback({ message: `registerSubdomains: Upsert error. Will retry. ${upsertError.message}` });
+                throw new BoxError(BoxError.EXTERNAL_ERROR, upsertError.message, { domain: location }); // try again
+            }
+
+            return upsertError ? new BoxError(BoxError.EXTERNAL_ERROR, upsertError.message, location) : null;
+        });
+
+        if (error) throw error;
+    }
+}
+
+async function unregisterLocations(locations, progressCallback) {
+    assert(Array.isArray(locations));
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const ip = await sysinfo.getServerIp();
+
+    for (const location of locations) {
+        const error = await promiseRetry({ times: 30, interval: 5000 }, async function () {
+            progressCallback({ message: `Unregistering location: ${location.subdomain ? (location.subdomain + '.') : ''}${location.domain}` });
+
+            const [error] = await safe(removeDnsRecords(location.subdomain, location.domain, 'A', [ ip ]));
+            if (error && error.reason === BoxError.NOT_FOUND) return;
+            if (error && (error.reason === BoxError.BUSY || error.reason === BoxError.EXTERNAL_ERROR)) {
+                progressCallback({ message: `Error unregistering location. Will retry. ${error.message}`});
+                throw new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }); // try again
+            }
+
+            return error ? new BoxError(BoxError.EXTERNAL_ERROR, error.message, { domain: location }) : null; // give up for other errors
+        });
+
+        if (error) throw error;
+    }
+}
+
+async function syncDnsRecords(options, progressCallback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    if (options.domain && options.type === 'mail') return await mail.setDnsRecords(options.domain);
+
+    let allDomains = await domains.list();
+
+    if (options.domain) allDomains = allDomains.filter(d => d.domain === options.domain);
+
+    const mailSubdomain = settings.mailFqdn().substr(0, settings.mailFqdn().length - settings.mailDomain().length - 1);
+
+    const allApps = await apps.list();
+
+    let progress = 1, errors = [];
+
+    // we sync by domain only to get some nice progress
+    for (const domain of allDomains) {
+        progressCallback({ percent: progress, message: `Updating DNS of ${domain.domain}`});
+        progress += Math.round(100/(1+allDomains.length));
+
+        let locations = [];
+        if (domain.domain === settings.dashboardDomain()) locations.push({ subdomain: constants.DASHBOARD_LOCATION, domain: settings.dashboardDomain() });
+        if (domain.domain === settings.mailDomain() && settings.mailFqdn() !== settings.dashboardFqdn()) locations.push({ subdomain: mailSubdomain, domain: settings.mailDomain() });
+
+        allApps.forEach(function (app) {
+            const appLocations = [{ subdomain: app.location, domain: app.domain }].concat(app.alternateDomains).concat(app.aliasDomains);
+            locations = locations.concat(appLocations.filter(al => al.domain === domain.domain));
+        });
+
+        try {
+            await registerLocations(locations, { overwriteDns: true }, progressCallback);
+            progressCallback({ message: `Updating mail DNS of ${domain.domain}`});
+            await mail.setDnsRecords(domain.domain);
+        } catch (error) {
+            errors.push({ domain: domain.domain, message: error.message });
+        }
+    }
+
+    return { errors };
+}
+
+// a note on TXT records. It doesn't have quotes ("") at the DNS level. Those quotes
+// are added for DNS server software to enclose spaces. Such quotes may also be returned
+// by the DNS REST API of some providers
+function resolve(hostname, rrtype, options, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof rrtype, 'string');
+    assert(options && typeof options === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const defaultOptions = { server: '127.0.0.1', timeout: 5000 }; // unbound runs on 127.0.0.1
+    const resolver = new dns.Resolver();
+    options = _.extend({ }, defaultOptions, options);
+
+    // Only use unbound on a Cloudron
+    if (constants.CLOUDRON) resolver.setServers([ options.server ]);
+
+    // should callback with ECANCELLED but looks like we might hit https://github.com/nodejs/node/issues/14814
+    const timerId = setTimeout(resolver.cancel.bind(resolver), options.timeout || 5000);
+
+    resolver.resolve(hostname, rrtype, function (error, result) {
+        clearTimeout(timerId);
+
+        if (error && error.code === 'ECANCELLED') error.code = 'TIMEOUT';
+
+        // result is an empty array if there was no error but there is no record. when you query a random
+        // domain, it errors with ENOTFOUND. But if you query an existing domain (A record) but with different
+        // type (CNAME) it is not an error and empty array
+        // for TXT records, result is 2d array of strings
+        callback(error, result);
+    });
+}

src/dns/cloudflare.js

@@ -1,22 +1,21 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     async = require('async'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/cloudflare'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dns = require('../dns.js'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js'),
@@ -110,12 +109,12 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     debug('upsert: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
@@ -186,7 +185,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     getZoneByName(dnsConfig, zoneName, function(error, zone) {
         if (error) return callback(error);
@@ -206,12 +205,12 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     getZoneByName(dnsConfig, zoneName, function(error, zone) {
         if (error) return callback(error);
@@ -256,7 +255,7 @@ function wait(domainObject, location, type, value, options, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     debug('wait: %s for zone %s of type %s', fqdn, zoneName, type);
 

src/dns/digitalocean.js

@@ -1,28 +1,27 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     async = require('async'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/digitalocean'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
 
-var DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com';
+const DIGITALOCEAN_ENDPOINT = 'https://api.digitalocean.com';
 
 function formatError(response) {
     return util.format('DigitalOcean DNS error [%s] %j', response.statusCode, response.body);
@@ -69,7 +68,7 @@ function getInternal(dnsConfig, zoneName, name, type, callback) {
 
                 iteratorDone();
             });
-    }, function () { return !!nextPage; }, function (error) {
+    }, function (testDone) { return testDone(null, !!nextPage); }, function (error) {
         debug('getInternal:', error, JSON.stringify(matchingRecords));
 
         if (error) return callback(error);
@@ -82,12 +81,12 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
@@ -167,7 +166,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     getInternal(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -185,12 +184,12 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     getInternal(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -230,7 +229,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/gandi.js

@@ -1,26 +1,25 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/gandi'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dns = require('../dns.js'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
 
-var GANDI_API = 'https://dns.api.gandi.net/api/v5';
+const GANDI_API = 'https://dns.api.gandi.net/api/v5';
 
 function formatError(response) {
     return util.format(`Gandi DNS error [${response.statusCode}] ${response.body.message}`);
@@ -39,12 +38,12 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -75,7 +74,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type}`);
 
@@ -98,12 +97,12 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -130,7 +129,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/gcdns.js

@@ -1,23 +1,21 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/gcdns'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dns = require('../dns.js'),
     GCDNS = require('@google-cloud/dns').DNS,
-    util = require('util'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
@@ -73,12 +71,12 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     debug('add: %s for zone %s of type %s with values %j', fqdn, zoneName, type, values);
 
@@ -120,7 +118,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
         if (error) return callback(error);
@@ -144,12 +142,12 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        fqdn = domains.fqdn(location, domainObject);
+        fqdn = dns.fqdn(location, domainObject);
 
     getZoneByName(getDnsCredentials(dnsConfig), zoneName, function (error, zone) {
         if (error) return callback(error);
@@ -183,7 +181,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/godaddy.js

@@ -1,21 +1,20 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/godaddy'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dns = require('../dns.js'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -45,12 +44,12 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -91,7 +90,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type}`);
 
@@ -118,12 +117,12 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '@';
+        name = dns.getName(domainObject, location, type) || '@';
 
     debug(`get: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -165,7 +164,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/interface.js

@@ -7,18 +7,17 @@
 // -------------------------------------------
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
-var assert = require('assert'),
-    BoxError = require('../boxerror.js'),
-    util = require('util');
+const assert = require('assert'),
+    BoxError = require('../boxerror.js');
 
 function removePrivateFields(domainObject) {
     // in-place removal of tokens and api keys with constants.SECRET_PLACEHOLDER
@@ -34,7 +33,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     // Result: none
@@ -57,7 +56,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     // Result: none

src/dns/linode.js

@@ -1,22 +1,21 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
-let async = require('async'),
+const async = require('async'),
     assert = require('assert'),
     constants = require('../constants.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/linode'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dns = require('../dns.js'),
     superagent = require('superagent'),
     util = require('util'),
     waitForDns = require('./waitfordns.js');
@@ -99,7 +98,7 @@ function getZoneRecords(dnsConfig, zoneName, name, type, callback) {
 
                     iteratorDone();
                 });
-        }, function () { return more; }, function (error) {
+        }, function (testDone) { return testDone(null, more); }, function (error) {
             debug('getZoneRecords:', error, JSON.stringify(records));
 
             if (error) return callback(error);
@@ -117,7 +116,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -135,12 +134,12 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     debug('upsert: %s for zone %s of type %s with values %j', name, zoneName, type, values);
 
@@ -217,12 +216,12 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     getZoneRecords(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -263,7 +262,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/manual.js

@@ -10,12 +10,10 @@ exports = module.exports = {
     verifyDnsConfig: verifyDnsConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/manual'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
-    util = require('util'),
+    dns = require('../dns.js'),
     waitForDns = require('./waitfordns.js');
 
 function removePrivateFields(domainObject) {
@@ -31,7 +29,7 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     debug('upsert: %s for zone %s of type %s with values %j', location, domainObject.zoneName, type, values);
@@ -52,7 +50,7 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     return callback();
@@ -66,7 +64,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/namecheap.js

@@ -1,21 +1,21 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    verifyDnsConfig: verifyDnsConfig,
-    wait: wait
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    verifyDnsConfig,
+    wait
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecheap'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dns = require('../dns.js'),
+    querystring = require('querystring'),
     safe = require('safetydance'),
     superagent = require('superagent'),
     sysinfo = require('../sysinfo.js'),
@@ -34,30 +34,25 @@ function injectPrivateFields(newConfig, currentConfig) {
     if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
-function getQuery(dnsConfig, callback) {
+async function getQuery(dnsConfig) {
     assert.strictEqual(typeof dnsConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
 
-    sysinfo.getServerIp(function (error, ip) {
-        if (error) return callback(error);
+    const ip = await sysinfo.getServerIp();
 
-        callback(null, {
-            ApiUser: dnsConfig.username,
-            ApiKey: dnsConfig.token,
-            UserName: dnsConfig.username,
-            ClientIp: ip
-        });
-    });
+    return {
+        ApiUser: dnsConfig.username,
+        ApiKey: dnsConfig.token,
+        UserName: dnsConfig.username,
+        ClientIp: ip
+    };
 }
 
-function getInternal(dnsConfig, zoneName, subdomain, type, callback) {
+function getZone(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getQuery(dnsConfig, function (error, query) {
+    util.callbackify(getQuery)(dnsConfig, function (error, query) {
         if (error) return callback(error);
 
         query.Command = 'namecheap.domains.dns.getHosts';
@@ -89,13 +84,13 @@ function getInternal(dnsConfig, zoneName, subdomain, type, callback) {
     });
 }
 
-function setInternal(dnsConfig, zoneName, hosts, callback) {
+function setZone(dnsConfig, zoneName, hosts, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert(Array.isArray(hosts));
     assert.strictEqual(typeof callback, 'function');
 
-    getQuery(dnsConfig, function (error, query) {
+    util.callbackify(getQuery)(dnsConfig, function (error, query) {
         if (error) return callback(error);
 
         query.Command = 'namecheap.domains.dns.setHosts';
@@ -116,7 +111,10 @@ function setInternal(dnsConfig, zoneName, hosts, callback) {
             }
         });
 
-        superagent.post(ENDPOINT).query(query).end(function (error, result) {
+        // namecheap recommends sending as POSTDATA with > 10 records
+        const qs = querystring.stringify(query);
+
+        superagent.post(ENDPOINT).set('Content-Type', 'application/x-www-form-urlencoded').send(qs).end(function (error, result) {
             if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
             var parser = new xml2js.Parser();
@@ -144,27 +142,27 @@ function upsert(domainObject, subdomain, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
-    subdomain = domains.getName(domainObject, subdomain, type) || '@';
+    subdomain = dns.getName(domainObject, subdomain, type) || '@';
 
     debug('upsert: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+    getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
 
         // Array to keep track of records that need to be inserted
         let toInsert = [];
 
-        for (var i = 0; i < values.length; i++) {
+        for (let i = 0; i < values.length; i++) {
             let curValue = values[i];
             let wasUpdate = false;
 
-            for (var j = 0; j < result.length; j++) {
+            for (let j = 0; j < result.length; j++) {
                 let curHost = result[j];
 
                 if (curHost.Type === type && curHost.Name === subdomain) {
@@ -198,9 +196,9 @@ function upsert(domainObject, subdomain, type, values, callback) {
             }
         }
 
-        let toUpsert = result.concat(toInsert);
+        const hosts = result.concat(toInsert);
 
-        setInternal(dnsConfig, zoneName, toUpsert, callback);
+        setZone(dnsConfig, zoneName, hosts, callback);
     });
 }
 
@@ -213,18 +211,18 @@ function get(domainObject, subdomain, type, callback) {
     const dnsConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
-    subdomain = domains.getName(domainObject, subdomain, type) || '@';
+    subdomain = dns.getName(domainObject, subdomain, type) || '@';
 
-    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+    getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
 
         // We need to filter hosts to ones with this subdomain and type
-        let actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
+        const actualHosts = result.filter((host) => host.Type === type && host.Name === subdomain);
 
         // We only return the value string
-        var tmp = actualHosts.map(function (record) { return record.Address; });
+        const tmp = actualHosts.map(function (record) { return record.Address; });
 
-        debug('get: %j', tmp);
+        debug(`get: subdomain: ${subdomain} type:${type} value:${JSON.stringify(tmp)}`);
 
         return callback(null, tmp);
     });
@@ -234,39 +232,29 @@ function del(domainObject, subdomain, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config;
     const zoneName = domainObject.zoneName;
 
-    subdomain = domains.getName(domainObject, subdomain, type) || '@';
+    subdomain = dns.getName(domainObject, subdomain, type) || '@';
 
     debug('del: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getInternal(dnsConfig, zoneName, subdomain, type, function (error, result) {
+    getZone(dnsConfig, zoneName, function (error, result) {
         if (error) return callback(error);
 
         if (result.length === 0) return callback();
+        const originalLength = result.length;
 
-        let removed = false;
-
-        for (var i = 0; i < values.length; i++) {
+        for (let i = 0; i < values.length; i++) {
             let curValue = values[i];
 
-            for (var j = 0; j < result.length; j++) {
-                let curHost = result[i];
-
-                if (curHost.Type === type && curHost.Name === subdomain && curHost.Address === curValue) {
-                    removed = true;
-
-                    result.splice(i, 1); // Remove element from result array
-                }
-            }
+            result = result.filter(curHost => curHost.Type !== type || curHost.Name !== subdomain || curHost.Address !== curValue);
         }
 
-        // Only set hosts if we actually removed a host
-        if (removed) return setInternal(dnsConfig, zoneName, result, callback);
+        if (result.length !== originalLength) return setZone(dnsConfig, zoneName, result, callback);
 
         callback();
     });
@@ -325,7 +313,7 @@ function wait(domainObject, subdomain, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(subdomain, domainObject);
+    const fqdn = dns.fqdn(subdomain, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }

src/dns/namecom.js

@@ -1,24 +1,22 @@
 'use strict';
 
 exports = module.exports = {
-    removePrivateFields: removePrivateFields,
-    injectPrivateFields: injectPrivateFields,
-    upsert: upsert,
-    get: get,
-    del: del,
-    wait: wait,
-    verifyDnsConfig: verifyDnsConfig
+    removePrivateFields,
+    injectPrivateFields,
+    upsert,
+    get,
+    del,
+    wait,
+    verifyDnsConfig
 };
 
-var assert = require('assert'),
+const assert = require('assert'),
     BoxError = require('../boxerror.js'),
     constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecom'),
-    dns = require('../native-dns.js'),
-    domains = require('../domains.js'),
+    dns = require('../dns.js'),
     safe = require('safetydance'),
     superagent = require('superagent'),
-    util = require('util'),
     waitForDns = require('./waitfordns.js');
 
 const NAMECOM_API = 'https://api.name.com/v4';
@@ -157,12 +155,12 @@ function upsert(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     debug(`upsert: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -183,7 +181,7 @@ function get(domainObject, location, type, callback) {
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     getInternal(dnsConfig, zoneName, name, type, function (error, result) {
         if (error) return callback(error);
@@ -200,12 +198,12 @@ function del(domainObject, location, type, values, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof type, 'string');
-    assert(util.isArray(values));
+    assert(Array.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName,
-        name = domains.getName(domainObject, location, type) || '';
+        name = dns.getName(domainObject, location, type) || '';
 
     debug(`del: ${name} in zone ${zoneName} of type ${type} with values ${JSON.stringify(values)}`);
 
@@ -235,7 +233,7 @@ function wait(domainObject, location, type, value, options, callback) {
     assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
     assert.strictEqual(typeof callback, 'function');
 
-    const fqdn = domains.fqdn(location, domainObject);
+    const fqdn = dns.fqdn(location, domainObject);
 
     waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
 }