do not reconfigure apps when infra version has not changed

this is in preparation of incremental infra updates. in that case,
an equal infra version is not enough (the images could have changed).
infra version will only signify if containers need to be recreated
wholesum.

.jshintrc

@@ -1,7 +1,8 @@
 {
-	"node": true,
-	"browser": true,
-	"unused": true,
-	"globalstrict": true,
-	"predef": [ "angular", "$" ]
+    "node": true,
+    "browser": true,
+    "unused": true,
+    "globalstrict": true,
+    "predef": [ "angular", "$" ],
+    "esnext": true
 }

CHANGES

@@ -458,3 +458,128 @@
 - Allow users to choose a username on first sign up
 - Fix app graphs
 
+[0.12.0]
+- Fix upload of large backups
+- Postgres addon whitelists pg_trgm and hstore extensions
+- Suppress boring update emails from patch releases
+- Setup bounce alerts for emails
+- Query admin's name in activation wizard
+- Admin emails are now delivered as no-reply
+- Fix crash when user attempts to set a duplicate email
+- Improved mongodb crash recovery
+
+[0.12.1]
+- Fix crash when backing up apps
+
+[0.12.2]
+- Improved error handling for addons
+
+[0.12.3]
+- LDAP: Do not set sn attribute when user has no surname
+
+[0.12.4]
+- Install app only after platform is ready
+
+[0.12.5]
+- Get alerts for app task failures
+- Fix update issue when one or more apps are in failed state
+
+[0.12.6]
+- Allow setting an alternate external domain for apps
+
+[0.12.7]
+- Fix changing password
+
+[0.13.0]
+- Upgrade to ubuntu 16.04
+- Add event log
+
+[0.13.1]
+- Make activity log viewable to admins
+- Fix geoip lookup
+
+[0.13.2]
+- Fix crash in app auto updater
+- Fix crash with empty timezone
+
+[0.13.3]
+- Enable auth in email addon
+- Add search for activity log
+- Add tutorial for first time users
+
+[0.13.4]
+- Fix mail addon restart issue
+
+[0.14.0]
+- You have mail :-)
+
+[0.14.1]
+- 2-character usernames are now allowed
+- Make cloudron CLI push/pull more robust
+
+[0.14.2]
+- Update mail addon
+
+[0.15.0]
+- [REST API](https://cloudron.io/references/api.html) is now in public beta
+- Enable Developer mode by default for new Cloudrons
+- Reverse proxy fixes for apps exposing a WebDav server
+- Allow admins to optionally set the username and displayName on user creation
+- Fix app autoupdate logic to detect if one or more in-use port bindings was removed
+
+[0.15.1]
+- Fix mail connectivity from IPv6 clients
+- Add API token management UI
+- Improved UI to enter email aliases
+
+[0.15.2]
+- Allow restoring apps from any previous backup
+
+[0.15.3]
+- Show installation progress in a tooltip
+
+[0.16.0]
+- Allow apps to be configured in configuring state
+- Improved platform architecture that allows incremental infrastructure updates
+- Implement app clone
+
+[0.16.1]
+- Fix UI layout issue in tokens page
+- Resume app tasks only when configured and platform ready
+- Allow errored apps to be reconfigured
+
+[0.16.2]
+- Fix assert when backing up apps in errored state
+- Fix bug where multiple redis installations caused an error
+
+[0.16.3]
+- Timeout in 10mins if app restore fails because of external domain CNAME setup
+
+[0.16.4]
+- Setup email aliases to only alias names for the Cloudron domain
+
+[0.16.5]
+- Allow sending email with alias as the From
+
+[0.16.6]
+- Add plan migration interface
+- Initial EC2 support
+
+[0.17.0]
+- Public beta release of Cloudron Mail Server
+- Add new DNS & Certs UI that enables easy migration to a custom domain
+- Allow sending and receiving email from alias subaddresses
+- Fix installation issue with some apps on the naked domain
+
+[0.17.1]
+- Preliminary user impersonation support
+- Fix crash in mail container when generating bounces
+
+[0.17.2]
+- Add config option to embed apps in other sites
+
+[0.17.3]
+- Incremental infrastructure update logic
+- Keep eventlogs only for a week
+- Throttle OOM mails
+

baseimage/createDOImage

@@ -10,7 +10,6 @@ readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
 export JSON="${SOURCE_DIR}/node_modules/.bin/json"
 
-provider="digitalocean"
 installer_revision=$(git rev-parse HEAD)
 box_name=""
 server_id=""
@@ -23,14 +22,13 @@ deploy_env="dev"
 [[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
 readonly GNU_GETOPT
 
-args=$(${GNU_GETOPT} -o "" -l "provider:,revision:,regions:,size:,name:,no-destroy,env:" -n "$0" -- "$@")
+args=$(${GNU_GETOPT} -o "" -l "revision:,regions:,size:,name:,no-destroy,env:" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
     --env) deploy_env="$2"; shift 2;;
     --revision) installer_revision="$2"; shift 2;;
-    --provider) provider="$2"; shift 2;;
     --name) box_name="$2"; destroy_server="no"; shift 2;;
     --no-destroy) destroy_server="no"; shift 2;;
     --) break;;
@@ -38,28 +36,23 @@ while true; do
     esac
 done
 
-echo "Creating image using ${provider}"
-if [[ "${provider}" == "digitalocean" ]]; then
-    if [[ "${deploy_env}" == "staging" ]]; then
-        assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING
-        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}"
-    elif [[ "${deploy_env}" == "dev" ]]; then
-        assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV
-        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}"
-    elif [[ "${deploy_env}" == "prod" ]]; then
-        assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD
-        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}"
-    else
-        echo "No such env ${deploy_env}."
-        exit 1
-    fi
-
-    vps="/bin/bash ${SCRIPT_DIR}/digitalocean.sh"
+echo "Creating digitalocean image"
+if [[ "${deploy_env}" == "staging" ]]; then
+    assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING
+    export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}"
+elif [[ "${deploy_env}" == "dev" ]]; then
+    assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV
+    export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}"
+elif [[ "${deploy_env}" == "prod" ]]; then
+    assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD
+    export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}"
 else
-    echo "Unknown provider : ${provider}"
+    echo "No such env ${deploy_env}."
     exit 1
 fi
 
+vps="/bin/bash ${SCRIPT_DIR}/digitalocean.sh"
+
 readonly ssh_keys="${HOME}/.ssh/id_rsa_caas_${deploy_env}"
 readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
 readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
@@ -137,8 +130,8 @@ while true; do
     sleep 30
 done
 
-echo "Copying INFRA_VERSION"
-$scp22 "${SCRIPT_DIR}/../setup/INFRA_VERSION" root@${server_ip}:.
+echo "Copying infra_version.js"
+$scp22 "${SCRIPT_DIR}/../src/infra_version.js" root@${server_ip}:.
 
 echo "Copying box source"
 cd "${SOURCE_DIR}"

baseimage/createEC2Image

@@ -0,0 +1,185 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+export JSON="${SOURCE_DIR}/node_modules/.bin/json"
+
+installer_revision=$(git rev-parse HEAD)
+instance_id=""
+server_ip=""
+destroy_server="yes"
+
+ami_id="ami-f9e30f96"
+region="eu-central-1"
+aws_credentials="baseimage"
+security_group="sg-b9a473d1"
+instance_type="t2.small"
+subnet_id="subnet-801402e9"
+key_pair_name="id_rsa_yellowtent"
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "revisio0n:,no-destroy" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --revision) installer_revision="$2"; shift 2;;
+    --no-destroy) destroy_server="no"; shift 2;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+readonly ssh_keys="${HOME}/.ssh/id_rsa_yellowtent"
+readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+readonly ssh202="ssh -p 202 -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly ssh22="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+if [[ ! -f "${ssh_keys}" ]]; then
+    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
+    exit 1
+fi
+
+function debug() {
+    echo "$@" >&2
+}
+
+function get_pretty_revision() {
+    local git_rev="$1"
+    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
+
+    echo "${sha1}"
+}
+
+now=$(date "+%Y-%m-%d-%H%M%S")
+pretty_revision=$(get_pretty_revision "${installer_revision}")
+
+echo "Creating EC2 instance"
+instance_id=$(aws ec2 run-instances --image-id ${ami_id} --region ${region} --profile ${aws_credentials} --security-group-ids ${security_group} --instance-type ${instance_type} --key-name ${key_pair_name} --subnet-id ${subnet_id} --associate-public-ip-address | $JSON Instances[0].InstanceId)
+echo "Got InstanceId: ${instance_id}"
+
+# name the instance
+aws ec2 create-tags --profile ${aws_credentials} --resources ${instance_id} --tags "Key=Name,Value=baseimage-${pretty_revision}"
+
+echo "Waiting for instance to be running..."
+while true; do
+    event_status=`aws ec2 describe-instances --instance-id ${instance_id} --region ${region} --profile ${aws_credentials} | $JSON Reservations[0].Instances[0].State.Name`
+    if [[ "${event_status}" == "running" ]]; then
+        break
+    fi
+    debug -n "."
+    sleep 10
+done
+
+server_ip=$(aws ec2 describe-instances --instance-id ${instance_id} --region ${region} --profile ${aws_credentials} | $JSON Reservations[0].Instances[0].PublicIpAddress)
+echo "Server IP is: ${server_ip}"
+
+while true; do
+    echo "Trying to copy init script to server"
+    if $scp22 "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" ubuntu@${server_ip}:.; then
+        break
+    fi
+    echo "Timedout, trying again in 30 seconds"
+    sleep 30
+done
+
+echo "Copying infra_version.js"
+$scp22 "${SCRIPT_DIR}/../src/infra_version.js" ubuntu@${server_ip}:.
+
+echo "Copying box source"
+cd "${SOURCE_DIR}"
+git archive --format=tar HEAD | $ssh22 "ubuntu@${server_ip}" "cat - > /tmp/box.tar.gz"
+
+echo "Enabling root ssh access"
+if ! $ssh22 "ubuntu@${server_ip}" "sudo sed -e 's/.* \(ssh-rsa.*\)/\1/' -i /root/.ssh/authorized_keys"; then
+    echo "Unable to enable root access"
+    echo "Make sure to cleanup the ec2 instance ${instance_id}"
+    exit 1
+fi
+
+echo "Executing init script"
+if ! $ssh22 "root@${server_ip}" "/bin/bash /home/ubuntu/initializeBaseUbuntuImage.sh ${installer_revision}"; then
+    echo "Init script failed"
+    echo "Make sure to cleanup the ec2 instance ${instance_id}"
+    exit 1
+fi
+
+snapshot_name="cloudron-${pretty_revision}-${now}"
+
+echo "Creating ami image ${snapshot_name}"
+image_id=$(aws ec2 create-image --region ${region} --profile ${aws_credentials} --instance-id ${instance_id} --name ${snapshot_name} | $JSON ImageId)
+
+echo "Image creation started for image id: ${image_id}"
+
+echo "Waiting for image creation to finish..."
+while true; do
+    event_status=`aws ec2 describe-images --region ${region} --profile ${aws_credentials} --image-id ${image_id} | $JSON Images[0].State`
+    if [[ "${event_status}" == "available" ]]; then
+        break
+    fi
+    debug -n "."
+    sleep 10
+done
+
+echo "Terminating instance"
+aws ec2 terminate-instances --region ${region} --profile ${aws_credentials} --instance-ids ${instance_id}
+
+echo "Make image public"
+aws ec2 modify-image-attribute --region ${region} --profile ${aws_credentials} --image-id ${image_id} --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
+
+
+# http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
+# Images are currently created in eu-central-1
+echo "Coping image to other regions"
+ec2_regions=( "us-east-1" "us-west-1" "us-west-2" "ap-south-1" "ap-northeast-2" "ap-southeast-1" "ap-southeast-2" "ap-northeast-1" "eu-west-1" "sa-east-1" )
+ec2_amis=( )
+
+for r in ${ec2_regions[@]}; do
+    echo "=> ${r}"
+    ami_id=$(aws ec2 copy-image --region ${r} --profile ${aws_credentials} --source-image-id ${image_id} --source-region ${region} --name ${snapshot_name} | $JSON ImageId)
+
+    # append in the same order as the regions
+    ec2_amis+=( ${ami_id} )
+done
+
+# wait for all images to be available
+echo "Waiting for images to be ready (first will take the longest)..."
+region_string="${region}=${image_id}"
+i=0
+while [ $i -lt ${#ec2_regions[*]} ]; do
+    echo "=> ${ec2_regions[$i]} ${ec2_amis[$i]}"
+    while true; do
+        event_status=`aws ec2 describe-images --region ${ec2_regions[$i]} --profile ${aws_credentials} --image-id ${ec2_amis[$i]} | $JSON Images[0].State`
+        if [[ "${event_status}" == "available" ]]; then
+            echo "done"
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+
+    # now make it public
+    aws ec2 modify-image-attribute --region ${ec2_regions[$i]} --profile ${aws_credentials} --image-id ${ec2_amis[$i]} --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
+
+    # append to output string for release tool
+    region_string+=",${ec2_regions[$i]}=${ec2_amis[$i]}"
+
+    # inc the iteration counter
+    i=$(( $i + 1));
+done
+
+echo ""
+echo "--------------------------------------------------"
+echo "New image id is: ${image_id}"
+echo "Image region string for release:"
+echo "${region_string}"
+echo "--------------------------------------------------"
+echo ""

baseimage/digitalocean.sh

@@ -10,7 +10,7 @@ if [[ -z "${JSON}" ]]; then
     exit 1
 fi
 
-readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:"
+readonly CURL="curl --retry 5 -s -u ${DIGITAL_OCEAN_TOKEN}:"
 
 function debug() {
     echo "$@" >&2
@@ -30,7 +30,7 @@ function create_droplet() {
     local box_name="$2"
 
     local image_region="sfo1"
-    local ubuntu_image_slug="ubuntu-15-10-x64"
+    local ubuntu_image_slug="ubuntu-16-04-x64"
     local box_size="512mb"
 
     local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"
@@ -49,9 +49,9 @@ function get_droplet_ip() {
 
 function get_droplet_id() {
     local droplet_name="$1"
-    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=100" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
+    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=200" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
     [[ -z "$id" ]] && exit 1
-    echo "$id"  
+    echo "$id"
 }
 
 function power_off_droplet() {
@@ -109,13 +109,24 @@ function get_image_id() {
     local snapshot_name="$1"
     local image_id=""
 
-    image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \
-       | $JSON images \
-       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id)
-
-    if [[ -n "${image_id}" ]]; then
-        echo "${image_id}"
+    if ! response=$($CURL "https://api.digitalocean.com/v2/images?per_page=200"); then
+        echo "Failed to get image listing. ${response}"
+        return 1
     fi
+
+    if ! image_id=$(echo "$response" \
+       | $JSON images \
+       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id); then
+        echo "Failed to parse curl response: ${response}"
+        return 1
+    fi
+
+    if [[ -z "${image_id}" ]]; then
+        echo "Failed to get image id of ${snapshot_name}. reponse: ${response}"
+        return 1
+    fi
+
+    echo "${image_id}"
 }
 
 function snapshot_droplet() {
@@ -128,16 +139,26 @@ function snapshot_droplet() {
     debug -n "Waiting for snapshot to complete"
 
     while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if ! response=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}"); then
+            echo "Could not get action status. ${response}"
+            continue
+        fi
+        if ! event_status=$(echo "${response}" | $JSON action.status); then
+            echo "Could not parse action.status from response. ${response}"
+            continue
+        fi
         if [[ "${event_status}" == "completed" ]]; then
             break
         fi
         debug -n "."
         sleep 10
     done
-    debug ""
+    debug "! done"
 
-    get_image_id "${snapshot_name}"
+    if ! image_id=$(get_image_id "${snapshot_name}"); then
+        return 1
+    fi
+    echo "${image_id}"
 }
 
 function destroy_droplet() {
@@ -177,7 +198,7 @@ function transfer_image_to_all_regions() {
     local image_id="$1"
 
     xfer_events=()
-    image_regions=(ams3) ## sfo1 is where the image is created
+    image_regions=(ams2) ## sfo1 is where the image is created
     for image_region in ${image_regions[@]}; do
         xfer_event=$(transfer_image ${image_id} ${image_region})
         echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}"

baseimage/initializeBaseUbuntuImage.sh

@@ -6,7 +6,6 @@ readonly USER=yellowtent
 readonly USER_HOME="/home/${USER}"
 readonly INSTALLER_SOURCE_DIR="${USER_HOME}/installer"
 readonly INSTALLER_REVISION="$1"
-readonly SELFHOSTED=$(( $# > 1 ? 1 : 0 ))
 readonly USER_DATA_FILE="/root/user_data.img"
 readonly USER_DATA_DIR="/home/yellowtent/data"
 
@@ -17,19 +16,7 @@ function die {
     exit 1
 }
 
-[[ "$(systemd --version 2>&1)" == *"systemd 225"* ]] || die "Expecting systemd to be 225"
-
-if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
-    source "${SOURCE_DIR}/INFRA_VERSION"
-else
-    echo "No INFRA_VERSION found, skip pulling docker images"
-fi
-
-if [ ${SELFHOSTED} == 0 ]; then
-    echo "!! Initializing Ubuntu image for CaaS"
-else
-    echo "!! Initializing Ubuntu image for Selfhosting"
-fi
+[[ "$(systemd --version 2>&1)" == *"systemd 229"* ]] || die "Expecting systemd to be 229"
 
 echo "==== Create User ${USER} ===="
 if ! id "${USER}"; then
@@ -66,15 +53,11 @@ iptables -P OUTPUT ACCEPT
 # NOTE: keep these in sync with src/apps.js validatePortBindings
 # allow ssh, http, https, ping, dns
 iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-if [ ${SELFHOSTED} == 0 ]; then
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,202,443,886 -j ACCEPT
-else
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,22,443,886 -j ACCEPT
-fi
+iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,202,443,587,993,4190 -j ACCEPT
 iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
 iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
 iptables -A INPUT -p udp --sport 53 -j ACCEPT
-iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+iptables -A INPUT -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
 
 # loopback
 iptables -A INPUT -i lo -j ACCEPT
@@ -156,34 +139,26 @@ update-grub
 # now add the user to the docker group
 usermod "${USER}" -a -G docker
 
-if [ -z $(echo "${INFRA_VERSION}") ]; then
-    echo "Skip pulling base docker images"
-else
-    echo "=== Pulling base docker images ==="
-    docker pull "${BASE_IMAGE}"
+echo "==== Install nodejs ===="
+# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
+mkdir -p /usr/local/node-4.1.1
+curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
+ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
+ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
+apt-get install -y python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
-    echo "=== Pulling mysql addon image ==="
-    docker pull "${MYSQL_IMAGE}"
+echo "==== Downloading docker images ===="
+images=$(node -e "var i = require('${SOURCE_DIR}/infra_version.js'); console.log(i.baseImage, Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
-    echo "=== Pulling postgresql addon image ==="
-    docker pull "${POSTGRESQL_IMAGE}"
-
-    echo "=== Pulling redis addon image ==="
-    docker pull "${REDIS_IMAGE}"
-
-    echo "=== Pulling mongodb addon image ==="
-    docker pull "${MONGODB_IMAGE}"
-
-    echo "=== Pulling graphite docker images ==="
-    docker pull "${GRAPHITE_IMAGE}"
-
-    echo "=== Pulling mail relay ==="
-    docker pull "${MAIL_IMAGE}"
-fi
+echo "Pulling images: ${images}"
+for image in ${images}; do
+    docker pull "${image}"
+done
 
 echo "==== Install nginx ===="
 apt-get -y install nginx-full
-[[ "$(nginx -v 2>&1)" == *"nginx/1.9."* ]] || die "Expecting nginx version to be 1.9.x"
+[[ "$(nginx -v 2>&1)" == *"nginx/1.10."* ]] || die "Expecting nginx version to be 1.10.x"
 
 echo "==== Install build-essential ===="
 apt-get -y install build-essential rcconf
@@ -191,11 +166,11 @@ apt-get -y install build-essential rcconf
 echo "==== Install mysql ===="
 debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
 debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
-apt-get -y install mysql-server
-[[ "$(mysqld --version 2>&1)" == *"5.6."* ]] || die "Expecting nginx version to be 5.6.x"
+apt-get -y install mysql-server-5.7
+[[ "$(mysqld --version 2>&1)" == *"5.7."* ]] || die "Expecting mysql version to be 5.7.x"
 
-echo "==== Install pwgen and swaks ===="
-apt-get -y install pwgen swaks
+echo "==== Install pwgen and swaks awscli ===="
+apt-get -y install pwgen swaks awscli
 
 echo "==== Install collectd ==="
 if ! apt-get install -y collectd collectd-utils; then
@@ -210,25 +185,11 @@ echo "==== Install logrotate ==="
 apt-get install -y cron logrotate
 systemctl enable cron
 
-echo "==== Install nodejs ===="
-# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
-mkdir -p /usr/local/node-4.1.1
-curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
-ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
-ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
-apt-get install -y python   # Install python which is required for npm rebuild
-[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
-
 echo "=== Rebuilding npm packages ==="
 cd "${INSTALLER_SOURCE_DIR}" && npm install --production
 chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"
 
 echo "==== Install installer systemd script ===="
-provisionEnv="PROVISION=digitalocean"
-if [ ${SELFHOSTED} == 1 ]; then
-    provisionEnv="PROVISION=local"
-fi
-
 cat > /etc/systemd/system/cloudron-installer.service <<EOF
 [Unit]
 Description=Cloudron Installer
@@ -238,7 +199,7 @@ BindsTo=systemd-journald.service
 [Service]
 Type=idle
 ExecStart="${INSTALLER_SOURCE_DIR}/src/server.js"
-Environment="DEBUG=installer*,connect-lastmile" ${provisionEnv}
+Environment="DEBUG=installer*,connect-lastmile"
 ; kill any child (installer.sh) as well
 KillMode=control-group
 Restart=on-failure
@@ -265,12 +226,13 @@ EOF
 
 # Allocate swap files
 # https://bbs.archlinux.org/viewtopic.php?id=194792 ensures this runs after do-resize.service
+# On ubuntu ec2 we use cloud-init https://wiki.archlinux.org/index.php/Cloud-init
 echo "==== Install box-setup systemd script ===="
 cat > /etc/systemd/system/box-setup.service <<EOF
 [Unit]
 Description=Box Setup
 Before=docker.service collectd.service mysql.service
-After=do-resize.service
+After=do-resize.service cloud-init.service
 
 [Service]
 Type=oneshot
@@ -311,16 +273,14 @@ chown root:systemd-journal /var/log/journal
 systemctl restart systemd-journald
 setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
 
-if [ ${SELFHOSTED} == 0 ]; then
-    echo "==== Install ssh ==="
-    apt-get -y install openssh-server
-    # https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
-    sed -e 's/^#\?Port .*/Port 202/g' \
-        -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
-        -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
-        -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
-        -i /etc/ssh/sshd_config
+echo "==== Install ssh ==="
+apt-get -y install openssh-server
+# https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
+sed -e 's/^#\?Port .*/Port 202/g' \
+    -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
+    -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
+    -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
+    -i /etc/ssh/sshd_config
 
-    # required so we can connect to this machine since port 22 is blocked by iptables by now
-    systemctl reload sshd
-fi
+# required so we can connect to this machine since port 22 is blocked by iptables by now
+systemctl reload sshd

crashnotifier.js

@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-var assert = require('assert'),
-    mailer = require('./src/mailer.js'),
-    safe = require('safetydance'),
-    path = require('path'),
-    util = require('util');
-
-var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
-
-function collectLogs(program, callback) {
-    assert.strictEqual(typeof program, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
-    callback(null, logs);
-}
-
-function sendCrashNotification(processName) {
-    collectLogs(processName, function (error, result) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            result = util.format('Failed to collect logs.', error);
-        }
-
-        console.log('Sending crash notification email for', processName);
-        mailer.sendCrashNotification(processName, result);
-    });
-}
-
-function main() {
-    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
-
-    var processName = process.argv[2];
-    console.log('Started crash notifier for', processName);
-
-    sendCrashNotification(processName);
-}
-
-main();

crashnotifierservice.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+'use strict';
+
+var sendFailureLogs = require('./src/logcollector').sendFailureLogs;
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    sendFailureLogs(processName, { unit: processName });
+}
+
+main();

installer.sh.ejs

@@ -1,164 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-echo ""
-echo "======== Cloudron Installer ========"
-echo ""
-
-if [ $# -lt 4 ]; then
-    echo "Usage: ./installer.sh <fqdn> <aws key id> <aws key secret> <bucket> <provider> <revision>"
-    exit 1
-fi
-
-# commandline arguments
-readonly fqdn="${1}"
-readonly aws_access_key_id="${2}"
-readonly aws_access_key_secret="${3}"
-readonly aws_backup_bucket="${4}"
-readonly provider="${5}"
-readonly revision="${6}"
-
-# environment specific urls
-<% if (env === 'prod') { %>
-readonly api_server_origin="https://api.cloudron.io"
-readonly web_server_origin="https://cloudron.io"
-<% } else { %>
-readonly api_server_origin="https://api.<%= env %>.cloudron.io"
-readonly web_server_origin="https://<%= env %>.cloudron.io"
-<% } %>
-readonly release_bucket_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases"
-readonly versions_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases/versions.json"
-readonly installer_code_url="${release_bucket_url}/box-${revision}.tar.gz"
-
-# runtime consts
-readonly installer_code_file="/tmp/box.tar.gz"
-readonly installer_tmp_dir="/tmp/box"
-readonly cert_folder="/tmp/certificates"
-
-# check for fqdn in /ets/hosts
-echo "[INFO] checking for hostname entry"
-readonly hostentry_found=$(grep "${fqdn}" /etc/hosts || true)
-if [[ -z $hostentry_found ]]; then
-    echo "[WARNING] No entry for ${fqdn} found in /etc/hosts"
-    echo "Adding an entry ..."
-
-    cat >> /etc/hosts <<EOF
-
-# The following line was added by the Cloudron installer script
-127.0.1.1 ${fqdn} ${fqdn}
-EOF
-else
-    echo "Valid hostname entry found in /etc/hosts"
-fi
-echo ""
-
-echo "[INFO] ensure minimal dependencies ..."
-apt-get update
-apt-get install -y curl
-echo ""
-
-echo "[INFO] Generating certificates ..."
-rm -rf "${cert_folder}"
-mkdir -p "${cert_folder}"
-
-cat > "${cert_folder}/CONFIG" <<EOF
-[ req ]
-default_bits           = 1024
-default_keyfile        = keyfile.pem
-distinguished_name     = req_distinguished_name
-prompt                 = no
-req_extensions         = v3_req
-
-[ req_distinguished_name ]
-C                      = DE
-ST                     = Berlin
-L                      = Berlin
-O                      = Cloudron UG
-OU                     = Cloudron
-CN                     = ${fqdn}
-emailAddress           = cert@cloudron.io
-
-[ v3_req ]
-# Extensions to add to a certificate request
-basicConstraints = CA:FALSE
-keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-subjectAltName = @alt_names
-
-[alt_names]
-DNS.1 = ${fqdn}
-DNS.2 = *.${fqdn}
-EOF
-
-# generate cert files
-openssl genrsa 2048 > "${cert_folder}/host.key"
-openssl req -new -out "${cert_folder}/host.csr" -key "${cert_folder}/host.key" -config "${cert_folder}/CONFIG"
-openssl x509 -req -days 3650 -in "${cert_folder}/host.csr" -signkey "${cert_folder}/host.key" -out "${cert_folder}/host.cert" -extensions v3_req -extfile "${cert_folder}/CONFIG"
-
-# make them json compatible, by collapsing to one line
-tls_cert=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.cert")
-tls_key=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.key")
-echo ""
-
-echo "[INFO] Fetching installer code ..."
-curl "${installer_code_url}" -o "${installer_code_file}"
-echo ""
-
-echo "[INFO] Extracting installer code to ${installer_tmp_dir} ..."
-rm -rf "${installer_tmp_dir}" && mkdir -p "${installer_tmp_dir}"
-tar xvf "${installer_code_file}" -C "${installer_tmp_dir}"
-echo ""
-
-echo "Creating initial provisioning config ..."
-cat > /root/provision.json <<EOF
-{
-    "sourceTarballUrl": "",
-    "data": {
-        "apiServerOrigin": "${api_server_origin}",
-        "webServerOrigin": "${web_server_origin}",
-        "fqdn": "${fqdn}",
-        "token": "",
-        "isCustomDomain": true,
-        "boxVersionsUrl": "${versions_url}",
-        "version": "",
-        "tlsCert": "${tls_cert}",
-        "tlsKey": "${tls_key}",
-        "provider": "${provider}",
-        "backupConfig": {
-            "provider": "s3",
-            "accessKeyId": "${aws_access_key_id}",
-            "secretAccessKey": "${aws_access_key_secret}",
-            "bucket": "${aws_backup_bucket}",
-            "prefix": "backups"
-        },
-        "dnsConfig": {
-            "provider": "route53",
-            "accessKeyId": "${aws_access_key_id}",
-            "secretAccessKey": "${aws_access_key_secret}"
-        },
-        "tlsConfig": {
-            "provider": "letsencrypt-<%= env %>"
-        }
-    }
-}
-EOF
-
-echo "[INFO] Running Ubuntu initializing script ..."
-/bin/bash "${installer_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "${revision}" selfhosting
-echo ""
-
-echo "[INFO] Reloading systemd daemon ..."
-systemctl daemon-reload
-echo ""
-
-echo "[INFO] Restart docker ..."
-systemctl restart docker
-echo ""
-
-echo "[FINISHED] Now starting Cloudron init jobs ..."
-systemctl start box-setup
-
-# TODO this is only for convenience we should probably just let the user do a restart
-sleep 5 && sync
-systemctl start cloudron-installer
-journalctl -u cloudron-installer.service -f

installer/package.json

@@ -21,6 +21,7 @@
     "json": "^9.0.3",
     "morgan": "^1.5.1",
     "proxy-middleware": "^0.15.0",
+    "request": "^2.72.0",
     "safetydance": "0.0.19",
     "semver": "^5.1.0",
     "superagent": "^0.21.0"

installer/src/server.js

@@ -16,6 +16,7 @@ var assert = require('assert'),
     json = require('body-parser').json,
     lastMile = require('connect-lastmile'),
     morgan = require('morgan'),
+    request = require('request'),
     superagent = require('superagent');
 
 exports = module.exports = {
@@ -29,17 +30,42 @@ var CLOUDRON_CONFIG_FILE = '/home/yellowtent/configs/cloudron.conf';
 var gHttpServer = null; // update server; used for updates
 
 function provisionDigitalOcean(callback) {
-    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
-
     superagent.get('http://169.254.169.254/metadata/v1.json').end(function (error, result) {
         if (error || result.statusCode !== 200) {
             console.error('Error getting metadata', error);
             return callback(new Error('Error getting metadata'));
         }
 
-        var userData = JSON.parse(result.body.user_data);
+        callback(null, JSON.parse(result.body.user_data));
+    });
+}
 
-        installer.provision(userData, callback);
+function provisionEC2(callback) {
+    // need to use request, since octet-stream data
+    request('http://169.254.169.254/latest/user-data', function (error, response, body) {
+        if (error || response.statusCode !== 200) {
+            console.error('Error getting metadata', error);
+            return callback(new Error('Error getting metadata'));
+        }
+
+        callback(null, JSON.parse(body));
+    });
+}
+
+function provision(callback) {
+    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
+
+    // try first digitalocean, then ec2
+    provisionDigitalOcean(function (error, userData) {
+        if (!error) return installer.provision(userData, callback);
+
+        provisionEC2(function (error, userData) {
+            if (!error) return installer.provision(userData, callback);
+
+            console.error('Unable to get meta data', error);
+
+            callback(new Error('Error getting metadata'));
+        });
     });
 }
 
@@ -122,7 +148,7 @@ function start(callback) {
 
         actions = [
             startUpdateServer,
-            provisionDigitalOcean
+            provision
         ];
     }
 

installer/systemd/box-setup.sh

@@ -4,7 +4,6 @@ set -eu -o pipefail
 
 readonly USER_HOME="/home/yellowtent"
 readonly APPS_SWAP_FILE="/apps.swap"
-readonly BACKUP_SWAP_FILE="/backup.swap" # used when doing app backups
 readonly USER_DATA_FILE="/root/user_data.img"
 readonly USER_DATA_DIR="/home/yellowtent/data"
 
@@ -17,13 +16,15 @@ if [[ -b "/dev/xvda1" ]]; then
     disk_device="/dev/xvda1"
 fi
 
+# allow root access over ssh
+sed -e 's/.* \(ssh-rsa.*\)/\1/' -i /root/.ssh/authorized_keys
+
 # all sizes are in mb
 readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }')
 readonly swap_size="${physical_memory}" # if you change this, fix enoughResourcesAvailable() in client.js
 readonly app_count=$((${physical_memory} / 200)) # estimated app count
 readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
 readonly disk_size=$((disk_size_gb * 1024))
-readonly backup_swap_size=1024
 readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
 readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1
 
@@ -32,8 +33,7 @@ echo "Physical memory: ${physical_memory}"
 echo "Estimated app count: ${app_count}"
 echo "Disk size: ${disk_size}"
 
-# Allocate two sets of swap files - one for general app usage and another for backup
-# The backup swap is setup for swap on the fly by the backup scripts
+# Allocate swap for general app usage
 if [[ ! -f "${APPS_SWAP_FILE}" ]]; then
     echo "Creating Apps swap file of size ${swap_size}M"
     fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}"
@@ -45,17 +45,8 @@ else
     echo "Apps Swap file already exists"
 fi
 
-if [[ ! -f "${BACKUP_SWAP_FILE}" ]]; then
-    echo "Creating Backup swap file of size ${backup_swap_size}M"
-    fallocate -l "${backup_swap_size}m" "${BACKUP_SWAP_FILE}"
-    chmod 600 "${BACKUP_SWAP_FILE}"
-    mkswap "${BACKUP_SWAP_FILE}"
-else
-    echo "Backups Swap file already exists"
-fi
-
 echo "Resizing data volume"
-home_data_size=$((disk_size - system_size - swap_size - backup_swap_size - ext4_reserved))
+home_data_size=$((disk_size - system_size - swap_size - ext4_reserved))
 echo "Resizing up btrfs user data to size ${home_data_size}M"
 umount "${USER_DATA_DIR}" || true
 # Do not preallocate (non-sparse). Doing so overallocates for data too much in advance and causes problems when using many apps with smaller data

migrations/20160419070047-apps-add-altDomain.js

@@ -0,0 +1,17 @@
+'use strict';
+
+var dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN altDomain VARCHAR(256)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN altDomain', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160430062446-eventlog-add-table.js

@@ -0,0 +1,24 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE eventlog(" +
+            "id VARCHAR(128) NOT NULL," +
+            "source JSON," +
+            "creationTime TIMESTAMP," +
+            "action VARCHAR(128) NOT NULL," +
+            "data JSON," +
+            "PRIMARY KEY (id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE eventlog', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160506111925-users-add-showTutorial.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN showTutorial BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN showTutorial', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160527010012-mailboxes-add-table.js

@@ -0,0 +1,24 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE mailboxes(' +
+				'name VARCHAR(128) NOT NULL,' +
+				'aliasTarget VARCHAR(128),' +
+				'creationTime TIMESTAMP,' +
+				'PRIMARY KEY (name))';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE mailboxes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160528090040-mailboxes-import-existing-users.js

@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+// imports mailbox entries for existing users
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function addUserMailboxes(done) {
+            db.all('SELECT username FROM users', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (r, next) {
+                    if (!r.username) return next();
+
+                    db.runSql('INSERT INTO mailboxes (name) VALUES (?)', [ r.username ], next);
+                }, done);
+            });
+        },
+       db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+  callback();
+};

migrations/20160614021819-apps-drop-lastBackupConfigJson.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupConfigJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160621032914-apps-alter-installationProgress.js

@@ -0,0 +1,16 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY installationProgress TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY installationProgress VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160714130142-apps-add-xFrameOptions.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN xFrameOptions VARCHAR(512)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN xFrameOptions', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -19,6 +19,7 @@ CREATE TABLE IF NOT EXISTS users(
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
     displayName VARCHAR(512) DEFAULT '',
+    showTutorial BOOLEAN DEFAULT 0,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS groups(
@@ -37,7 +38,7 @@ CREATE TABLE IF NOT EXISTS tokens(
     identifier VARCHAR(128) NOT NULL,
     clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
-    expires BIGINT NOT NULL,
+    expires BIGINT NOT NULL, // FIXME: make this a timestamp
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
@@ -53,7 +54,7 @@ CREATE TABLE IF NOT EXISTS apps(
     id VARCHAR(128) NOT NULL UNIQUE,
     appStoreId VARCHAR(128) NOT NULL,
     installationState VARCHAR(512) NOT NULL,
-    installationProgress VARCHAR(512),
+    installationProgress TEXT,
     runState VARCHAR(512),
     health VARCHAR(128),
     containerId VARCHAR(128),
@@ -61,15 +62,16 @@ CREATE TABLE IF NOT EXISTS apps(
     httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
     location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
-    accessRestrictionJson TEXT,
+    accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
     oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
     memoryLimit BIGINT DEFAULT 0,
+    altDomain VARCHAR(256),
+    xFrameOptions VARCHAR(512),
 
-    lastBackupId VARCHAR(128),
-    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+    lastBackupId VARCHAR(128), // tracks last valid backup, can be removed
 
-    oldConfigJson TEXT, // used to pass old config for apptask
+    oldConfigJson TEXT, // used to pass old config for apptask, can be removed when we use a queue
 
     PRIMARY KEY(id));
 
@@ -84,7 +86,7 @@ CREATE TABLE IF NOT EXISTS authcodes(
     authCode VARCHAR(128) NOT NULL UNIQUE,
     userId VARCHAR(128) NOT NULL,
     clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL,
+    expiresAt BIGINT NOT NULL, // ## FIXME: make this a timestamp
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
@@ -99,7 +101,7 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
     FOREIGN KEY(appId) REFERENCES apps(id));
 
 CREATE TABLE IF NOT EXISTS backups(
-    id VARCHAR(128) NOT NULL,
+    filename VARCHAR(128) NOT NULL,
     creationTime TIMESTAMP,
     version VARCHAR(128) NOT NULL, /* app version or box version */
     type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
@@ -107,3 +109,23 @@ CREATE TABLE IF NOT EXISTS backups(
     state VARCHAR(16) NOT NULL,
 
     PRIMARY KEY (filename));
+
+CREATE TABLE IF NOT EXISTS eventlog(
+    id VARCHAR(128) NOT NULL,
+    action VARCHAR(128) NOT NULL,
+    source JSON, /* { userId, username, ip }. userId can be null for cron,sysadmin */
+    data JSON, /* free flowing json based on action */
+    creationTime TIMESTAMP, /* FIXME: precision must be TIMESTAMP(2) */
+
+    PRIMARY KEY (id));
+
+/* Future fields:
+   * accessRestriction - to determine who can access it. So this has foreign keys
+   * quota - per mailbox quota
+*/
+CREATE TABLE IF NOT EXISTS mailboxes(
+    name VARCHAR(128) NOT NULL,
+    aliasTarget VARCHAR(128), /* the target name type is an alias */
+    creationTime TIMESTAMP,
+
+    PRIMARY KEY (id));

package.json

@@ -14,13 +14,11 @@
   ],
   "dependencies": {
     "async": "^1.2.1",
-    "attempt": "^1.0.1",
     "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "bytes": "^2.1.0",
-    "cloudron-manifestformat": "^2.3.0",
+    "cloudron-manifestformat": "^2.4.0",
     "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.13",
+    "connect-lastmile": "^0.1.0",
     "connect-timeout": "^1.5.0",
     "cookie-parser": "^1.3.5",
     "cookie-session": "^1.1.0",
@@ -28,17 +26,17 @@
     "csurf": "^1.6.6",
     "db-migrate": "^0.9.2",
     "debug": "^2.2.0",
-    "dockerode": "^2.2.2",
+    "dockerode": "^2.2.10",
     "ejs": "^2.2.4",
-    "ejs-cli": "^1.0.1",
+    "ejs-cli": "^1.2.0",
     "express": "^4.12.4",
     "express-session": "^1.11.3",
     "hat": "0.0.3",
+    "ini": "^1.3.4",
     "json": "^9.0.3",
     "ldapjs": "^0.7.1",
-    "memorystream": "^0.3.0",
     "mime": "^1.3.4",
-    "morgan": "^1.6.0",
+    "morgan": "^1.7.0",
     "multiparty": "^4.1.2",
     "mysql": "^2.7.0",
     "native-dns": "^0.7.0",
@@ -58,17 +56,16 @@
     "proxy-middleware": "^0.13.0",
     "safetydance": "^0.1.1",
     "semver": "^4.3.6",
-    "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
-    "superagent": "^1.5.0",
+    "superagent": "^1.8.3",
     "supererror": "^0.7.1",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
     "tldjs": "^1.6.2",
     "underscore": "^1.7.0",
-    "ursa": "^0.9.1",
+    "ursa": "^0.9.3",
     "valid-url": "^1.0.9",
-    "validator": "^4.4.0",
-    "x509": "^0.2.2"
+    "validator": "^4.9.0",
+    "x509": "^0.2.4"
   },
   "devDependencies": {
     "apidoc": "*",
@@ -91,7 +88,6 @@
     "mocha": "*",
     "nock": "^3.4.0",
     "node-sass": "^3.0.0-alpha.0",
-    "redis": "^2.4.2",
     "request": "^2.65.0",
     "sinon": "^1.12.2",
     "yargs": "^3.15.0"

setup/INFRA_VERSION

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# If you change the infra version, be sure to put a warning
-# in the change log
-
-INFRA_VERSION=24
-
-# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
-# These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.10.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
-MONGODB_IMAGE=cloudron/mongodb:0.8.0
-REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.9.0
-GRAPHITE_IMAGE=cloudron/graphite:0.8.0
-
-MYSQL_REPO=cloudron/mysql
-POSTGRESQL_REPO=cloudron/postgresql
-MONGODB_REPO=cloudron/mongodb
-REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
-MAIL_REPO=cloudron/mail
-GRAPHITE_REPO=cloudron/graphite

setup/argparser.sh

@@ -10,7 +10,8 @@ arg_fqdn=""
 arg_is_custom_domain="false"
 arg_restore_key=""
 arg_restore_url=""
-arg_retire="false"
+arg_retire_reason=""
+arg_retire_info=""
 arg_tls_config=""
 arg_tls_cert=""
 arg_tls_key=""
@@ -23,20 +24,30 @@ arg_update_config=""
 arg_provider=""
 arg_app_bundle=""
 
-args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
+args=$(getopt -o "" -l "data:,retire-reason:,retire-info:" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
-    --retire)
-        arg_retire="true"
-        shift
+    --retire-reason)
+        arg_retire_reason="$2"
+        shift 2
+        ;;
+    --retire-info)
+        arg_retire_info="$2"
+        shift 2
         ;;
     --data)
-        # only read mandatory non-empty parameters here
-        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_is_custom_domain arg_box_versions_url arg_version <<EOF
-        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn isCustomDomain boxVersionsUrl version | tr '\n' ' ')
-EOF
+        # these params must be valid in all cases
+        arg_fqdn=$(echo "$2" | $json fqdn)
+        arg_is_custom_domain=$(echo "$2" | $json isCustomDomain)
+
+        # only update/restore have this valid (but not migrate)
+        arg_api_server_origin=$(echo "$2" | $json apiServerOrigin)
+        arg_web_server_origin=$(echo "$2" | $json webServerOrigin)
+        arg_box_versions_url=$(echo "$2" | $json boxVersionsUrl)
+        arg_version=$(echo "$2" | $json version)
+
         # read possibly empty parameters here
         arg_app_bundle=$(echo "$2" | $json appBundle)
         [[ "${arg_app_bundle}" == "" ]] && arg_app_bundle="[]"

setup/container/mysql.cnf

@@ -4,4 +4,4 @@
 # http://bugs.mysql.com/bug.php?id=68514
 [mysqld]
 performance_schema=OFF
-max_connection=50
+max_connections=50

setup/container/sudoers

@@ -25,9 +25,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh
 
-Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
-
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 

setup/container/systemd/crashnotifier@.service

@@ -7,7 +7,7 @@ StopWhenUnneeded=false
 [Service]
 Type=idle
 WorkingDirectory=/home/yellowtent/box
-ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+ExecStart="/home/yellowtent/box/crashnotifierservice.js" %I
 Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
 KillMode=process
 User=yellowtent

setup/splashpage.sh

@@ -9,8 +9,6 @@ readonly BOX_SRC_DIR="/home/yellowtent/box"
 readonly DATA_DIR="/home/yellowtent/data"
 readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
 
-source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
-
 echo "Setting up nginx update page"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
@@ -24,17 +22,24 @@ rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
 
 # create nginx config
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+readonly current_infra=$(node -e "console.log(require('${script_dir}/../src/infra_version.js').version);")
+existing_infra="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && existing_infra=$(node -e "console.log(JSON.parse(require('fs').readFileSync('${DATA_DIR}/INFRA_VERSION', 'utf8')).version);")
+if [[ "${arg_retire_reason}" != "" || "${existing_infra}" != "${current_infra}" ]]; then
+    echo "Showing progress bar on all subdomains in retired mode or infra update. retire: ${arg_retire_reason} existing: ${existing_infra} current: ${current_infra}"
     rm -f ${DATA_DIR}/nginx/applications/*
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 else
+    echo "Show progress bar only on admin domain for normal update"
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 fi
 
-echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
+if [[ "${arg_retire_reason}" == "migrate" ]]; then
+    echo "{ \"migrate\": { \"percent\": \"10\", \"message\": \"Migrating cloudron. This could take up to 15 minutes.\", \"info\": ${arg_retire_info} }, \"backup\": null, \"apiServerOrigin\": \"${arg_api_server_origin}\" }" > "${SETUP_WEBSITE_DIR}/progress.json"
+else
+    echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
+fi
 
 nginx -s reload

setup/start.sh

@@ -34,12 +34,20 @@ set_progress() {
 set_progress "1" "Create container"
 $script_dir/container.sh
 
+set_progress "5" "Adjust system settings"
+hostnamectl set-hostname "${arg_fqdn}"
+
+# ec2 instances use lots of cpu for swapping, which can be significantly reduced adjusting the swappiness
+if [[ "${arg_provider}" == 'ec2' ]]; then
+    sysctl vm.swappiness=0
+fi
+
 set_progress "10" "Ensuring directories"
 # keep these in sync with paths.js
 [[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
 mkdir -p "${DATA_DIR}/box/appicons"
 mkdir -p "${DATA_DIR}/box/certs"
-mkdir -p "${DATA_DIR}/box/mail"
+mkdir -p "${DATA_DIR}/box/mail/dkim/${arg_fqdn}"
 mkdir -p "${DATA_DIR}/box/acme" # acme keys
 mkdir -p "${DATA_DIR}/graphite"
 
@@ -107,7 +115,7 @@ if [[ -f "${DATA_DIR}/box/certs/${admin_fqdn}.cert" && -f "${DATA_DIR}/box/certs
     admin_key_file="${DATA_DIR}/box/certs/${admin_fqdn}.key"
 fi
 ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 
 mkdir -p "${DATA_DIR}/nginx/cert"
 if [[ -f "${DATA_DIR}/box/certs/host.cert" && -f "${DATA_DIR}/box/certs/host.key" ]]; then
@@ -120,11 +128,9 @@ fi
 
 set_progress "33" "Changing ownership"
 chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+chown "${USER}:${USER}" "${DATA_DIR}/INFRA_VERSION" || true
 chown "${USER}:${USER}" "${DATA_DIR}"
 
-set_progress "40" "Setting up infra"
-${script_dir}/start/setup_infra.sh "${arg_fqdn}"
-
 set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
 set -eu
@@ -138,7 +144,6 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
     "fqdn": "${arg_fqdn}",
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
-    "adminEmail": "admin@${arg_fqdn}",
     "provider": "${arg_provider}",
     "database": {
         "hostname": "localhost",
@@ -191,18 +196,20 @@ if [[ ! -z "${arg_tls_config}" ]]; then
         -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
 fi
 
-# Add webadmin oauth client
 # The domain might have changed, therefor we have to update the record
 # !!! This needs to be in sync with the webadmin, specifically login_callback.js
-echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add webadmin api cient"
+readonly ADMIN_SCOPES="cloudron,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"Settings\", \"built-in\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
 
-echo "Add localhost test oauth client"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add SDK api client"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-sdk\", \"SDK\", \"built-in\", \"secret-sdk\", \"${admin_origin}\", \"*,roleSdk\")" box
+
+echo "Add cli api client"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-cli\", \"Cloudron Tool\", \"built-in\", \"secret-cli\", \"${admin_origin}\", \"*,roleSdk\")" box
 
 set_progress "80" "Starting Cloudron"
 systemctl start cloudron.target

setup/start/nginx/appconfig.ejs

@@ -18,11 +18,15 @@ server {
     # https://bettercrypto.org/static/applied-crypto-hardening.pdf
     # https://mozilla.github.io/server-side-tls/ssl-config-generator/
     # https://cipherli.st/
+    # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
     ssl_prefer_server_ciphers on;
     ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-    ssl_ciphers 'AES128+EECDH:AES128+EDH';
+    ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
     add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
 
+    # https://developer.mozilla.org/en-US/docs/Web/HTTP/X-Frame-Options
+    add_header X-Frame-Options "<%= xFrameOptions %>";
+
     proxy_http_version 1.1;
     proxy_intercept_errors on;
     proxy_read_timeout       3500;
@@ -58,6 +62,7 @@ server {
             client_max_body_size 1m;
         }
 
+        # the read timeout is between successive reads and not the whole connection
         location ~ ^/api/v1/apps/.*/exec$ {
             proxy_pass   http://127.0.0.1:3000;
             proxy_read_timeout 30m;

setup/start/nginx/nginx.conf

@@ -24,7 +24,14 @@ http {
 
     sendfile        on;
 
-    keepalive_timeout  65;
+    # timeout for client to finish sending headers
+    client_header_timeout 30s;
+
+    # timeout for reading client request body (successive read timeout and not whole body!)
+    client_body_timeout 60s;
+
+    # keep-alive connections timeout in 65s. this is because many browsers timeout in 60 seconds
+    keepalive_timeout  65s;
 
     # HTTP server
     server {
@@ -50,22 +57,15 @@ http {
         }
     }
 
-    # We have to enable https for nginx to read in the vhost in http request
-    # and send a 404. This is a side-effect of using wildcard DNS
+    # This server handles the naked domain for custom domains.
+    # It can also be used for wildcard subdomain 404. This feature is not used by the Cloudron itself
+    # because box always sets up DNS records for app subdomains.
     server {
         listen 443 default_server;
         ssl on;
         ssl_certificate      cert/host.cert;
         ssl_certificate_key  cert/host.key;
 
-        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
-        proxy_buffer_size       128k;
-        proxy_buffers           4 256k;
-        proxy_busy_buffers_size 256k;
-
-        # Disable check to allow unlimited body sizes
-        client_max_body_size 0;
-
         error_page 404 = @fallback;
         location @fallback {
             internal;
@@ -79,6 +79,7 @@ http {
             rewrite ^/$ /nakeddomain.html break;
         }
 
+        # required for /api/v1/cloudron/avatar
         location /api/ {
             proxy_pass http://127.0.0.1:3000;
             client_max_body_size 1m;

setup/start/setup_infra.sh

@@ -1,129 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly DATA_DIR="/home/yellowtent/data"
-
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
-
-arg_fqdn="$1"
-
-# removing containers ensures containers are launched with latest config updates
-# restore code in appatask does not delete old containers
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
-    echo "Infrastructure is upto date"
-    exit 0
-fi
-
-echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
-
-existing_containers=$(docker ps -qa)
-echo "Remove containers: ${existing_containers}"
-if [[ -n "${existing_containers}" ]]; then
-    echo "${existing_containers}" | xargs docker rm -f
-fi
-
-# graphite
-graphite_container_id=$(docker run --restart=always -d --name="graphite" \
-    -m 75m \
-    --memory-swap 150m \
-    -p 127.0.0.1:2003:2003 \
-    -p 127.0.0.1:2004:2004 \
-    -p 127.0.0.1:8000:8000 \
-    -v "${DATA_DIR}/graphite:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${GRAPHITE_IMAGE}")
-echo "Graphite container id: ${graphite_container_id}"
-if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old graphite images"
-fi
-
-# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
-mail_container_id=$(docker run --restart=always -d --name="mail" \
-    -m 75m \
-    --memory-swap 150m \
-    -h "${arg_fqdn}" \
-    -e "DOMAIN_NAME=${arg_fqdn}" \
-    -v "${DATA_DIR}/box/mail:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${MAIL_IMAGE}")
-echo "Mail container id: ${mail_container_id}"
-if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mail images"
-fi
-
-# mysql
-mysql_addon_root_password=$(pwgen -1 -s)
-docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
-cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
-readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
-readonly MYSQL_ROOT_HOST='${docker0_ip}'
-EOF
-mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -m 256m \
-    --memory-swap 512m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MYSQL_IMAGE}")
-echo "MySQL container id: ${mysql_container_id}"
-if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mysql images"
-fi
-
-# postgresql
-postgresql_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
-readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
-EOF
-postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
-    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${POSTGRESQL_IMAGE}")
-echo "PostgreSQL container id: ${postgresql_container_id}"
-if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old postgresql images"
-fi
-
-# mongodb
-mongodb_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
-readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
-EOF
-mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
-    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MONGODB_IMAGE}")
-echo "Mongodb container id: ${mongodb_container_id}"
-if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mongodb images"
-fi
-
-# redis
-if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old redis images"
-fi
-
-# only touch apps in installed state. any other state is just resumed by the taskmanager
-if [[ "${infra_version}" == "none" ]]; then
-    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
-    echo "Marking installed apps for restore"
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
-else
-    # if existing infra was found, just mark apps for reconfiguration
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
-fi
-
-echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"

src/addons.js

@@ -7,7 +7,6 @@ exports = module.exports = {
     restoreAddons: restoreAddons,
 
     getEnvironment: getEnvironment,
-    getLinksSync: getLinksSync,
     getBindsSync: getBindsSync,
     getContainerNamesSync: getContainerNamesSync,
 
@@ -19,30 +18,34 @@ exports = module.exports = {
 var appdb = require('./appdb.js'),
     assert = require('assert'),
     async = require('async'),
-    child_process = require('child_process'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
-    DatabaseError = require('./databaseerror.js'),
+    ClientsError = clients.ClientsError,
     debug = require('debug')('box:addons'),
-    docker = require('./docker.js').connection,
+    docker = require('./docker.js'),
+    dockerConnection = docker.connection,
     fs = require('fs'),
     generatePassword = require('password-generator'),
     hat = require('hat'),
-    MemoryStream = require('memorystream'),
+    infra = require('./infra_version.js'),
     once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
-    spawn = child_process.spawn,
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');
 
 var NOOP = function (app, options, callback) { return callback(); };
 
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
+    email: {
+        setup: setupEmail,
+        teardown: teardownEmail,
+        backup: NOOP,
+        restore: setupEmail
+    },
     ldap: {
         setup: setupLdap,
         teardown: teardownLdap,
@@ -79,6 +82,12 @@ var KNOWN_ADDONS = {
         backup: backupPostgreSql,
         restore: restorePostgreSql
     },
+    recvmail: {
+        setup: setupRecvMail,
+        teardown: teardownRecvMail,
+        backup: NOOP,
+        restore: setupRecvMail
+    },
     redis: {
         setup: setupRedis,
         teardown: teardownRedis,
@@ -199,28 +208,6 @@ function getEnvironment(app, callback) {
     appdb.getAddonConfigByAppId(app.id, callback);
 }
 
-function getLinksSync(app, addons) {
-    assert.strictEqual(typeof app, 'object');
-    assert(!addons || typeof addons === 'object');
-
-    var links = [ ];
-
-    if (!addons) return links;
-
-    for (var addon in addons) {
-        switch (addon) {
-        case 'mysql': links.push('mysql:mysql'); break;
-        case 'postgresql': links.push('postgresql:postgresql'); break;
-        case 'sendmail': links.push('mail:mail'); break;
-        case 'redis': links.push('redis-' + app.id + ':redis-' + app.id); break;
-        case 'mongodb': links.push('mongodb:mongodb'); break;
-        default: break;
-        }
-    }
-
-    return links;
-}
-
 function getBindsSync(app, addons) {
     assert.strictEqual(typeof app, 'object');
     assert(!addons || typeof addons === 'object');
@@ -267,22 +254,18 @@ function setupOauth(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile';
 
-    debugApp(app, 'setupOauth: id:%s clientSecret:%s', id, clientSecret);
+    clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);
 
-    clientdb.delByAppIdAndType(appId, clientdb.TYPE_OAUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_OAUTH, clientSecret, redirectURI, scope, function (error) {
+        clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) {
             if (error) return callback(error);
 
             var env = [
-                'OAUTH_CLIENT_ID=' + id,
-                'OAUTH_CLIENT_SECRET=' + clientSecret,
+                'OAUTH_CLIENT_ID=' + result.id,
+                'OAUTH_CLIENT_SECRET=' + result.clientSecret,
                 'OAUTH_ORIGIN=' + config.adminOrigin()
             ];
 
@@ -300,8 +283,8 @@ function teardownOauth(app, options, callback) {
 
     debugApp(app, 'teardownOauth');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_OAUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'oauth', callback);
     });
@@ -313,23 +296,20 @@ function setupSimpleAuth(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-' + uuid.v4();
     var scope = 'profile';
 
-    debugApp(app, 'setupSimpleAuth: id:%s', id);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_SIMPLE_AUTH, '', '', scope, function (error) {
+        clients.add(appId, clients.TYPE_SIMPLE_AUTH, '', scope, function (error, result) {
             if (error) return callback(error);
 
             var env = [
-                'SIMPLE_AUTH_SERVER=172.17.0.1',
+                'SIMPLE_AUTH_SERVER=172.18.0.1',
                 'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
-                'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_CLIENT_ID=' + id
+                'SIMPLE_AUTH_URL=http://172.18.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
+                'SIMPLE_AUTH_ORIGIN=http://172.18.0.1:' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_CLIENT_ID=' + result.id
             ];
 
             debugApp(app, 'Setting simple auth addon config to %j', env);
@@ -346,26 +326,57 @@ function teardownSimpleAuth(app, options, callback) {
 
     debugApp(app, 'teardownSimpleAuth');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'simpleauth', callback);
     });
 }
 
+function setupEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation)
+    var env = [
+        'MAIL_SMTP_SERVER=mail',
+        'MAIL_SMTP_PORT=2525',
+        'MAIL_IMAP_SERVER=mail',
+        'MAIL_IMAP_PORT=9993',
+        'MAIL_SIEVE_SERVER=mail',
+        'MAIL_SIEVE_PORT=4190',
+        'MAIL_DOMAIN=' + config.fqdn()
+    ];
+
+    debugApp(app, 'Setting up Email');
+
+    appdb.setAddonConfig(app.id, 'email', env, callback);
+}
+
+function teardownEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Tearing down Email');
+
+    appdb.unsetAddonConfig(app.id, 'email', callback);
+}
+
 function setupLdap(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var env = [
-        'LDAP_SERVER=172.17.0.1',
+        'LDAP_SERVER=172.18.0.1',
         'LDAP_PORT=' + config.get('ldapPort'),
-        'LDAP_URL=ldap://172.17.0.1:' + config.get('ldapPort'),
+        'LDAP_URL=ldap://172.18.0.1:' + config.get('ldapPort'),
         'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
         'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron',
         'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron',
-        'LDAP_BIND_PASSWORD=' + hat(256) // this is ignored
+        'LDAP_BIND_PASSWORD=' + hat(8 * 128) // this is ignored
     ];
 
     debugApp(app, 'Setting up LDAP');
@@ -388,19 +399,17 @@ function setupSendMail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var username = app.location ? app.location + '-app' : 'no-reply'; // use no-reply for bare domains
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
 
-    var env = [
-        'MAIL_SMTP_SERVER=mail',
-        'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
-        'MAIL_SMTP_USERNAME=' + username,
-        'MAIL_SMTP_PASSWORD=' + hat(256), // this is ignored
-        'MAIL_DOMAIN=' + config.fqdn()
-    ];
+    var cmd = [ '/addons/mail/service.sh', 'add-send', from ];
 
-    debugApp(app, 'Setting up sendmail');
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
 
-    appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting sendmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+    });
 }
 
 function teardownSendMail(app, options, callback) {
@@ -408,9 +417,55 @@ function teardownSendMail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debugApp(app, 'Tearing down sendmail');
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
 
-    appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    var cmd = [ '/addons/mail/service.sh', 'remove-send', from ];
+
+    debugApp(app, 'Tearing down sendmail : %j', cmd);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    });
+}
+
+function setupRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up recvmail');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'add-recv', to ];
+
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
+
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting recvmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'recvmail', env, callback);
+    });
+}
+
+function teardownRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'remove-recv', to ];
+
+    debugApp(app, 'Tearing down recvmail: %j', cmd);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'recvmail', callback);
+    });
 }
 
 function setupMySql(app, options, callback) {
@@ -420,31 +475,14 @@ function setupMySql(app, options, callback) {
 
     debugApp(app, 'Setting up mysql');
 
-    var container = docker.getContainer('mysql');
     var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'add-prefix' : 'add', app.id ];
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { bufferStdout: true }, function (error, stdout) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mysql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mysql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mysql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mysql', env, callback);
     });
 }
 
@@ -453,24 +491,14 @@ function teardownMySql(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer('mysql');
     var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'remove-prefix' : 'remove', app.id ];
 
     debugApp(app, 'Tearing down mysql');
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { }, function (error) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mysql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mysql', callback);
     });
 }
 
@@ -482,15 +510,9 @@ function backupMySql(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMySql: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupMySql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ];
 
-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mysql', cmd, { stdout: output }, callback);
 }
 
 function restoreMySql(app, options, callback) {
@@ -504,17 +526,8 @@ function restoreMySql(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
         input.on('error', callback);
 
-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMySql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMySql failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ];
+        docker.execContainer('mysql', cmd, { stdin: input }, callback);
     });
 }
 
@@ -525,31 +538,14 @@ function setupPostgreSql(app, options, callback) {
 
     debugApp(app, 'Setting up postgresql');
 
-    var container = docker.getContainer('postgresql');
     var cmd = [ '/addons/postgresql/service.sh', 'add', app.id ];
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { bufferStdout: true }, function (error, stdout) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting postgresql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'postgresql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting postgresql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'postgresql', env, callback);
     });
 }
 
@@ -558,24 +554,14 @@ function teardownPostgreSql(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer('postgresql');
     var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ];
 
     debugApp(app, 'Tearing down postgresql');
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { }, function (error) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'postgresql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'postgresql', callback);
     });
 }
 
@@ -587,19 +573,13 @@ function backupPostgreSql(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'postgresql', '/addons/postgresql/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupPostgreSql: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupPostgreSql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/postgresql/service.sh', 'backup', app.id ];
 
-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('postgresql', cmd, { stdout: output }, callback);
 }
 
 function restorePostgreSql(app, options, callback) {
-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    callback = once(callback);
 
     setupPostgreSql(app, options, function (error) {
         if (error) return callback(error);
@@ -609,17 +589,9 @@ function restorePostgreSql(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
         input.on('error', callback);
 
-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'postgresql', '/addons/postgresql/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restorePostgreSql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restorePostgreSql failed with status ' + code : null);
-        });
+        var cmd = [ '/addons/postgresql/service.sh', 'restore', app.id ];
 
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        docker.execContainer('postgresql', cmd, { stdin: input }, callback);
     });
 }
 
@@ -630,31 +602,14 @@ function setupMongoDb(app, options, callback) {
 
     debugApp(app, 'Setting up mongodb');
 
-    var container = docker.getContainer('mongodb');
     var cmd = [ '/addons/mongodb/service.sh', 'add', app.id ];
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { bufferStdout: true }, function (error, stdout) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mongodb addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mongodb', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mongodb addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mongodb', env, callback);
     });
 }
 
@@ -663,24 +618,14 @@ function teardownMongoDb(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer('mongodb');
     var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ];
 
     debugApp(app, 'Tearing down mongodb');
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { }, function (error) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mongodb', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mongodb', callback);
     });
 }
 
@@ -692,15 +637,9 @@ function backupMongoDb(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mongodb', '/addons/mongodb/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMongoDb: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupMongoDb failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mongodb/service.sh', 'backup', app.id ];
 
-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mongodb', cmd, { stdout: output }, callback);
 }
 
 function restoreMongoDb(app, options, callback) {
@@ -714,53 +653,11 @@ function restoreMongoDb(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
         input.on('error', callback);
 
-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mongodb', '/addons/mongodb/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMongoDb: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMongoDb failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mongodb/service.sh', 'restore', app.id ];
+        docker.execContainer('mongodb', cmd, { stdin: input }, callback);
     });
 }
 
-
-function forwardRedisPort(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    docker.getContainer('redis-' + appId).inspect(function (error, data) {
-        if (error) return callback(new Error('Unable to inspect container:' + error));
-
-        var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
-        if (!Number.isInteger(redisPort)) return callback(new Error('Unable to get container port mapping'));
-
-        return callback(null);
-    });
-}
-
-function stopAndRemoveRedis(container, callback) {
-    function ignoreError(func) {
-        return function (callback) {
-            func(function (error) {
-                if (error) debug('stopAndRemoveRedis: Ignored error:', error);
-                callback();
-            });
-        };
-    }
-
-    // stopping redis with SIGTERM makes it commit the database to disk
-    async.series([
-        ignoreError(container.stop.bind(container, { t: 10 })),
-        ignoreError(container.wait.bind(container)),
-        ignoreError(container.remove.bind(container, { force: true, v: true }))
-    ], callback);
-}
-
 // Ensures that app's addon redis container is running. Can be called when named container already exists/running
 function setupRedis(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
@@ -777,57 +674,32 @@ function setupRedis(app, options, callback) {
 
     if (!safe.fs.mkdirSync(redisDataDir) && safe.error.code !== 'EEXIST') return callback(new Error('Error creating redis data dir:' + safe.error));
 
-    var createOptions = {
-        name: 'redis-' + app.id,
-        Hostname: 'redis-' + app.location,
-        Tty: true,
-        Image: 'cloudron/redis:0.8.0', // if you change this, fix setup/INFRA_VERSION as well
-        Cmd: null,
-        Volumes: {
-            '/tmp': {},
-            '/run': {}
-        },
-        VolumesFrom: [],
-        HostConfig: {
-            Binds: [
-                redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
-                redisDataDir + ':/var/lib/redis:rw'
-            ],
-            Memory: 1024 * 1024 * 75, // 100mb
-            MemorySwap: 1024 * 1024 * 75 * 2, // 150mb
-            PortBindings: {
-                '6379/tcp': [{ HostPort: '0', HostIp: '127.0.0.1' }]
-            },
-            ReadonlyRootfs: true,
-            RestartPolicy: {
-                'Name': 'always',
-                'MaximumRetryCount': 0
-            }
-        }
-    };
+    const tag = infra.images.redis.tag, redisName = 'redis-' + app.id;
+    const cmd = `docker run --restart=always -d --name=${redisName} \
+                --net cloudron \
+                --net-alias ${redisName} \
+                -m 100m \
+                --memory-swap 150m \
+                -v ${redisVarsFile}:/etc/redis/redis_vars.sh:ro \
+                -v ${redisDataDir}:/var/lib/redis:rw \
+                --read-only -v /tmp -v /run ${tag}`;
 
     var env = [
         'REDIS_URL=redis://redisuser:' + redisPassword + '@redis-' + app.id,
         'REDIS_PASSWORD=' + redisPassword,
-        'REDIS_HOST=redis-' + app.id,
+        'REDIS_HOST=' + redisName,
         'REDIS_PORT=6379'
     ];
 
-    var redisContainer = docker.getContainer(createOptions.name);
-    stopAndRemoveRedis(redisContainer, function () {
-        docker.createContainer(createOptions, function (error) {
-            if (error && error.statusCode !== 409) return callback(error); // if not already created
-
-            redisContainer.start(function (error) {
-                if (error && error.statusCode !== 304) return callback(error); // if not already running
-
-                appdb.setAddonConfig(app.id, 'redis', env, function (error) {
-                    if (error) return callback(error);
-
-                    forwardRedisPort(app.id, callback);
-                });
-            });
-        });
+    async.series([
+        // stop so that redis can flush itself with SIGTERM
+        shell.execSync.bind(null, 'stopRedis', `docker stop --time=10 ${redisName} 2>/dev/null || true`),
+        shell.execSync.bind(null, 'stopRedis', `docker rm --volumes ${redisName} 2>/dev/null || true`),
+        shell.execSync.bind(null, 'startRedis', cmd),
+        appdb.setAddonConfig.bind(null, app.id, 'redis', env)
+    ], function (error) {
+        if (error) debug('Error setting up redis: ', error);
+        callback(error);
     });
 }
 
@@ -836,7 +708,7 @@ function teardownRedis(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-   var container = docker.getContainer('redis-' + app.id);
+   var container = dockerConnection.getContainer('redis-' + app.id);
 
    var removeOptions = {
        force: true, // kill container if it's running
@@ -859,15 +731,7 @@ function teardownRedis(app, options, callback) {
 function backupRedis(app, options, callback) {
     debugApp(app, 'Backing up redis');
 
-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    var cmd = [ '/addons/redis/service.sh', 'backup' ]; // the redis dir is volume mounted
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'redis-' + app.id, '/addons/redis/service.sh', 'backup' ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupRedis: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupRedis failed with status ' + code : null);
-    });
-
-    cp.stdout.pipe(process.stdout);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('redis-' + app.id, cmd, { }, callback);
 }

src/appdb.js

@@ -4,7 +4,6 @@
 
 exports = module.exports = {
     get: get,
-    getBySubdomain: getBySubdomain,
     getByHttpPort: getByHttpPort,
     getByContainerId: getByContainerId,
     add: add,
@@ -27,6 +26,7 @@ exports = module.exports = {
 
     // installation codes (keep in sync in UI)
     ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
+    ISTATE_PENDING_CLONE: 'pending_clone', // clone
     ISTATE_PENDING_CONFIGURE: 'pending_configure', // config (location, port) changes and on infra update
     ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
     ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
@@ -59,7 +59,7 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.memoryLimit' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain', 'apps.xFrameOptions' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -70,10 +70,6 @@ function postProcess(result) {
     result.manifest = safe.JSON.parse(result.manifestJson);
     delete result.manifestJson;
 
-    assert(result.lastBackupConfigJson === null || typeof result.lastBackupConfigJson === 'string');
-    result.lastBackupConfig = safe.JSON.parse(result.lastBackupConfigJson);
-    delete result.lastBackupConfigJson;
-
     assert(result.oldConfigJson === null || typeof result.oldConfigJson === 'string');
     result.oldConfig = safe.JSON.parse(result.oldConfigJson);
     delete result.oldConfigJson;
@@ -96,6 +92,9 @@ function postProcess(result) {
     result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
     if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
     delete result.accessRestrictionJson;
+
+    // TODO remove later once all apps have this attribute
+    result.xFrameOptions = result.xFrameOptions || 'SAMEORIGIN';
 }
 
 function get(id, callback) {
@@ -114,22 +113,6 @@ function get(id, callback) {
     });
 }
 
-function getBySubdomain(subdomain, callback) {
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
 function getByHttpPort(httpPort, callback) {
     assert.strictEqual(typeof httpPort, 'number');
     assert.strictEqual(typeof callback, 'function');
@@ -177,26 +160,32 @@ function getAll(callback) {
     });
 }
 
-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, callback) {
+function add(id, appStoreId, manifest, location, portBindings, data, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof manifest.version, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert.strictEqual(typeof memoryLimit, 'number');
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
 
     var manifestJson = JSON.stringify(manifest);
+
+    var accessRestriction = data.accessRestriction || null;
     var accessRestrictionJson = JSON.stringify(accessRestriction);
+    var memoryLimit = data.memoryLimit || 0;
+    var altDomain = data.altDomain || null;
+    var xFrameOptions = data.xFrameOptions || '';
+    var installationState = data.installationState || exports.ISTATE_PENDING_INSTALL;
+    var lastBackupId = data.lastBackupId || null; // used when cloning
 
     var queries = [ ];
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit) VALUES (?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId ]
     });
 
     Object.keys(portBindings).forEach(function (env) {
@@ -300,9 +289,6 @@ function updateWithConstraints(id, app, constraints, callback) {
         if (p === 'manifest') {
             fields.push('manifestJson = ?');
             values.push(JSON.stringify(app[p]));
-        } else if (p === 'lastBackupConfig') {
-            fields.push('lastBackupConfigJson = ?');
-            values.push(JSON.stringify(app[p]));
         } else if (p === 'oldConfig') {
             fields.push('oldConfigJson = ?');
             values.push(JSON.stringify(app[p]));
@@ -361,14 +347,17 @@ function setInstallationCommand(appId, installationState, values, callback) {
     // uninstall is allowed in any state
     // force update is allowed in any state including pending_uninstall! (for better or worse)
     // restore is allowed from installed or error state
-    // update and configure are allowed only in installed state
+    // configure is allowed in installed state or currently configuring or in error state
+    // update and backup are allowed only in installed state
 
     if (installationState === exports.ISTATE_PENDING_UNINSTALL || installationState === exports.ISTATE_PENDING_FORCE_UPDATE) {
         updateWithConstraints(appId, values, '', callback);
     } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
         updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
-    } else if (installationState === exports.ISTATE_PENDING_UPDATE || installationState === exports.ISTATE_PENDING_CONFIGURE || installationState === exports.ISTATE_PENDING_BACKUP) {
+    } else if (installationState === exports.ISTATE_PENDING_UPDATE || installationState === exports.ISTATE_PENDING_BACKUP) {
         updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
+    } else if (installationState === exports.ISTATE_PENDING_CONFIGURE) {
+        updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "pending_configure" OR installationState = "error")', callback);
     } else {
         callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));
     }

src/apphealthmonitor.js

@@ -17,7 +17,7 @@ exports = module.exports = {
 };
 
 var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
-var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
+var UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes
 var gHealthInfo = { }; // { time, emailSent }
 var gRunTimeout = null;
 var gDockerEventStream = null;
@@ -138,13 +138,16 @@ function run() {
 
 /*
     OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:0.3.3 /bin/bash
+        docker run -ti -m 100M cloudron/base:0.8.1 /bin/bash
         apt-get update && apt-get install stress
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
 function processDockerEvents() {
     // note that for some reason, the callback is called only on the first event
     debug('Listening for docker events');
+    const OOM_MAIL_LIMIT = 60 * 60 * 1000; // 60 minutes
+    var lastOomMailTime = new Date(new Date() - OOM_MAIL_LIMIT);
+
     docker.getEvents({ filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
         if (error) return console.error(error);
 
@@ -154,15 +157,19 @@ function processDockerEvents() {
         stream.on('data', function (data) {
             var ev = JSON.parse(data);
             debug('Container ' + ev.id + ' went OOM');
-            appdb.getByContainerId(ev.id, function (error, app) {
+            appdb.getByContainerId(ev.id, function (error, app) { // this can error for addons
                 var program = error || !app.appStoreId ? ev.id : app.appStoreId;
                 var context = JSON.stringify(ev);
+                var now = new Date();
                 if (app) context = context + '\n\n' + JSON.stringify(app, null, 4) + '\n';
 
                 debug('OOM Context: %s', context);
 
                 // do not send mails for dev apps
-                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+                if ((!app || app.appStoreId !== '') && (now - lastOomMailTime > OOM_MAIL_LIMIT)) {
+                    mailer.unexpectedExit(program, context); // app can be null if it's an addon crash
+                    lastOomMailTime = now;
+                }
             });
         });
 
@@ -194,7 +201,7 @@ function stop(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clearTimeout(gRunTimeout);
-    gDockerEventStream.end();
+    if (gDockerEventStream) gDockerEventStream.end();
 
     callback();
 }

src/apptask.js

@@ -1,7 +1,5 @@
 #!/usr/bin/env node
 
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -19,7 +17,8 @@ exports = module.exports = {
     _verifyManifest: verifyManifest,
     _registerSubdomain: registerSubdomain,
     _unregisterSubdomain: unregisterSubdomain,
-    _waitForDnsPropagation: waitForDnsPropagation
+    _waitForDnsPropagation: waitForDnsPropagation,
+    _waitForAltDomainDnsPropagation: waitForAltDomainDnsPropagation
 };
 
 require('supererror')({ splatchError: true });
@@ -35,16 +34,16 @@ var addons = require('./addons.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
+    backups = require('./backups.js'),
     certificates = require('./certificates.js'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
+    ClientsError = clients.ClientsError,
     database = require('./database.js'),
-    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apptask'),
     docker = require('./docker.js'),
     ejs = require('ejs'),
     fs = require('fs'),
-    hat = require('hat'),
     manifestFormat = require('cloudron-manifestformat'),
     net = require('net'),
     nginx = require('./nginx.js'),
@@ -57,7 +56,7 @@ var addons = require('./addons.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
-    uuid = require('node-uuid'),
+    waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
 var COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
@@ -100,9 +99,7 @@ function configureNginx(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var vhost = config.appFqdn(app.location);
-
-    certificates.ensureCertificate(vhost, function (error, certFilePath, keyFilePath) {
+    certificates.ensureCertificate(app, function (error, certFilePath, keyFilePath) {
         if (error) return callback(error);
 
         nginx.configureApp(app, certFilePath, keyFilePath, callback);
@@ -164,20 +161,18 @@ function allocateOAuthProxyCredentials(app, callback) {
 
     if (!nginx.requiresOAuthProxy(app)) return callback(null);
 
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile';
 
-    clientdb.add(id, app.id, clientdb.TYPE_PROXY, clientSecret, redirectURI, scope, callback);
+    clients.add(app.id, clients.TYPE_PROXY, redirectURI, scope, callback);
 }
 
 function removeOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_PROXY, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) {
+    clients.delByAppIdAndType(app.id, clients.TYPE_PROXY, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) {
             debugApp(app, 'Error removing OAuth client id', error);
             return callback(error);
         }
@@ -231,17 +226,19 @@ function downloadIcon(app, callback) {
 
     var iconUrl = config.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';
 
-    superagent
-        .get(iconUrl)
-        .buffer(true)
-        .end(function (error, res) {
-            if (error && !error.response) return callback(new Error('Network error downloading icon:' + error.message));
-            if (res.statusCode !== 200) return callback(null); // ignore error. this can also happen for apps installed with cloudron-cli
+    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
+        superagent
+            .get(iconUrl)
+            .buffer(true)
+            .end(function (error, res) {
+                if (error && !error.response) return retryCallback(new Error('Network error downloading icon:' + error.message));
+                if (res.statusCode !== 200) return retryCallback(null); // ignore error. this can also happen for apps installed with cloudron-cli
 
-            if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return callback(new Error('Error saving icon:' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return retryCallback(new Error('Error saving icon:' + safe.error.message));
 
-            callback(null);
-    });
+                retryCallback(null);
+        });
+    }, callback);
 }
 
 function registerSubdomain(app, callback) {
@@ -318,20 +315,25 @@ function waitForDnsPropagation(app, callback) {
         return callback(null);
     }
 
-    function retry(error) {
-        debugApp(app, 'waitForDnsPropagation: ', error);
-        setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
-    }
+    async.retry({ interval: 5000, times: 120 }, function checkStatus(retryCallback) {
+        subdomains.status(app.dnsRecordId, function (error, result) {
+            if (error) return retryCallback(new Error('Failed to get dns record status : ' + error.message));
 
-    subdomains.status(app.dnsRecordId, function (error, result) {
-        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
+            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
 
-        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
+            if (result !== 'done') return retryCallback(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
 
-        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
+            retryCallback(null, result);
+        });
+    }, callback);
+}
 
-        callback(null);
-    });
+function waitForAltDomainDnsPropagation(app, callback) {
+    if (!app.altDomain) return callback(null);
+
+    // try for 10 minutes before giving up. this allows the user to "reconfigure" the app in the case where
+    // an app has an external domain and cloudron is migrated to custom domain.
+    waitForDns(app.altDomain, config.appFqdn(app.location), 'CNAME', { interval: 10000, times: 60 }, callback);
 }
 
 // updates the app object and the database
@@ -410,9 +412,12 @@ function install(app, callback) {
 
         runApp.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for External Domain CNAME setup' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app), // required when restoring and !lastBackupId
+
         updateApp.bind(null, app, { installationProgress: '95, Configure nginx' }),
         configureNginx.bind(null, app),
 
@@ -436,7 +441,7 @@ function backup(app, callback) {
 
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
-        apps.backupApp.bind(null, app, app.manifest.addons),
+        backups.backupApp.bind(null, app, app.manifest),
 
         // done!
         function (callback) {
@@ -501,7 +506,7 @@ function restore(app, callback) {
         createVolume.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }),
-        apps.restoreApp.bind(null, app, app.manifest.addons, backupId),
+        backups.restoreApp.bind(null, app, app.manifest.addons, backupId),
 
         updateApp.bind(null, app, { installationProgress: '75, Creating container' }),
         createContainer.bind(null, app),
@@ -511,9 +516,12 @@ function restore(app, callback) {
 
         runApp.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for External Domain CNAME setup' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
         updateApp.bind(null, app, { installationProgress: '95, Configuring Nginx' }),
         configureNginx.bind(null, app),
 
@@ -573,6 +581,9 @@ function configure(app, callback) {
         updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for External Domain CNAME setup' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
         updateApp.bind(null, app, { installationProgress: '90, Configuring Nginx' }),
         configureNginx.bind(null, app),
 
@@ -616,7 +627,6 @@ function update(app, callback) {
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
         deleteContainers.bind(null, app),
-        addons.teardownAddons.bind(null, app, unusedAddons),
         function deleteImageIfChanged(done) {
              if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
 
@@ -629,10 +639,13 @@ function update(app, callback) {
 
             async.series([
                 updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
-                apps.backupApp.bind(null, app, app.oldConfig.manifest.addons)
+                backups.backupApp.bind(null, app, app.oldConfig.manifest)
             ], next);
         },
 
+        // only delete unused addons after backup
+        addons.teardownAddons.bind(null, app, unusedAddons),
+
         updateApp.bind(null, app, { installationProgress: '45, Downloading icon' }),
         downloadIcon.bind(null, app),
 
@@ -767,6 +780,7 @@ function startTask(appId, callback) {
         case appdb.ISTATE_PENDING_BACKUP: return backup(app, callback);
         case appdb.ISTATE_INSTALLED: return handleRunCommand(app, callback);
         case appdb.ISTATE_PENDING_INSTALL: return install(app, callback);
+        case appdb.ISTATE_PENDING_CLONE: return restore(app, callback);
         case appdb.ISTATE_PENDING_FORCE_UPDATE: return update(app, callback);
         case appdb.ISTATE_ERROR:
             debugApp(app, 'Internal error. apptask launched with error status.');
@@ -783,6 +797,11 @@ if (require.main === module) {
 
     debug('Apptask for %s', process.argv[2]);
 
+    process.on('SIGTERM', function () {
+        debug('taskmanager sent SIGTERM since it got a new task for this app');
+        process.exit(0);
+    });
+
     initialize(function (error) {
         if (error) throw error;
 

src/auth.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -10,17 +8,16 @@ exports = module.exports = {
 var assert = require('assert'),
     BasicStrategy = require('passport-http').BasicStrategy,
     BearerStrategy = require('passport-http-bearer').Strategy,
-    clientdb = require('./clientdb'),
+    clients = require('./clients'),
     ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    ClientsError = clients.ClientsError,
     DatabaseError = require('./databaseerror'),
     debug = require('debug')('box:auth'),
     LocalStrategy = require('passport-local').Strategy,
     crypto = require('crypto'),
-    groups = require('./groups'),
     passport = require('passport'),
     tokendb = require('./tokendb'),
     user = require('./user'),
-    userdb = require('./userdb'),
     UserError = user.UserError,
     _ = require('underscore');
 
@@ -32,7 +29,7 @@ function initialize(callback) {
     });
 
     passport.deserializeUser(function(userId, callback) {
-        userdb.get(userId, function (error, result) {
+        user.get(userId, function (error, result) {
             if (error) return callback(error);
 
             var md5 = crypto.createHash('md5').update(result.email.toLowerCase()).digest('hex');
@@ -67,8 +64,8 @@ function initialize(callback) {
             debug('BasicStrategy: detected client id %s instead of username:password', username);
             // username is actually client id here
             // password is client secret
-            clientdb.get(username, function (error, client) {
-                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+            clients.get(username, function (error, client) {
+                if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
                 if (client.clientSecret != password) return callback(null, false);
                 return callback(null, client);
@@ -85,8 +82,8 @@ function initialize(callback) {
     }));
 
     passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clientdb.get(clientId, function(error, client) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+        clients.get(clientId, function(error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
             if (error) { return callback(error); }
             if (client.clientSecret != clientSecret) { return callback(null, false); }
             return callback(null, client);
@@ -101,37 +98,12 @@ function initialize(callback) {
             // scopes here can define what capabilities that token carries
             // passport put the 'info' object into req.authInfo, where we can further validate the scopes
             var info = { scope: token.scope };
-            var tokenType;
 
-            if (token.identifier.indexOf(tokendb.PREFIX_DEV) === 0) {
-                token.identifier = token.identifier.slice(tokendb.PREFIX_DEV.length);
-                tokenType = tokendb.TYPE_DEV;
-            } else if (token.identifier.indexOf(tokendb.PREFIX_APP) === 0) {
-                tokenType = tokendb.TYPE_APP;
-                return callback(null, { id: token.identifier.slice(tokendb.PREFIX_APP.length), tokenType: tokenType }, info);
-            } else if (token.identifier.indexOf(tokendb.PREFIX_USER) === 0) {
-                tokenType = tokendb.TYPE_USER;
-                token.identifier = token.identifier.slice(tokendb.PREFIX_USER.length);
-            } else {
-                // legacy tokens assuming a user access token
-                tokenType = tokendb.TYPE_USER;
-            }
-
-            userdb.get(token.identifier, function (error, user) {
+            user.get(token.identifier, function (error, user) {
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
 
-                // amend the tokenType of the token owner
-                user.tokenType = tokenType;
-
-                // amend the admin flag
-                groups.isMember(groups.ADMIN_GROUP_ID, user.id, function (error, isAdmin) {
-                    if (error) return callback(error);
-
-                    user.admin = isAdmin;
-
-                    callback(null, user, info);
-                });
+                callback(null, user, info);
             });
         });
     }));
@@ -144,4 +116,3 @@ function uninitialize(callback) {
 
     callback(null);
 }
-

src/backups.js

@@ -6,21 +6,53 @@ exports = module.exports = {
     getPaged: getPaged,
     getByAppIdPaged: getByAppIdPaged,
 
-    getBackupUrl: getBackupUrl,
-    getAppBackupUrl: getAppBackupUrl,
     getRestoreUrl: getRestoreUrl,
+    getRestoreConfig: getRestoreConfig,
 
-    copyLastBackup: copyLastBackup
+    ensureBackup: ensureBackup,
+
+    backup: backup,
+    backupApp: backupApp,
+    restoreApp: restoreApp,
+
+    backupBoxAndApps: backupBoxAndApps
 };
 
-var assert = require('assert'),
+var addons = require('./addons.js'),
+    appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
+    async = require('async'),
+    assert = require('assert'),
     backupdb = require('./backupdb.js'),
     caas = require('./storage/caas.js'),
     config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:backups'),
+    eventlog = require('./eventlog.js'),
+    locker = require('./locker.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    progress = require('./progress.js'),
     s3 = require('./storage/s3.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
     settings = require('./settings.js'),
-    util = require('util');
+    superagent = require('superagent'),
+    util = require('util'),
+    webhooks = require('./webhooks.js');
+
+var BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
+    BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
+    RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
 
 function BackupsError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
@@ -43,6 +75,7 @@ function BackupsError(reason, errorOrMessage) {
 util.inherits(BackupsError, Error);
 BackupsError.EXTERNAL_ERROR = 'external error';
 BackupsError.INTERNAL_ERROR = 'internal error';
+BackupsError.BAD_STATE = 'bad state';
 BackupsError.MISSING_CREDENTIALS = 'missing credentials';
 
 // choose which storage backend we use for test purpose we use s3
@@ -79,7 +112,7 @@ function getByAppIdPaged(page, perPage, appId, callback) {
     });
 }
 
-function getBackupUrl(appBackupIds, callback) {
+function getBoxBackupCredentials(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
     assert.strictEqual(typeof callback, 'function');
 
@@ -90,57 +123,68 @@ function getBackupUrl(appBackupIds, callback) {
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).getBackupUrl(backupConfig, filename, function (error, result) {
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
             if (error) return callback(error);
 
-            var obj = {
-                id: result.id,
-                url: result.url,
-                backupKey: backupConfig.key
-            };
+            result.id = filename;
+            result.s3Url = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + filename;
+            result.backupKey = backupConfig.key;
 
-            debug('getBackupUrl: id:%s url:%s backupKey:%s', obj.id, obj.url, obj.backupKey);
+            debug('getBoxBackupCredentials: %j', result);
 
-            backupdb.add({ id: result.id, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
-                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-                callback(null, obj);
-            });
+            callback(null, result);
         });
     });
 }
 
-function getAppBackupUrl(app, callback) {
+function getAppBackupCredentials(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var now = new Date();
-    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), app.manifest.version);
+    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), manifest.version);
     var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';
 
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).getBackupUrl(backupConfig, configFilename, function (error, configResult) {
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
             if (error) return callback(error);
 
-            api(backupConfig.provider).getBackupUrl(backupConfig, dataFilename, function (error, dataResult) {
-                if (error) return callback(error);
+            result.id = dataFilename;
+            result.s3ConfigUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + configFilename;
+            result.s3DataUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + dataFilename;
+            result.backupKey = backupConfig.key;
 
-                var obj = {
-                    id: dataResult.id,
-                    url: dataResult.url,
-                    configUrl: configResult.url,
-                    backupKey: backupConfig.key // only data is encrypted
-                };
+            debug('getAppBackupCredentials: %j', result);
 
-                debug('getAppBackupUrl: %j', obj);
+            callback(null, result);
+        });
+    });
+}
 
-                backupdb.add({ id: obj.id, version: app.manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
-                    if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
+function getRestoreConfig(backupId, callback) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-                    callback(null, obj);
-                });
+    var configFile = backupId.replace(/\.tar\.gz$/, '.json');
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).getRestoreUrl(backupConfig, configFile, function (error, result) {
+            if (error) return callback(error);
+
+            superagent.get(result.url).buffer(true).end(function (error, response) {
+                if (error && !error.response) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+                if (response.statusCode !== 200) return callback(new Error('Invalid response code when getting config.json : ' + response.statusCode));
+
+                var config = safe.JSON.parse(response.text);
+                if (!config) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error in config:' + safe.error.message));
+
+                return callback(null, config);
             });
         });
     });
@@ -170,23 +214,29 @@ function getRestoreUrl(backupId, callback) {
     });
 }
 
-function copyLastBackup(app, callback) {
+function copyLastBackup(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, (new Date()).toISOString(), app.manifest.version);
+    var now = new Date();
+    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, now.toISOString(), manifest.version);
+    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, now.toISOString(), manifest.version);
 
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
+        debug('copyLastBackup: copying archive %s to %s', app.lastBackupId, toFilenameArchive);
+
         api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilenameArchive, function (error) {
             if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
             // TODO change that logic by adjusting app.lastBackupId to not contain the file type
             var configFileId = app.lastBackupId.slice(0, -'.tar.gz'.length) + '.json';
 
+            debug('copyLastBackup: copying config %s to %s', configFileId, toFilenameConfig);
+
             api(backupConfig.provider).copyObject(backupConfig, configFileId, toFilenameConfig, function (error) {
                 if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
@@ -195,3 +245,253 @@ function copyLastBackup(app, callback) {
         });
     });
 }
+
+function backupBoxWithAppBackupIds(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+
+    getBoxBackupCredentials(appBackupIds, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        debug('backupBoxWithAppBackupIds:  %j', result);
+
+        var args = [ result.s3Url, result.accessKeyId, result.secretAccessKey, result.region, result.backupKey ];
+        if (result.sessionToken) args.push(result.sessionToken);
+
+        shell.sudo('backupBox', [ BACKUP_BOX_CMD ].concat(args), function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debug('backupBoxWithAppBackupIds: success');
+
+            backupdb.add({ id: result.id, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                    if (error) return callback(error);
+                    callback(null, result.id);
+                });
+            });
+        });
+    });
+}
+
+// this function expects you to have a lock
+// function backupBox(callback) {
+//    apps.getAll(function (error, allApps) {
+//         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+//
+//         var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
+//         appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+//
+//         backupBoxWithAppBackupIds(appBackupIds, callback);
+//     });
+// }
+
+function canBackupApp(app) {
+    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
+    // state not good for consistent backup (i.e addons may not have been setup completely)
+    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
+            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
+// s3 does not allow changing creation time, so copying the last backup is easy way out for now
+function reuseOldAppBackup(app, manifest, callback) {
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    copyLastBackup(app, manifest, function (error, newBackupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'reuseOldAppBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
+
+        callback(null, newBackupId);
+    });
+}
+
+function createNewAppBackup(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    getAppBackupCredentials(app, manifest, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'createNewAppBackup: backup url:%s backup config url:%s', result.s3DataUrl, result.s3ConfigUrl);
+
+        var args = [ app.id, result.s3ConfigUrl, result.s3DataUrl, result.accessKeyId, result.secretAccessKey, result.region, result.backupKey ];
+        if (result.sessionToken) args.push(result.sessionToken);
+
+        async.series([
+            addons.backupAddons.bind(null, app, manifest.addons),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD ].concat(args))
+        ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debugApp(app, 'createNewAppBackup: %s done', result.id);
+
+            backupdb.add({ id: result.id, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                callback(null, result.id);
+            });
+        });
+    });
+}
+
+function setRestorePoint(appId, lastBackupId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.update(appId, { lastBackupId: lastBackupId }, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function backupApp(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var backupFunction;
+
+    if (!canBackupApp(app)) {
+        if (!app.lastBackupId) {
+            debugApp(app, 'backupApp: cannot backup app');
+            return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy and never backed up previously'));
+        }
+
+        backupFunction = reuseOldAppBackup.bind(null, app, manifest);
+    } else {
+        var appConfig = apps.getAppConfig(app);
+        appConfig.manifest = manifest;
+        backupFunction = createNewAppBackup.bind(null, app, manifest);
+
+        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
+            return callback(safe.error);
+        }
+    }
+
+    backupFunction(function (error, backupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'backupApp: successful id:%s', backupId);
+
+        setRestorePoint(app.id, backupId, function (error) {
+            if (error) return callback(error);
+
+            return callback(null, backupId);
+        });
+    });
+}
+
+// this function expects you to have a lock
+function backupBoxAndApps(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+
+    callback = callback || NOOP_CALLBACK;
+
+    eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { });
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        var processed = 0;
+        var step = 100/(allApps.length+1);
+
+        progress.set(progress.BACKUP, processed, '');
+
+        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
+            ++processed;
+
+            backupApp(app, app.manifest, function (error, backupId) {
+                if (error && error.reason !== BackupsError.BAD_STATE) {
+                    debugApp(app, 'Unable to backup', error);
+                    return iteratorCallback(error);
+                }
+
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
+
+                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
+            });
+        }, function appsBackedUp(error, backupIds) {
+            if (error) {
+                progress.set(progress.BACKUP, 100, error.message);
+                return callback(error);
+            }
+
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
+
+            backupBoxWithAppBackupIds(backupIds, function (error, filename) {
+                progress.set(progress.BACKUP, 100, error ? error.message : '');
+
+                eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { errorMessage: error ? error.message : null, filename: filename });
+
+                callback(error, filename);
+            });
+        });
+    });
+}
+
+function backup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_FULL_BACKUP);
+    if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message));
+
+    progress.set(progress.BACKUP, 0, 'Starting'); // ensure tools can 'wait' on progress
+
+    backupBoxAndApps(auditSource, function (error) { // start the backup operation in the background
+        if (error) debug('backup failed.', error);
+
+        locker.unlock(locker.OP_FULL_BACKUP);
+    });
+
+    callback(null);
+}
+
+function ensureBackup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+
+    getPaged(1, 1, function (error, backups) {
+        if (error) {
+            debug('Unable to list backups', error);
+            return callback(error); // no point trying to backup if appstore is down
+        }
+
+        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
+            debug('Previous backup was %j, no need to backup now', backups[0]);
+            return callback(null);
+        }
+
+        backup(auditSource, callback);
+    });
+}
+
+function restoreApp(app, addonsToRestore, backupId, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof addonsToRestore, 'object');
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+    assert(app.lastBackupId);
+
+    getRestoreUrl(backupId, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
+
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            addons.restoreAddons(app, addonsToRestore, callback);
+        });
+    });
+}

src/cert/acme.js

@@ -19,7 +19,10 @@ var CA_PROD = 'https://acme-v01.api.letsencrypt.org',
     LE_AGREEMENT = 'https://letsencrypt.org/documents/LE-SA-v1.0.1-July-27-2015.pdf';
 
 exports = module.exports = {
-    getCertificate: getCertificate
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'acme'
 };
 
 function AcmeError(reason, errorOrMessage) {

src/cert/caas.js

@@ -1,7 +1,10 @@
 'use strict';
 
 exports = module.exports = {
-    getCertificate: getCertificate
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'caas'
 };
 
 var assert = require('assert'),

src/certificates.js

@@ -1,5 +1,19 @@
 'use strict';
 
+exports = module.exports = {
+    installAdminCertificate: installAdminCertificate,
+    renewAll: renewAll,
+    setFallbackCertificate: setFallbackCertificate,
+    setAdminCertificate: setAdminCertificate,
+    CertificatesError: CertificatesError,
+    validateCertificate: validateCertificate,
+    ensureCertificate: ensureCertificate,
+    getAdminCertificatePath: getAdminCertificatePath,
+
+    // exported for testing
+    _getApi: getApi
+};
+
 var acme = require('./cert/acme.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
@@ -9,6 +23,7 @@ var acme = require('./cert/acme.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:src/certificates'),
+    eventlog = require('./eventlog.js'),
     fs = require('fs'),
     mailer = require('./mailer.js'),
     nginx = require('./nginx.js'),
@@ -17,22 +32,11 @@ var acme = require('./cert/acme.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     sysinfo = require('./sysinfo.js'),
-    tld = require('tldjs'),
     user = require('./user.js'),
     util = require('util'),
     waitForDns = require('./waitfordns.js'),
     x509 = require('x509');
 
-exports = module.exports = {
-    installAdminCertificate: installAdminCertificate,
-    autoRenew: autoRenew,
-    setFallbackCertificate: setFallbackCertificate,
-    setAdminCertificate: setAdminCertificate,
-    CertificatesError: CertificatesError,
-    validateCertificate: validateCertificate,
-    ensureCertificate: ensureCertificate
-};
-
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 function CertificatesError(reason, errorOrMessage) {
@@ -56,15 +60,24 @@ function CertificatesError(reason, errorOrMessage) {
 util.inherits(CertificatesError, Error);
 CertificatesError.INTERNAL_ERROR = 'Internal Error';
 CertificatesError.INVALID_CERT = 'Invalid certificate';
+CertificatesError.NOT_FOUND = 'Not Found';
+
+function getApi(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-function getApi(callback) {
     settings.getTlsConfig(function (error, tlsConfig) {
         if (error) return callback(error);
 
-        var api = tlsConfig.provider === 'caas' ? caas : acme;
+        // use acme if we have altDomain or the tlsConfig is not caas
+        var api = (app.altDomain || tlsConfig.provider) !== 'caas' ? acme : caas;
 
         var options = { };
-        options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
+        if (tlsConfig.provider === 'caas') {
+            options.prod = !config.isDev(); // with altDomain, we will choose acme setting based on this
+        } else { // acme
+            options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
+        }
 
         // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
         // we cannot use admin@fqdn because the user might not have set it up.
@@ -79,8 +92,6 @@ function getApi(callback) {
 }
 
 function installAdminCertificate(callback) {
-    if (cloudron.isConfiguredSync()) return callback();
-
     settings.getTlsConfig(function (error, tlsConfig) {
         if (error) return callback(error);
 
@@ -89,11 +100,10 @@ function installAdminCertificate(callback) {
         sysinfo.getIp(function (error, ip) {
             if (error) return callback(error);
 
-            var zoneName = tld.getDomain(config.fqdn());
-            waitForDns(config.adminFqdn(), ip, zoneName, function (error) {
-                if (error) return callback(error); // this cannot happen because we retry forever
+            waitForDns(config.adminFqdn(), ip, 'A', { interval: 30000, times: 50000 }, function (error) {
+                if (error) return callback(error);
 
-                ensureCertificate(config.adminFqdn(), function (error, certFilePath, keyFilePath) {
+                ensureCertificate({ location: constants.ADMIN_LOCATION }, function (error, certFilePath, keyFilePath) {
                     if (error) { // currently, this can never happen
                         debug('Error obtaining certificate. Proceed anyway', error);
                         return callback();
@@ -119,23 +129,25 @@ function isExpiringSync(certFilePath, hours) {
     return result.status === 1; // 1 - expired 0 - not expired
 }
 
-function autoRenew(callback) {
-    debug('autoRenew: Checking certificates for renewal');
-    callback = callback || NOOP_CALLBACK;
+function renewAll(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('renewAll: Checking certificates for renewal');
 
     apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        allApps.push({ location: 'my' }); // inject fake webadmin app
+        allApps.push({ location: constants.ADMIN_LOCATION }); // inject fake webadmin app
 
         var expiringApps = [ ];
         for (var i = 0; i < allApps.length; i++) {
-            var appDomain = config.appFqdn(allApps[i].location);
+            var appDomain = allApps[i].altDomain || config.appFqdn(allApps[i].location);
             var certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.cert');
             var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
 
             if (!safe.fs.existsSync(keyFilePath)) {
-                debug('autoRenew: no existing key file for %s. skipping', appDomain);
+                debug('renewAll: no existing key file for %s. skipping', appDomain);
                 continue;
             }
 
@@ -144,33 +156,36 @@ function autoRenew(callback) {
             }
         }
 
-        debug('autoRenew: %j needs to be renewed', expiringApps.map(function (a) { return config.appFqdn(a.location); }));
+        debug('renewAll: %j needs to be renewed', expiringApps.map(function (a) { return a.altDomain || config.appFqdn(a.location); }));
 
-        getApi(function (error, api, apiOptions) {
-            if (error) return callback(error);
+        async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
+            var domain = app.altDomain || config.appFqdn(app.location);
 
-            async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
-                var domain = config.appFqdn(app.location);
-                debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
+            getApi(app, function (error, api, apiOptions) {
+                if (error) return callback(error);
+
+                debug('renewAll: renewing cert for %s with options %j', domain, apiOptions);
 
                 api.getCertificate(domain, apiOptions, function (error) {
                     var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
                     var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
 
-                    mailer.certificateRenewed(domain, error ? error.message : '');
+                    var errorMessage = error ? error.message : '';
+                    eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, auditSource, { domain: domain, errorMessage: errorMessage });
+                    mailer.certificateRenewed(domain, errorMessage);
 
                     if (error) {
-                        debug('autoRenew: could not renew cert for %s because %s', domain, error);
+                        debug('renewAll: could not renew cert for %s because %s', domain, error);
 
                         // check if we should fallback if we expire in the coming day
                         if (!isExpiringSync(certFilePath, 24 * 1)) return iteratorCallback();
 
-                        debug('autoRenew: using fallback certs for %s since it expires soon', domain, error);
+                        debug('renewAll: using fallback certs for %s since it expires soon', domain, error);
 
                         certFilePath = 'cert/host.cert';
                         keyFilePath = 'cert/host.key';
                     } else {
-                        debug('autoRenew: certificate for %s renewed', domain);
+                        debug('renewAll: certificate for %s renewed', domain);
                     }
 
                     // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback
@@ -252,12 +267,20 @@ function setFallbackCertificate(cert, key, callback) {
     });
 }
 
+function getFallbackCertificatePath(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // any user fallback cert is always copied over to nginx cert dir
+    callback(null, path.join(paths.NGINX_CERT_DIR, 'host.cert'), path.join(paths.NGINX_CERT_DIR, 'host.key'));
+}
+
+// FIXME: setting admin cert needs to restart the mail container because it uses admin cert
 function setAdminCertificate(cert, key, callback) {
     assert.strictEqual(typeof cert, 'string');
     assert.strictEqual(typeof key, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var vhost = config.appFqdn(constants.ADMIN_LOCATION);
+    var vhost = config.adminFqdn();
     var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
     var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
 
@@ -271,10 +294,24 @@ function setAdminCertificate(cert, key, callback) {
     nginx.configureAdmin(certFilePath, keyFilePath, callback);
 }
 
-function ensureCertificate(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
+function getAdminCertificatePath(callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    var vhost = config.adminFqdn();
+    var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
+    var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
+
+    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, certFilePath, keyFilePath);
+
+    getFallbackCertificatePath(callback);
+}
+
+function ensureCertificate(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var domain = app.altDomain || config.appFqdn(app.location);
+
     // check if user uploaded a specific cert. ideally, we should not mix user certs and automatic certs as we do here...
     var userCertFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
     var userKeyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
@@ -287,7 +324,7 @@ function ensureCertificate(domain, callback) {
 
     debug('ensureCertificate: %s cert require renewal', domain);
 
-    getApi(function (error, api, apiOptions) {
+    getApi(app, function (error, api, apiOptions) {
         if (error) return callback(error);
 
         debug('ensureCertificate: getting certificate for %s with options %j', domain, apiOptions);

src/clientdb.js

@@ -5,6 +5,7 @@
 exports = module.exports = {
     get: get,
     getAll: getAll,
+    getAllWithTokenCount: getAllWithTokenCount,
     getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
     add: add,
     del: del,
@@ -14,13 +15,7 @@ exports = module.exports = {
     delByAppId: delByAppId,
     delByAppIdAndType: delByAppIdAndType,
 
-    _clear: clear,
-
-    TYPE_EXTERNAL: 'external',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
-    TYPE_PROXY: 'addon-proxy',
-    TYPE_ADMIN: 'admin'
+    _clear: clear
 };
 
 var assert = require('assert'),
@@ -52,14 +47,24 @@ function getAll(callback) {
     });
 }
 
+function getAllWithTokenCount(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
 function getAllWithTokenCountByIdentifier(identifier, callback) {
     assert.strictEqual(typeof identifier, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        callback(null, results);
 
+        callback(null, results);
     });
 }
 
@@ -71,7 +76,7 @@ function getByAppId(appId, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result[0]);
+        callback(null, result[0]);
     });
 }
 
@@ -84,7 +89,7 @@ function getByAppIdAndType(appId, type, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result[0]);
+        callback(null, result[0]);
     });
 }
 
@@ -127,7 +132,7 @@ function delByAppId(appId, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null);
+        callback(null);
     });
 }
 
@@ -140,17 +145,17 @@ function delByAppIdAndType(appId, type, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null);
+        callback(null);
     });
 }
 
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
+    database.query('DELETE FROM clients WHERE id!="cid-webadmin" AND id!="cid-sdk" AND id!="cid-cli"', function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        return callback(null);
+        callback(null);
     });
 }
 

src/clients.js

@@ -6,9 +6,32 @@ exports = module.exports = {
     add: add,
     get: get,
     del: del,
-    getAllWithDetailsByUserId: getAllWithDetailsByUserId,
+    getAll: getAll,
+    getByAppIdAndType: getByAppIdAndType,
     getClientTokensByUserId: getClientTokensByUserId,
-    delClientTokensByUserId: delClientTokensByUserId
+    delClientTokensByUserId: delClientTokensByUserId,
+    delByAppIdAndType: delByAppIdAndType,
+    addClientTokenByUserId: addClientTokenByUserId,
+    delToken: delToken,
+
+    // keep this in sync with start.sh ADMIN_SCOPES that generates the cid-webadmin
+    SCOPE_APPS: 'apps',
+    SCOPE_DEVELOPER: 'developer',
+    SCOPE_PROFILE: 'profile',
+    SCOPE_CLOUDRON: 'cloudron',
+    SCOPE_SETTINGS: 'settings',
+    SCOPE_USERS: 'users',
+
+    // roles are handled just like the above scopes, they are parallel to scopes
+    // scopes enclose API groups, roles specify the usage role
+    SCOPE_ROLE_SDK: 'roleSdk',
+
+    // client type enums
+    TYPE_EXTERNAL: 'external',
+    TYPE_BUILT_IN: 'built-in',
+    TYPE_OAUTH: 'addon-oauth',
+    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
+    TYPE_PROXY: 'addon-proxy'
 };
 
 var assert = require('assert'),
@@ -16,7 +39,6 @@ var assert = require('assert'),
     hat = require('hat'),
     appdb = require('./appdb.js'),
     tokendb = require('./tokendb.js'),
-    constants = require('./constants.js'),
     async = require('async'),
     clientdb = require('./clientdb.js'),
     DatabaseError = require('./databaseerror.js'),
@@ -43,14 +65,41 @@ function ClientsError(reason, errorOrMessage) {
 util.inherits(ClientsError, Error);
 ClientsError.INVALID_SCOPE = 'Invalid scope';
 ClientsError.INVALID_CLIENT = 'Invalid client';
+ClientsError.INVALID_TOKEN = 'Invalid token';
+ClientsError.BAD_FIELD = 'Bad field';
+ClientsError.NOT_FOUND = 'Not found';
+ClientsError.INTERNAL_ERROR = 'Internal Error';
+ClientsError.NOT_ALLOWED = 'Not allowed to remove this client';
+
+function validateName(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (name.length < 1) return new ClientsError(ClientsError.BAD_FIELD, 'Name must be atleast 1 character');
+    if (name.length > 128) return new ClientsError(ClientsError.BAD_FIELD, 'Name too long');
+
+    if (/[^a-zA-Z0-9\-]/.test(name)) return new ClientsError(ClientsError.BAD_FIELD, 'Username can only contain alphanumerals and dash');
+
+    return null;
+}
 
 function validateScope(scope) {
     assert.strictEqual(typeof scope, 'string');
 
-    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
-    if (scope === '*') return null;
+    var VALID_SCOPES = [
+        exports.SCOPE_APPS,
+        exports.SCOPE_DEVELOPER,
+        exports.SCOPE_PROFILE,
+        exports.SCOPE_CLOUDRON,
+        exports.SCOPE_SETTINGS,
+        exports.SCOPE_USERS,
+        '*',    // includes all scopes, but not roles
+        exports.SCOPE_ROLE_SDK
+    ];
 
-    // TODO maybe validate all individual scopes if they exist
+    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE, 'Empty scope not allowed');
+
+    var allValid = scope.split(',').every(function (s) { return VALID_SCOPES.indexOf(s) !== -1; });
+    if (!allValid) return new ClientsError(ClientsError.INVALID_SCOPE, 'Invalid scope. Available scopes are ' + VALID_SCOPES.join(', '));
 
     return null;
 }
@@ -62,11 +111,18 @@ function add(appId, type, redirectURI, scope, callback) {
     assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    // allow whitespace
+    scope = scope.split(',').map(function (s) { return s.trim(); }).join(',');
+
     var error = validateScope(scope);
     if (error) return callback(error);
 
+    // appId is also client name
+    error = validateName(appId);
+    if (error) return callback(error);
+
     var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
+    var clientSecret = hat(8 * 128);
 
     clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
         if (error) return callback(error);
@@ -89,6 +145,7 @@ function get(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clientdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
         if (error) return callback(error);
         callback(null, result);
     });
@@ -99,24 +156,24 @@ function del(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clientdb.del(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
         if (error) return callback(error);
         callback(null, result);
     });
 }
 
-function getAllWithDetailsByUserId(userId, callback) {
-    assert.strictEqual(typeof userId, 'string');
+function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.getAllWithTokenCountByIdentifier(tokendb.PREFIX_USER + userId, function (error, results) {
+    clientdb.getAll(function (error, results) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
         if (error) return callback(error);
 
         var tmp = [];
         async.each(results, function (record, callback) {
-            if (record.type === clientdb.TYPE_ADMIN) {
-                record.name = constants.ADMIN_NAME;
-                record.location = constants.ADMIN_LOCATION;
+            if (record.type === exports.TYPE_EXTERNAL || record.type === exports.TYPE_BUILT_IN) {
+                // the appId in this case holds the name
+                record.name = record.appId;
 
                 tmp.push(record);
 
@@ -125,14 +182,13 @@ function getAllWithDetailsByUserId(userId, callback) {
 
             appdb.get(record.appId, function (error, result) {
                 if (error) {
-                    console.error('Failed to get app details for oauth client', result, error);
+                    console.error('Failed to get app details for oauth client', record.appId, error);
                     return callback(null);  // ignore error so we continue listing clients
                 }
 
-                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
-                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
+                if (record.type === exports.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
+                if (record.type === exports.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
+                if (record.type === exports.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
 
                 record.location = result.location;
 
@@ -147,15 +203,27 @@ function getAllWithDetailsByUserId(userId, callback) {
     });
 }
 
+function getByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.getByAppIdAndType(appId, type, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
 function getClientTokensByUserId(clientId, userId, callback) {
     assert.strictEqual(typeof clientId, 'string');
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    tokendb.getByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error, result) {
+    tokendb.getByIdentifierAndClientId(userId, clientId, function (error, result) {
         if (error && error.reason === DatabaseError.NOT_FOUND) {
             // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                 if (error) return callback(error);
                 callback(null, []);
             });
@@ -171,10 +239,10 @@ function delClientTokensByUserId(clientId, userId, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    tokendb.delByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error) {
+    tokendb.delByIdentifierAndClientId(userId, clientId, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) {
             // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                 if (error) return callback(error);
                 callback(null);
             });
@@ -184,3 +252,57 @@ function delClientTokensByUserId(clientId, userId, callback) {
         callback(null);
     });
 }
+
+function delByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.delByAppIdAndType(appId, type, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null);
+    });
+}
+
+function addClientTokenByUserId(clientId, userId, expiresAt, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof expiresAt, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        var token = tokendb.generateToken();
+
+        tokendb.add(token, userId, result.id, expiresAt, result.scope, function (error) {
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null, {
+                accessToken: token,
+                identifier: userId,
+                clientId: result.id,
+                scope: result.id,
+                expires: expiresAt
+            });
+        });
+    });
+}
+
+function delToken(clientId, tokenId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof tokenId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        tokendb.del(tokenId, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.INVALID_TOKEN, 'Invalid token'));
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null);
+        });
+    });
+}

src/cloudron.js

@@ -12,11 +12,9 @@ exports = module.exports = {
     sendHeartbeat: sendHeartbeat,
 
     updateToLatest: updateToLatest,
-    update: update,
     reboot: reboot,
-    backup: backup,
     retire: retire,
-    ensureBackup: ensureBackup,
+    migrate: migrate,
 
     isConfiguredSync: isConfiguredSync,
 
@@ -30,15 +28,15 @@ exports = module.exports = {
 };
 
 var apps = require('./apps.js'),
-    AppsError = require('./apps.js').AppsError,
     assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
-    BackupsError = require('./backups.js').BackupsError,
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:cloudron'),
     df = require('node-df'),
+    eventlog = require('./eventlog.js'),
     fs = require('fs'),
     locker = require('./locker.js'),
     mailer = require('./mailer.js'),
@@ -48,6 +46,7 @@ var apps = require('./apps.js'),
     progress = require('./progress.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
+    SettingsError = settings.SettingsError,
     shell = require('./shell.js'),
     subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
@@ -56,39 +55,33 @@ var apps = require('./apps.js'),
     updateChecker = require('./updatechecker.js'),
     user = require('./user.js'),
     UserError = user.UserError,
-    userdb = require('./userdb.js'),
+    user = require('./user.js'),
     util = require('util'),
-    uuid = require('node-uuid'),
-    webhooks = require('./webhooks.js');
+    _ = require('underscore');
 
 var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
-    BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
-    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
     INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
     RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh');
 
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
+// result to not depend on the appstore
+const BOX_AND_USER_TEMPLATE = {
+    box: {
+        region: null,
+        size: null,
+        plan: 'Custom Plan'
+    },
+    user: {
+        billing: false,
+        currency: ''
+    }
+};
+
 var gUpdatingDns = false,                // flag for dns update reentrancy
-    gCloudronDetails = null,             // cached cloudron details like region,size...
+    gBoxAndUserDetails = null,         // cached cloudron details like region,size...
     gIsConfigured = null;                // cached configured state so that return value is synchronous. null means we are not initialized yet
 
-function debugApp(app, args) {
-    assert(!app || typeof app === 'object');
-
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
-function ignoreError(func) {
-    return function (callback) {
-        func(function (error) {
-            if (error) console.error('Ignored error:', error);
-            callback();
-        });
-    };
-}
-
 function CloudronError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -112,17 +105,16 @@ CloudronError.BAD_FIELD = 'Field error';
 CloudronError.INTERNAL_ERROR = 'Internal Error';
 CloudronError.EXTERNAL_ERROR = 'External Error';
 CloudronError.ALREADY_PROVISIONED = 'Already Provisioned';
-CloudronError.BAD_USERNAME = 'Bad username';
-CloudronError.BAD_EMAIL = 'Bad email';
-CloudronError.BAD_PASSWORD = 'Bad password';
-CloudronError.BAD_NAME = 'Bad name';
 CloudronError.BAD_STATE = 'Bad state';
 CloudronError.ALREADY_UPTODATE = 'No Update Available';
 CloudronError.NOT_FOUND = 'Not found';
+CloudronError.SELF_UPGRADE_NOT_SUPPORTED = 'Self upgrade not supported';
 
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    ensureDkimKeySync();
+
     exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
     exports.events.on(exports.EVENT_FIRST_RUN, installAppBundle);
 
@@ -212,55 +204,62 @@ function setTimeZone(ip, callback) {
 
     debug('setTimeZone ip:%s', ip);
 
-    superagent.get('http://www.telize.com/geoip/' + ip).end(function (error, result) {
+    // https://github.com/bluesmoon/node-geoip
+    // https://github.com/runk/node-maxmind
+    // { url: 'http://freegeoip.net/json/%s', jpath: 'time_zone' },
+    // { url: 'http://ip-api.com/json/%s', jpath: 'timezone' },
+    // { url: 'http://geoip.nekudo.com/api/%s', jpath: 'time_zone }
+
+    superagent.get('http://freegeoip.net/json/' + ip).end(function (error, result) {
         if ((error && !error.response) || result.statusCode !== 200) {
             debug('Failed to get geo location: %s', error.message);
             return callback(null);
         }
 
-        if (!result.body.timezone) {
+        if (!result.body.time_zone || typeof result.body.time_zone !== 'string') {
             debug('No timezone in geoip response : %j', result.body);
             return callback(null);
         }
 
-        debug('Setting timezone to ', result.body.timezone);
+        debug('Setting timezone to ', result.body.time_zone);
 
-        settings.setTimeZone(result.body.timezone, callback);
+        settings.setTimeZone(result.body.time_zone, callback);
     });
 }
 
-function activate(username, password, email, displayName, ip, callback) {
+function activate(username, password, email, displayName, ip, auditSource, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof displayName, 'string');
     assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debug('activating user:%s email:%s', username, email);
 
     setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
 
-    user.createOwner(username, password, email, displayName, function (error, userObject) {
+    user.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
         if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
-        if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
-        if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
-        if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
+        if (error && error.reason === UserError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_FIELD, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
+        clients.get('cid-webadmin', function (error, result) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             // Also generate a token so the admin creation can also act as a login
             var token = tokendb.generateToken();
             var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-            tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
+            tokendb.add(token, userObject.id, result.id, expires, '*', function (error) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
                 // EE API is sync. do not keep the REST API reponse waiting
                 process.nextTick(function () { exports.events.emit(exports.EVENT_ACTIVATED); });
 
+                eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
+
                 callback(null, { token: token, expires: expires });
             });
         });
@@ -270,7 +269,7 @@ function activate(username, password, email, displayName, ip, callback) {
 function getStatus(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    userdb.count(function (error, count) {
+    user.count(function (error, count) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
         settings.getCloudronName(function (error, cloudronName) {
@@ -288,46 +287,34 @@ function getStatus(callback) {
     });
 }
 
-function getCloudronDetails(callback) {
+function getBoxAndUserDetails(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (gCloudronDetails) return callback(null, gCloudronDetails);
+    if (gBoxAndUserDetails) return callback(null, gBoxAndUserDetails);
 
-    if (!config.token()) {
-        gCloudronDetails = {
-            region: null,
-            size: null
-        };
-
-        return callback(null, gCloudronDetails);
-    }
+    // only supported for caas
+    if (config.provider() !== 'caas') return callback(null, {});
 
     superagent
         .get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn())
         .query({ token: config.token() })
         .end(function (error, result) {
-            if (error && !error.response) return callback(error);
-            if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            if (error && !error.response) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, 'Cannot reach appstore'));
+            if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
 
-            gCloudronDetails = result.body.box;
+            gBoxAndUserDetails = result.body;
 
-            return callback(null, gCloudronDetails);
+            return callback(null, gBoxAndUserDetails);
         });
 }
 
 function getConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    getCloudronDetails(function (error, result) {
-        if (error) {
-            debug('Failed to fetch cloudron details.', error);
+    getBoxAndUserDetails(function (error, result) {
+        if (error) debug('Failed to fetch cloudron details.', error.reason, error.message);
 
-            // set fallback values to avoid dependency on appstore
-            result = {
-                region: result ? result.region : null,
-                size: result ? result.size : null
-            };
-        }
+        result = _.extend(BOX_AND_USER_TEMPLATE, result || {});
 
         settings.getCloudronName(function (error, cloudronName) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
@@ -349,8 +336,11 @@ function getConfig(callback) {
                         progress: progress.get(),
                         isCustomDomain: config.isCustomDomain(),
                         developerMode: developerMode,
-                        region: result.region,
-                        size: result.size,
+                        region: result.box.region,
+                        size: result.box.size,
+                        billing: !!result.user.billing,
+                        plan: result.box.plan,
+                        currency: result.user.currency,
                         memory: os.totalmem(),
                         provider: config.provider(),
                         cloudronName: cloudronName
@@ -372,6 +362,21 @@ function sendHeartbeat() {
     });
 }
 
+function ensureDkimKeySync() {
+    var dkimPrivateKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/private');
+    var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
+
+    if (fs.existsSync(dkimPrivateKeyFile) && fs.existsSync(dkimPublicKeyFile)) {
+        debug('DKIM keys already present');
+        return;
+    }
+
+    debug('Generating new DKIM keys');
+
+    safe.child_process.execSync('openssl genrsa -out ' + dkimPrivateKeyFile + ' 1024');
+    safe.child_process.execSync('openssl rsa -in ' + dkimPrivateKeyFile + ' -out ' + dkimPublicKeyFile + ' -pubout -outform PEM');
+}
+
 function readDkimPublicKeySync() {
     var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
     var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
@@ -432,29 +437,33 @@ function addDnsRecords() {
     var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
 
     var dkimKey = readDkimPublicKeySync();
-    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('internal error failed to read dkim public key')));
+    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('Failed to read dkim public key')));
 
     sysinfo.getIp(function (error, ip) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        var webadminRecord = { subdomain: 'my', type: 'A', values: [ ip ] };
+        var webadminRecord = { subdomain: constants.ADMIN_LOCATION, type: 'A', values: [ ip ] };
         // t=s limits the domainkey to this domain and not it's subdomains
         var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
         // DMARC requires special setup if report email id is in different domain
         var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };
 
+        var mxRecord = { subdomain: '', type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] };
+
         var records = [ ];
         if (config.isCustomDomain()) {
             records.push(webadminRecord);
             records.push(dkimRecord);
+            records.push(mxRecord);
         } else {
-            // for custom domains, we show a nakeddomain.html page
+            // for non-custom domains, we show a nakeddomain.html page
             var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
 
             records.push(nakedDomainRecord);
             records.push(webadminRecord);
             records.push(dkimRecord);
             records.push(dmarcRecord);
+            records.push(mxRecord);
         }
 
         debug('addDnsRecords: %j', records);
@@ -490,8 +499,9 @@ function reboot(callback) {
     shell.sudo('reboot', [ REBOOT_CMD ], callback);
 }
 
-function update(boxUpdateInfo, callback) {
+function update(boxUpdateInfo, auditSource, callback) {
     assert.strictEqual(typeof boxUpdateInfo, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     if (!boxUpdateInfo) return callback(null);
@@ -499,6 +509,8 @@ function update(boxUpdateInfo, callback) {
     var error = locker.lock(locker.OP_BOX_UPDATE);
     if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
 
+    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
+
     // ensure tools can 'wait' on progress
     progress.set(progress.UPDATE, 0, 'Starting');
 
@@ -530,13 +542,16 @@ function update(boxUpdateInfo, callback) {
 }
 
 
-function updateToLatest(callback) {
+function updateToLatest(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var boxUpdateInfo = updateChecker.getUpdateInfo().box;
     if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
 
-    update(boxUpdateInfo, callback);
+    if (boxUpdateInfo.upgrade && config.provider() !== 'caas') return callback(new CloudronError(CloudronError.SELF_UPGRADE_NOT_SUPPORTED));
+
+    update(boxUpdateInfo, auditSource, callback);
 }
 
 function doShortCircuitUpdate(boxUpdateInfo, callback) {
@@ -559,7 +574,7 @@ function doUpgrade(boxUpdateInfo, callback) {
 
     progress.set(progress.UPDATE, 5, 'Backing up for upgrade');
 
-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps({ userId: null, username: 'upgrader' }, function (error) {
         if (error) return upgradeError(error);
 
         superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
@@ -573,7 +588,7 @@ function doUpgrade(boxUpdateInfo, callback) {
 
             // no need to unlock since this is the last thing we ever do on this box
             callback();
-            retire();
+            retire('upgrade');
         });
     });
 }
@@ -588,7 +603,7 @@ function doUpdate(boxUpdateInfo, callback) {
 
     progress.set(progress.UPDATE, 5, 'Backing up for update');
 
-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps({ userId: null, username: 'updater' }, function (error) {
         if (error) return updateError(error);
 
         // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
@@ -597,6 +612,7 @@ function doUpdate(boxUpdateInfo, callback) {
 
             // this data is opaque to the installer
             data: {
+                provider: config.provider(),
                 token: config.token(),
                 apiServerOrigin: config.apiServerOrigin(),
                 webServerOrigin: config.webServerOrigin(),
@@ -635,123 +651,6 @@ function doUpdate(boxUpdateInfo, callback) {
     });
 }
 
-function backup(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = locker.lock(locker.OP_FULL_BACKUP);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    // ensure tools can 'wait' on progress
-    progress.set(progress.BACKUP, 0, 'Starting');
-
-    // start the backup operation in the background
-    backupBoxAndApps(function (error) {
-        if (error) console.error('backup failed.', error);
-
-        locker.unlock(locker.OP_FULL_BACKUP);
-    });
-
-    callback(null);
-}
-
-function ensureBackup(callback) {
-    callback = callback || NOOP_CALLBACK;
-
-    backups.getPaged(1, 1, function (error, backups) {
-        if (error) {
-            debug('Unable to list backups', error);
-            return callback(error); // no point trying to backup if appstore is down
-        }
-
-        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
-            debug('Previous backup was %j, no need to backup now', backups[0]);
-            return callback(null);
-        }
-
-        backup(callback);
-    });
-}
-
-function backupBoxWithAppBackupIds(appBackupIds, callback) {
-    assert(util.isArray(appBackupIds));
-
-    backups.getBackupUrl(appBackupIds, function (error, result) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        debug('backup: url %s', result.url);
-
-        async.series([
-            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
-            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
-        ], function (error) {
-            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-            debug('backup: successful');
-
-            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
-                if (error) return callback(error);
-                callback(null, result.id);
-            });
-        });
-    });
-}
-
-// this function expects you to have a lock
-function backupBox(callback) {
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
-        appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
-
-        backupBoxWithAppBackupIds(appBackupIds, callback);
-    });
-}
-
-// this function expects you to have a lock
-function backupBoxAndApps(callback) {
-    callback = callback || NOOP_CALLBACK;
-
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var processed = 0;
-        var step = 100/(allApps.length+1);
-
-        progress.set(progress.BACKUP, processed, '');
-
-        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
-            ++processed;
-
-            apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                if (error && error.reason !== AppsError.BAD_STATE) {
-                    debugApp(app, 'Unable to backup', error);
-                    return iteratorCallback(error);
-                }
-
-                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
-
-                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
-            });
-        }, function appsBackedUp(error, backupIds) {
-            if (error) {
-                progress.set(progress.BACKUP, 100, error.message);
-                return callback(error);
-            }
-
-            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
-
-            backupBoxWithAppBackupIds(backupIds, function (error, filename) {
-                progress.set(progress.BACKUP, 100, error ? error.message : '');
-
-                callback(error, filename);
-            });
-        });
-    });
-}
-
 function installAppBundle(callback) {
     callback = callback || NOOP_CALLBACK;
 
@@ -763,23 +662,16 @@ function installAppBundle(callback) {
     }
 
     async.eachSeries(bundle, function (appInfo, iteratorCallback) {
-        var appstoreId = appInfo.appstoreId;
-        var parts = appstoreId.split('@');
+        debug('autoInstall: installing %s at %s', appInfo.appstoreId, appInfo.location);
 
-        var url = config.apiServerOrigin() + '/api/v1/apps/' + parts[0] + (parts[1] ? '/versions/' + parts[1] : '');
+        var data = {
+            appStoreId: appInfo.appstoreId,
+            location: appInfo.location,
+            portBindings: appInfo.portBindings || null,
+            accessRestriction: appInfo.accessRestriction || null,
+        };
 
-        superagent.get(url).end(function (error, result) {
-            if (error && !error.response) return iteratorCallback(new Error('Network error: ' + error.message));
-
-            if (result.statusCode !== 200) return iteratorCallback(util.format('Failed to get app info from store.', result.statusCode, result.text));
-
-            debug('autoInstall: installing %s at %s', appstoreId, appInfo.location);
-
-            apps.install(uuid.v4(), appstoreId, result.body.manifest, appInfo.location,
-                appInfo.portBindings || null, appInfo.accessRestriction || null,
-                null /* icon */, null /* cert */, null /* key */, 0 /* default mem limit */,
-                iteratorCallback);
-        });
+        apps.install(data, { userId: null, username: 'autoinstaller' }, iteratorCallback);
     }, function (error) {
         if (error) debug('autoInstallApps: ', error);
 
@@ -812,13 +704,73 @@ function checkDiskSpace(callback) {
     });
 }
 
-function retire(callback) {
+function retire(reason, info, callback) {
+    assert(reason === 'migrate' || reason === 'upgrade');
+    info = info || { };
     callback = callback || NOOP_CALLBACK;
 
     var data = {
+        apiServerOrigin: config.apiServerOrigin(),
         isCustomDomain: config.isCustomDomain(),
         fqdn: config.fqdn()
     };
-    shell.sudo('retire', [ RETIRE_CMD, JSON.stringify(data) ], callback);
+    shell.sudo('retire', [ RETIRE_CMD, reason, JSON.stringify(info), JSON.stringify(data) ], callback);
 }
 
+function doMigrate(options, callback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_MIGRATE);
+    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
+
+    function unlock(error) {
+        debug('Failed to migrate', error);
+        locker.unlock(locker.OP_MIGRATE);
+        progress.set(progress.MIGRATE, -1, 'Backup failed: ' + error.message);
+    }
+
+    progress.set(progress.MIGRATE, 10, 'Backing up for migration');
+
+    // initiate the migration in the background
+    backups.backupBoxAndApps({ userId: null, username: 'migrator' }, function (error, backupId) {
+        if (error) return unlock(error);
+
+        debug('migrate: domain: %s size %s region %s', options.domain, options.size, options.region);
+
+        options.restoreKey = backupId;
+
+        superagent
+          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
+          .query({ token: config.token() })
+          .send(options)
+          .end(function (error, result) {
+            if (error && !error.response) return unlock(error); // network error
+            if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
+            if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
+            if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            progress.set(progress.MIGRATE, 10, 'Migrating');
+
+            retire('migrate', _.pick(options, 'domain', 'size', 'region'));
+        });
+    });
+
+    callback(null);
+}
+
+function migrate(options, callback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!options.domain) return doMigrate(options, callback);
+
+    var dnsConfig = _.pick(options, 'domain', 'provider', 'accessKeyId', 'secretAccessKey', 'region', 'endpoint');
+
+    settings.setDnsConfig(dnsConfig, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_FIELD, error.message));
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        doMigrate(options, callback);
+    });
+}

src/config.js

@@ -26,10 +26,11 @@ exports = module.exports = {
     // these values are derived
     adminOrigin: adminOrigin,
     internalAdminOrigin: internalAdminOrigin,
+    sysadminOrigin: sysadminOrigin, // caas routes
     adminFqdn: adminFqdn,
+    mailFqdn: mailFqdn,
     appFqdn: appFqdn,
     zoneName: zoneName,
-    adminEmail: adminEmail,
 
     isDev: isDev,
 
@@ -72,12 +73,12 @@ function initConfig() {
     data.fqdn = 'localhost';
 
     data.token = null;
-    data.adminEmail = null;
     data.boxVersionsUrl = null;
     data.version = null;
     data.isCustomDomain = false;
     data.webServerOrigin = null;
-    data.internalPort = 3001;
+    data.smtpPort = 2525; // // this value comes from mail container
+    data.sysadminPort = 3001;
     data.ldapPort = 3002;
     data.oauthProxyPort = 3003;
     data.simpleAuthPort = 3004;
@@ -99,7 +100,6 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
-        data.adminEmail = 'test@cloudron.foo';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -138,10 +138,6 @@ function get(key) {
     return safe.query(data, key);
 }
 
-function adminEmail() {
-    return '"Cloudron" ' + get('adminEmail');
-}
-
 function apiServerOrigin() {
     return get('apiServerOrigin');
 }
@@ -166,6 +162,10 @@ function adminFqdn() {
     return appFqdn(constants.ADMIN_LOCATION);
 }
 
+function mailFqdn() {
+    return appFqdn(constants.MAIL_LOCATION);
+}
+
 function adminOrigin() {
     return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }
@@ -174,6 +174,10 @@ function internalAdminOrigin() {
     return 'http://127.0.0.1:' + get('port');
 }
 
+function sysadminOrigin() {
+    return 'http://127.0.0.1:' + get('sysadminPort');
+}
+
 function token() {
     return get('token');
 }

src/constants.js

@@ -4,11 +4,18 @@
 exports = module.exports = {
     ADMIN_LOCATION: 'my',
     API_LOCATION: 'api', // this is unused but reserved for future use (#403)
+    SMTP_LOCATION: 'smtp',
+    IMAP_LOCATION: 'imap',
+    MAIL_LOCATION: 'my', // not a typo! should be same as admin location until we figure out certificates
+    POSTMAN_LOCATION: 'postman', // used in dovecot bounces
+
     ADMIN_NAME: 'Settings',
 
     ADMIN_CLIENT_ID: 'webadmin', // oauth client id
     ADMIN_APPID: 'admin', // admin appid (settingsdb)
 
+    GHOST_USER_FILE: '/tmp/cloudron_ghost.json',
+
     DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024) // see also client.js
 };
 

src/cron.js

@@ -7,11 +7,13 @@ exports = module.exports = {
 
 var apps = require('./apps.js'),
     assert = require('assert'),
+    backups = require('./backups.js'),
     certificates = require('./certificates.js'),
     cloudron = require('./cloudron.js'),
     config = require('./config.js'),
     CronJob = require('cron').CronJob,
     debug = require('debug')('box:cron'),
+    eventlog = require('./eventlog.js'),
     janitor = require('./janitor.js'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
@@ -26,9 +28,11 @@ var gAutoupdaterJob = null,
     gDockerVolumeCleanerJob = null,
     gSchedulerSyncJob = null,
     gCertificateRenewJob = null,
-    gCheckDiskSpaceJob = null;
+    gCheckDiskSpaceJob = null,
+    gCleanupEventlogJob = null;
 
 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
+var AUDIT_SOURCE = { userId: null, username: 'cron' };
 
 // cron format
 // Seconds: 0-59
@@ -64,8 +68,8 @@ function recreateJobs(unusedTimeZone, callback) {
 
         if (gBackupJob) gBackupJob.stop();
         gBackupJob = new CronJob({
-            cronTime: '00 00 */4 * * *', // every 4 hours
-            onTick: cloudron.ensureBackup,
+            cronTime: '00 00 */4 * * *', // every 4 hours. backups.ensureBackup() will only trigger a backup once per day
+            onTick: backups.ensureBackup.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
             start: true,
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
@@ -102,6 +106,14 @@ function recreateJobs(unusedTimeZone, callback) {
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
 
+        if (gCleanupEventlogJob) gCleanupEventlogJob.stop();
+        gCleanupEventlogJob = new CronJob({
+            cronTime: '00 */30 * * * *', // every 30 minutes
+            onTick: eventlog.cleanup,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
         if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
         gDockerVolumeCleanerJob = new CronJob({
             cronTime: '00 00 */12 * * *', // every 12 hours
@@ -121,7 +133,7 @@ function recreateJobs(unusedTimeZone, callback) {
         if (gCertificateRenewJob) gCertificateRenewJob.stop();
         gCertificateRenewJob = new CronJob({
             cronTime: '00 00 */12 * * *', // every 12 hours
-            onTick: certificates.autoRenew,
+            onTick: certificates.renewAll.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
             start: true,
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
@@ -153,10 +165,10 @@ function autoupdatePatternChanged(pattern) {
             var updateInfo = updateChecker.getUpdateInfo();
             if (updateInfo.box) {
                 debug('Starting autoupdate to %j', updateInfo.box);
-                cloudron.update(updateInfo.box, NOOP_CALLBACK);
+                cloudron.updateToLatest(AUDIT_SOURCE, NOOP_CALLBACK);
             } else if (updateInfo.apps) {
                 debug('Starting app update to %j', updateInfo.apps);
-                apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
+                apps.updateApps(updateInfo.apps, AUDIT_SOURCE, NOOP_CALLBACK);
             } else {
                 debug('No auto updates available');
             }
@@ -192,6 +204,9 @@ function uninitialize(callback) {
     if (gCleanupTokensJob) gCleanupTokensJob.stop();
     gCleanupTokensJob = null;
 
+    if (gCleanupEventlogJob) gCleanupEventlogJob.stop();
+    gCleanupEventlogJob = null;
+
     if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
     gDockerVolumeCleanerJob = null;
 

src/database.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -121,7 +119,9 @@ function clear(callback) {
         require('./tokendb.js')._clear,
         require('./groupdb.js')._clear,
         require('./userdb.js')._clear,
-        require('./settingsdb.js')._clear
+        require('./settingsdb.js')._clear,
+        require('./eventlogdb.js')._clear,
+        require('./mailboxdb.js')._clear
     ], callback);
 }
 

src/developer.js

@@ -5,7 +5,7 @@
 exports = module.exports = {
     DeveloperError: DeveloperError,
 
-    enabled: enabled,
+    isEnabled: isEnabled,
     setEnabled: setEnabled,
     issueDeveloperToken: issueDeveloperToken,
     getNonApprovedApps: getNonApprovedApps
@@ -13,7 +13,9 @@ exports = module.exports = {
 
 var assert = require('assert'),
     config = require('./config.js'),
+    clients = require('./clients.js'),
     debug = require('debug')('box:developer'),
+    eventlog = require('./eventlog.js'),
     tokendb = require('./tokendb.js'),
     settings = require('./settings.js'),
     superagent = require('superagent'),
@@ -41,7 +43,7 @@ util.inherits(DeveloperError, Error);
 DeveloperError.INTERNAL_ERROR = 'Internal Error';
 DeveloperError.EXTERNAL_ERROR = 'External Error';
 
-function enabled(callback) {
+function isEnabled(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     settings.getDeveloperMode(function (error, enabled) {
@@ -50,27 +52,35 @@ function enabled(callback) {
     });
 }
 
-function setEnabled(enabled, callback) {
+function setEnabled(enabled, auditSource, callback) {
     assert.strictEqual(typeof enabled, 'boolean');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     settings.setDeveloperMode(enabled, function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+
+        eventlog.add(eventlog.ACTION_CLI_MODE, auditSource, { enabled: enabled });
+
         callback(null);
     });
 }
 
-function issueDeveloperToken(user, callback) {
+function issueDeveloperToken(user, auditSource, callback) {
     assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var token = tokendb.generateToken();
     var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+    var scopes = '*,' + clients.SCOPE_ROLE_SDK;
 
-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users', function (error) {
+    tokendb.add(token, user.id, 'cid-cli', expiresAt, scopes, function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
 
-        callback(null, { token: token, expiresAt: expiresAt });
+        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { authType: 'cli', userId: user.id, username: user.username });
+
+        callback(null, { token: token, expiresAt: new Date(expiresAt).toISOString() });
     });
 }
 
@@ -80,7 +90,7 @@ function getNonApprovedApps(callback) {
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
     superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
         if (error && !error.response) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, error));
-        if (result.statusCode === 401) {
+        if (result.statusCode === 401 || result.statusCode === 403) {
             debug('Failed to list apps in development. Appstore token invalid or missing. Returning empty list.', result.body);
             return callback(null, []);
         }

src/dns/route53.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -7,12 +5,14 @@ exports = module.exports = {
     get: get,
     del: del,
     update: update,
-    getChangeStatus: getChangeStatus
+    getChangeStatus: getChangeStatus,
+
+    // not part of "dns" interface
+    getHostedZone: getHostedZone
 };
 
 var assert = require('assert'),
     AWS = require('aws-sdk'),
-    config = require('../config.js'),
     debug = require('debug')('box:dns/route53'),
     SubdomainError = require('../subdomains.js').SubdomainError,
     util = require('util'),
@@ -52,6 +52,24 @@ function getZoneByName(dnsConfig, zoneName, callback) {
     });
 }
 
+function getHostedZone(dnsConfig, zoneName, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+ 
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.getHostedZone({ Id: zone.Id }, function (error, result) {
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+
+            callback(null, result);
+        });
+    });
+}
+
 function add(dnsConfig, zoneName, subdomain, type, values, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
@@ -65,7 +83,7 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
     getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
 
-        var fqdn = config.appFqdn(subdomain);
+        var fqdn = subdomain === '' ? zoneName : subdomain + '.' + zoneName;
         var records = values.map(function (v) { return { Value: v }; });
 
         var params = {
@@ -153,7 +171,7 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
     getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
 
-        var fqdn = config.appFqdn(subdomain);
+        var fqdn = subdomain === '' ? zoneName : subdomain + '.' + zoneName;
         var records = values.map(function (v) { return { Value: v }; });
 
         var resourceRecordSet = {

src/docker.js

@@ -1,17 +1,5 @@
 'use strict';
 
-var addons = require('./addons.js'),
-    async = require('async'),
-    assert = require('assert'),
-    config = require('./config.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:src/docker.js'),
-    Docker = require('dockerode'),
-    safe = require('safetydance'),
-    semver = require('semver'),
-    util = require('util'),
-    _ = require('underscore');
-
 exports = module.exports = {
     connection: connectionInstance(),
     downloadImage: downloadImage,
@@ -25,10 +13,12 @@ exports = module.exports = {
     deleteImage: deleteImage,
     deleteContainers: deleteContainers,
     createSubcontainer: createSubcontainer,
-    getContainerIdByIp: getContainerIdByIp
+    getContainerIdByIp: getContainerIdByIp,
+    execContainer: execContainer
 };
 
 function connectionInstance() {
+    var Docker = require('dockerode');
     var docker;
 
     if (process.env.BOX_ENV === 'test') {
@@ -44,6 +34,19 @@ function connectionInstance() {
     return docker;
 }
 
+var addons = require('./addons.js'),
+    async = require('async'),
+    assert = require('assert'),
+    child_process = require('child_process'),
+    config = require('./config.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:src/docker.js'),
+    once = require('once'),
+    safe = require('safetydance'),
+    spawn = child_process.spawn,
+    util = require('util'),
+    _ = require('underscore');
+
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
 
@@ -129,17 +132,18 @@ function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var docker = exports.connection,
-        isAppContainer = !cmd;
+        isAppContainer = !cmd; // non app-containers are like scheduler containers
 
     var manifest = app.manifest;
     var developmentMode = !!manifest.developmentMode;
     var exposedPorts = {}, dockerPortBindings = { };
+    var domain = app.altDomain || config.appFqdn(app.location);
     var stdEnv = [
         'CLOUDRON=1',
         'WEBADMIN_ORIGIN=' + config.adminOrigin(),
         'API_ORIGIN=' + config.adminOrigin(),
-        'APP_ORIGIN=https://' + config.appFqdn(app.location),
-        'APP_DOMAIN=' + config.appFqdn(app.location)
+        'APP_ORIGIN=https://' + domain,
+        'APP_DOMAIN=' + domain
     ];
 
     // docker portBindings requires ports to be exposed
@@ -167,18 +171,16 @@ function createSubcontainer(app, name, cmd, options, callback) {
     // developerMode does not restrict memory usage
     memoryLimit = developmentMode ? 0 : memoryLimit;
 
-    // for subcontainers, this should ideally be false. but docker does not allow network sharing if the app container is not running
-    // this means cloudron exec does not work
-    var isolatedNetworkNs = true;
-
     addons.getEnvironment(app, function (error, addonEnv) {
         if (error) return callback(new Error('Error getting addon environment : ' + error));
 
+        // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail
+        // location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns
+        // name to look up the internal docker ip. this makes curl from within container fail
+        // Note that Hostname has no effect on DNS. We have to use the --net-alias for dns.
+        // Hostname cannot be set with container NetworkMode
         var containerOptions = {
             name: name, // used for filtering logs
-            // do _not_ set hostname to app fqdn. doing so sets up the dns name to look up the internal docker ip. this makes curl from within container fail
-            // for subcontainers, this should not be set because we already share the network namespace with app container
-            Hostname: isolatedNetworkNs ? (semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location)) : null,
             Tty: isAppContainer,
             Image: app.manifest.dockerImage,
             Cmd: (isAppContainer && developmentMode) ? [ '/bin/bash', '-c', 'echo "Development mode. Use cloudron exec to debug. Sleeping" && sleep infinity' ] : cmd,
@@ -206,8 +208,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                 },
                 CpuShares: 512, // relative to 1024 for system processes
                 VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
-                NetworkMode: isolatedNetworkNs ? 'default' : ('container:' + app.containerId), // share network namespace with parent
-                Links: isolatedNetworkNs ? addons.getLinksSync(app, app.manifest.addons) : null, // links is redundant with --net=container
+                NetworkMode: isAppContainer ? 'cloudron' : ('container:' + app.containerId), // share network namespace with parent
                 SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
             }
         };
@@ -370,10 +371,10 @@ function getContainerIdByIp(ip, callback) {
 
         var bridge;
         result.forEach(function (n) {
-            if (n.Name === 'bridge') bridge = n;
+            if (n.Name === 'cloudron') bridge = n;
         });
 
-        if (!bridge) return callback(new Error('Unable to find the bridge network'));
+        if (!bridge) return callback(new Error('Unable to find the cloudron network'));
 
         var containerId;
         for (var id in bridge.Containers) {
@@ -389,3 +390,34 @@ function getContainerIdByIp(ip, callback) {
         callback(null, containerId);
     });
 }
+
+function execContainer(containerId, cmd, options, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert(util.isArray(cmd));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', '-i', containerId  ].concat(cmd));
+
+    var chunks = [ ];
+
+    if (options.stdout) {
+        cp.stdout.pipe(options.stdout);
+    } else if (options.bufferStdout) {
+        cp.stdout.on('data', function (chunk) { chunks.push(chunk); });
+    } else {
+        cp.stdout.pipe(process.stdout);
+    }
+
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debug('execContainer code: %s signal: %s', code, signal);
+        if (!callback.called) callback(code ? 'Failed with status ' + code : null, Buffer.concat(chunks));
+    });
+
+    cp.stderr.pipe(options.stderr || process.stderr);
+
+    if (options.stdin) options.stdin.pipe(cp.stdin).on('error', callback);
+}

src/eventlog.js

@@ -0,0 +1,116 @@
+'use strict';
+
+exports = module.exports = {
+    EventLogError: EventLogError,
+
+    add: add,
+    get: get,
+    getAllPaged: getAllPaged,
+    cleanup: cleanup,
+
+    // keep in sync with webadmin index.js filter
+    ACTION_ACTIVATE: 'cloudron.activate',
+    ACTION_APP_CLONE: 'app.clone',
+    ACTION_APP_CONFIGURE: 'app.configure',
+    ACTION_APP_INSTALL: 'app.install',
+    ACTION_APP_RESTORE: 'app.restore',
+    ACTION_APP_UNINSTALL: 'app.uninstall',
+    ACTION_APP_UPDATE: 'app.update',
+    ACTION_BACKUP_FINISH: 'backup.finish',
+    ACTION_BACKUP_START: 'backup.start',
+    ACTION_CERTIFICATE_RENEWAL: 'certificate.renew',
+    ACTION_CLI_MODE: 'settings.climode',
+    ACTION_START: 'cloudron.start',
+    ACTION_UPDATE: 'cloudron.update',
+    ACTION_USER_ADD: 'user.add',
+    ACTION_USER_LOGIN: 'user.login',
+    ACTION_USER_REMOVE: 'user.remove',
+    ACTION_USER_UPDATE: 'user.update'
+};
+
+var assert = require('assert'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:eventlog'),
+    eventlogdb = require('./eventlogdb.js'),
+    util = require('util'),
+    uuid = require('node-uuid');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function EventLogError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(EventLogError, Error);
+EventLogError.INTERNAL_ERROR = 'Internal error';
+EventLogError.NOT_FOUND = 'Not Found';
+
+function add(action, source, data, callback) {
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert(!callback || typeof callback === 'function');
+
+    callback = callback || NOOP_CALLBACK;
+
+    var id = uuid.v4();
+
+    eventlogdb.add(id, action, source, data, function (error) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, { id: id });
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new EventLogError(EventLogError.NOT_FOUND, 'No such event'));
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function getAllPaged(action, search, page, perPage, callback) {
+    assert(typeof action === 'string' || action === null);
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.getAllPaged(action, search, page, perPage, function (error, boxes) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, boxes);
+    });
+}
+
+function cleanup(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+     var d = new Date();
+     d.setDate(d.getDate() - 7); // 7 days ago
+
+    eventlogdb.delByCreationTime(d, function (error) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}

src/eventlogdb.js

@@ -0,0 +1,116 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getAllPaged: getAllPaged,
+    add: add,
+    count: count,
+    delByCreationTime: delByCreationTime,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    mysql = require('mysql'),
+    safe = require('safetydance'),
+    util = require('util');
+
+var EVENTLOGS_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
+
+// until mysql module supports automatic type coercion
+function postProcess(eventLog) {
+    eventLog.source = safe.JSON.parse(eventLog.source);
+    eventLog.data = safe.JSON.parse(eventLog.data);
+    return eventLog;
+}
+
+function get(eventId, callback) {
+    assert.strictEqual(typeof eventId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog WHERE id = ?', [ eventId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, postProcess(result[0]));
+    });
+}
+
+function getAllPaged(action, search, page, perPage, callback) {
+    assert(typeof action === 'string' || action === null);
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [];
+    var query = 'SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog';
+
+    if (action || search) query += ' WHERE';
+    if (search) query += ' data LIKE ' + mysql.escape('%' + search + '%');
+    if (action && search) query += ' AND ';
+
+    if (action) {
+        query += ' action=?';
+        data.push(action);
+    }
+
+    query += ' ORDER BY creationTime DESC LIMIT ?,?';
+
+    data.push((page-1)*perPage);
+    data.push(perPage);
+
+    database.query(query, data, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function add(id, action, source, data, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM eventlog', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM eventlog', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(error);
+    });
+}
+
+function delByCreationTime(creationTime, callback) {
+    assert(util.isDate(creationTime));
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM eventlog WHERE creationTime < ?', [ creationTime ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(error);
+    });
+}

src/groupdb.js

@@ -4,6 +4,7 @@ exports = module.exports = {
     get: get,
     getWithMembers: getWithMembers,
     getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
     add: add,
     del: del,
     count: count,
@@ -65,6 +66,19 @@ function getAll(callback) {
     });
 }
 
+function getAllWithMembers(callback) {
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' GROUP BY groups.id', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
+
+        callback(null, results);
+    });
+}
+
 function add(id, name, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof name, 'string');

src/groups.js

@@ -10,6 +10,7 @@ exports = module.exports = {
     get: get,
     getWithMembers: getWithMembers,
     getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
 
     getMembers: getMembers,
     addMember: addMember,
@@ -51,7 +52,7 @@ util.inherits(GroupError, Error);
 GroupError.INTERNAL_ERROR = 'Internal Error';
 GroupError.ALREADY_EXISTS = 'Already Exists';
 GroupError.NOT_FOUND = 'Not Found';
-GroupError.BAD_NAME = 'Bad name';
+GroupError.BAD_FIELD = 'Field error';
 GroupError.NOT_EMPTY = 'Not Empty';
 GroupError.NOT_ALLOWED = 'Not Allowed';
 
@@ -59,12 +60,12 @@ function validateGroupname(name) {
     assert.strictEqual(typeof name, 'string');
     var RESERVED = [ 'admins', 'users' ]; // ldap code uses 'users' pseudo group
 
-    if (name.length <= 2) return new GroupError(GroupError.BAD_NAME, 'name must be atleast 2 chars');
-    if (name.length >= 200) return new GroupError(GroupError.BAD_NAME, 'name too long');
+    if (name.length <= 2) return new GroupError(GroupError.BAD_FIELD, 'name must be atleast 2 chars');
+    if (name.length >= 200) return new GroupError(GroupError.BAD_FIELD, 'name too long');
 
-    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_NAME, 'name can only have A-Za-z0-9_-');
+    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_FIELD, 'name can only have A-Za-z0-9_-');
 
-    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_NAME, 'name is reserved');
+    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_FIELD, 'name is reserved');
 
     return null;
 }
@@ -133,6 +134,16 @@ function getAll(callback) {
     });
 }
 
+function getAllWithMembers(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAllWithMembers(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
 function getMembers(groupId, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof callback, 'function');

src/infra_version.js

@@ -0,0 +1,22 @@
+'use strict';
+
+// WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+// These constants are used in the installer script as well
+// Do not require anything here!
+
+exports = module.exports = {
+    // a version bump means that all containers (apps and addons) are recreated
+    'version': 40,
+
+    'baseImage': 'cloudron/base:0.8.1',
+
+    'images': {
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:0.12.0' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.11.0' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.10.0' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.9.0' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.18.0' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.9.0' }
+    }
+};
+

src/ldap.js

@@ -9,9 +9,12 @@ var assert = require('assert'),
     apps = require('./apps.js'),
     config = require('./config.js'),
     debug = require('debug')('box:ldap'),
+    eventlog = require('./eventlog.js'),
     user = require('./user.js'),
     UserError = user.UserError,
-    ldap = require('ldapjs');
+    ldap = require('ldapjs'),
+    mailboxes = require('./mailboxes.js'),
+    MailboxError = mailboxes.MailboxError;
 
 var gServer = null;
 
@@ -34,8 +37,204 @@ function getAppByRequest(req, callback) {
     if (sourceIp.split('.').length !== 4) return callback(new ldap.InsufficientAccessRightsError('Missing source identifier'));
 
     apps.getByIpAddress(sourceIp, function (error, app) {
-        // we currently allow access in case we can't find the source app
-        callback(null, app || null);
+        if (error) return callback(new ldap.OperationsError(error.message));
+
+        if (!app) return callback(new ldap.OperationsError('Could not detect app source'));
+
+        callback(null, app);
+    });
+}
+
+function userSearch(req, res, next) {
+    debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        // send user objects
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+
+            var groups = [ GROUP_USERS_DN ];
+            if (entry.admin) groups.push(GROUP_ADMINS_DN);
+
+            var displayName = entry.displayName || entry.username;
+            var nameParts = displayName.split(' ');
+            var firstName = nameParts[0];
+            var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['user'],
+                    objectcategory: 'person',
+                    cn: entry.id,
+                    uid: entry.id,
+                    mail: entry.email,
+                    // TODO: check mailboxes before we send this
+                    mailAlternateAddress: entry.username + '@' + config.fqdn(),
+                    displayname: displayName,
+                    givenName: firstName,
+                    username: entry.username,
+                    samaccountname: entry.username,      // to support ActiveDirectory clients
+                    memberof: groups
+                }
+            };
+
+            // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
+            // which is required to have atleast one character if present
+            if (lastName.length !== 0) obj.attributes.sn = lastName;
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function groupSearch(req, res, next) {
+    debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result){
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        var groups = [{
+            name: 'users',
+            admin: false
+        }, {
+            name: 'admins',
+            admin: true
+        }];
+
+        groups.forEach(function (group) {
+            var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+            var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['group'],
+                    cn: group.name,
+                    memberuid: members.map(function(entry) { return entry.id; })
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function mailboxSearch(req, res, next) {
+    debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.name + ',ou=mailboxes,dc=cloudron');
+
+            // TODO: send aliases
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['mailbox'],
+                    objectcategory: 'mailbox',
+                    cn: entry.name,
+                    uid: entry.name,
+                    mail: entry.name + '@' + config.fqdn()
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function authenticateUser(req, res, next) {
+    debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
+
+    // extract the common name which might have different attribute names
+    var attributeName = Object.keys(req.dn.rdns[0])[0];
+    var commonName = req.dn.rdns[0][attributeName];
+    if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+    var api;
+    if (attributeName === 'mail') {
+        api = user.verifyWithEmail;
+    } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
+        var parts = commonName.split('@');
+        if (parts[1] === config.fqdn()) { // internal email, verify with username
+            commonName = parts[0];
+            api = user.verifyWithUsername;
+        } else { // external email
+            api = user.verifyWithEmail;
+        }
+    } else if (commonName.indexOf('uid-') === 0) {
+        api = user.verify;
+    } else {
+        api = user.verifyWithUsername;
+    }
+
+    api(commonName, req.credentials || '', function (error, user) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.message));
+
+        req.user = user;
+
+        next();
+    });
+}
+
+function authorizeUserForApp(req, res, next) {
+    assert(req.user);
+
+    getAppByRequest(req, function (error, app) {
+        if (error) return next(error);
+
+        apps.hasAccessTo(app, req.user, function (error, result) {
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            // we return no such object, to avoid leakage of a users existence
+            if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+            eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: app.id }, { userId: req.user.id });
+
+            res.end();
+        });
+    });
+}
+
+function authorizeUserForMailbox(req, res, next) {
+    assert(req.user);
+
+    // We simply authorize the user to access a mailbox by his own name
+    mailboxes.get(req.user.username, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.message));
+
+        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: req.user.username }, { userId: req.user.username });
+
+        res.end();
     });
 }
 
@@ -44,130 +243,27 @@ function start(callback) {
 
     gServer = ldap.createServer({ log: gLogger });
 
-    gServer.search('ou=users,dc=cloudron', function (req, res, next) {
-        debug('user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+    gServer.search('ou=users,dc=cloudron', userSearch);
+    gServer.search('ou=groups,dc=cloudron', groupSearch);
+    gServer.bind('ou=users,dc=cloudron', authenticateUser, authorizeUserForApp);
 
-        user.list(function (error, result) {
-            if (error) return next(new ldap.OperationsError(error.toString()));
+    gServer.search('ou=mailboxes,dc=cloudron', mailboxSearch);
+    gServer.bind('ou=mailboxes,dc=cloudron', authenticateUser, authorizeUserForMailbox);
 
-            // send user objects
-            result.forEach(function (entry) {
-                var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
-
-                var groups = [ GROUP_USERS_DN ];
-                if (entry.admin) groups.push(GROUP_ADMINS_DN);
-
-                var tmp = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['user'],
-                        objectcategory: 'person',
-                        cn: entry.id,
-                        uid: entry.id,
-                        mail: entry.email,
-                        displayname: entry.displayName || entry.username,
-                        username: entry.username,
-                        samaccountname: entry.username,      // to support ActiveDirectory clients
-                        memberof: groups
-                    }
-                };
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
-        debug('group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
-
-        user.list(function (error, result){
-            if (error) return next(new ldap.OperationsError(error.toString()));
-
-            var groups = [{
-                name: 'users',
-                admin: false
-            }, {
-                name: 'admins',
-                admin: true
-            }];
-
-            groups.forEach(function (group) {
-                var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-                var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
-
-                var tmp = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['group'],
-                        cn: group.name,
-                        memberuid: members.map(function(entry) { return entry.id; })
-                    }
-                };
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
-                }
-            });
-
-            res.end();
-        });
+    // this is the bind for addons (after bind, they might search and authenticate)
+    gServer.bind('ou=addons,dc=cloudron', function(req, res, next) {
+        debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id
+        res.end();
     });
 
+    // this is the bind for apps (after bind, they might search and authenticate user)
     gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
         // TODO: validate password
         debug('application bind: %s', req.dn.toString());
         res.end();
     });
 
-    gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
-        debug('user bind: %s', req.dn.toString());
-
-        // extract the common name which might have different attribute names
-        var attributeName = Object.keys(req.dn.rdns[0])[0];
-        var commonName = req.dn.rdns[0][attributeName];
-        if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-        var api;
-        // if mail is specified, enforce mail check
-        if (commonName.indexOf('@') !== -1 || attributeName === 'mail') {
-            api = user.verifyWithEmail;
-        } else if (commonName.indexOf('uid-') === 0) {
-            api = user.verify;
-        } else {
-            api = user.verifyWithUsername;
-        }
-
-        // TODO this should be done after we verified the app has access to avoid leakage of user existence
-        api(commonName, req.credentials || '', function (error, userObject) {
-            if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
-            if (error) return next(new ldap.OperationsError(error));
-
-            getAppByRequest(req, function (error, app) {
-                if (error) return next(error);
-
-                if (!app) {
-                    debug('no app found for this container, allow access');
-                    return res.end();
-                }
-
-                apps.hasAccessTo(app, userObject, function (error, result) {
-                    if (error) return next(new ldap.OperationsError(error.toString()));
-
-                    // we return no such object, to avoid leakage of a users existence
-                    if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-                    res.end();
-                });
-            });
-        });
-    });
-
-    gServer.listen(config.get('ldapPort'), callback);
+    gServer.listen(config.get('ldapPort'), '0.0.0.0', callback);
 }
 
 function stop(callback) {

src/logcollector.js

@@ -0,0 +1,37 @@
+'use strict';
+
+exports = module.exports = {
+    sendFailureLogs: sendFailureLogs
+};
+
+var assert = require('assert'),
+    mailer = require('./mailer.js'),
+    safe = require('safetydance'),
+    path = require('path'),
+    util = require('util');
+
+var COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
+
+function collectLogs(unitName, callback) {
+    assert.strictEqual(typeof unitName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
+    callback(null, logs);
+}
+
+function sendFailureLogs(processName, options) {
+    assert.strictEqual(typeof processName, 'string');
+    assert.strictEqual(typeof options, 'object');
+
+    collectLogs(options.unit || processName, function (error, result) {
+        if (error) {
+            console.error('Failed to collect logs.', error);
+            result = util.format('Failed to collect logs.', error);
+        }
+
+        console.log('Sending failure logs for', processName);
+
+        mailer.unexpectedExit(processName, result);
+    });
+}

src/mail_templates/unexpected_exit.ejs

@@ -2,7 +2,7 @@
 
 Dear Cloudron Team,
 
-Unfortunately <%= program %> on <%= fqdn %> crashed unexpectedly!
+Unfortunately <%= program %> on <%= fqdn %> exited unexpectedly!
 
 Please see some excerpt of the logs below.
 

src/mail_templates/user_added.ejs

@@ -2,7 +2,7 @@
 
 Dear Admin,
 
-User with name <%= user.email %> was added in the Cloudron at <%= fqdn %>.
+User with email <%= user.email %> was added in the Cloudron at <%= fqdn %>.
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 

src/mailboxdb.js

@@ -0,0 +1,127 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    getAliases: getAliases,
+    setAliases: setAliases,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var MAILBOX_FIELDS = [ 'name', 'aliasTarget', 'creationTime' ].join(',');
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO mailboxes (name) VALUES (?)', [ name ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE mailboxes', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // deletes aliases as well
+    database.query('DELETE FROM mailboxes WHERE name=? OR aliasTarget = ?', [ name, name ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function postProcess(result) {
+    result.aliases = result.aliases ? result.aliases.split(',') : [ ];
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.name=? AND m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(results[0]);
+
+        callback(null, results[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM mailboxes WHERE aliasTarget = ?', args: [ name ] });
+    aliases.forEach(function (alias) {
+        queries.push({ query: 'INSERT INTO mailboxes (name, aliasTarget) VALUES (?, ?)', args: [ alias, name ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name FROM mailboxes WHERE aliasTarget=? ORDER BY name', [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results = results.map(function (r) { return r.name; });
+        callback(null, results);
+    });
+}

src/mailboxes.js

@@ -0,0 +1,199 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    setAliases: setAliases,
+    getAliases: getAliases,
+
+    setupAliases: setupAliases,
+
+    MailboxError: MailboxError
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:mailboxes'),
+    docker = require('./docker.js'),
+    mailboxdb = require('./mailboxdb.js'),
+    util = require('util');
+
+function MailboxError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(MailboxError, Error);
+MailboxError.ALREADY_EXISTS = 'already exists';
+MailboxError.BAD_FIELD = 'Field error';
+MailboxError.NOT_FOUND = 'not found';
+MailboxError.INTERNAL_ERROR = 'internal error';
+MailboxError.EXTERNAL_ERROR = 'external error';
+
+function validateName(name) {
+    var RESERVED_NAMES = [ 'no-reply', 'postmaster', 'mailer-daemon' ];
+
+    if (!name.length) return new MailboxError(MailboxError.BAD_FIELD, "name cannot be empty");
+
+    if (name.length < 2) return new MailboxError(MailboxError.BAD_FIELD, 'name too small');
+    if (name.length > 127) return new MailboxError(MailboxError.BAD_FIELD, 'name too long');
+    if (RESERVED_NAMES.indexOf(name) !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'name is reserved');
+
+    if (/[^a-zA-Z0-9.]/.test(name)) return new MailboxError(MailboxError.BAD_FIELD, 'name can only contain alphanumerals and dot');
+
+    if (name.indexOf('.app') !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'alias pattern is reserved for apps');
+
+    return null;
+}
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    name = name.toLowerCase();
+
+    var error = validateName(name);
+    if (error) return callback(error);
+
+    mailboxdb.add(name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        debug('Added mailbox %s', name);
+
+        var mailbox = {
+            name: name
+        };
+
+        callback(null, mailbox);
+    });
+}
+
+function pushAlias(name, aliases, callback) {
+    if (process.env.BOX_ENV === 'test') return callback();
+
+    var cmd = [ '/addons/mail/service.sh', 'set-alias', name ].concat(aliases);
+
+    debug('pushing alias for %s : %j', name, aliases);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(new MailboxError(MailboxError.EXTERNAL_ERROR, error));
+
+        callback();
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    pushAlias(name, [ ], function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.del(name, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+            debug('deleted mailbox %s', name);
+
+            callback();
+        });
+    });
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.get(name, function (error, mailbox) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, mailbox);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAll(function (error, results) {
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    for (var i = 0; i < aliases.length; i++) {
+        aliases[i] = aliases[i].toLowerCase();
+
+        var error = validateName(aliases[i]);
+        if (error) return callback(error);
+    }
+
+    pushAlias(name, aliases, function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.setAliases(name, aliases, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS, error.message))
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+
+            callback(null);
+        });
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAliases(name, function (error, aliases) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, aliases);
+    });
+}
+
+// push aliases to the mail container on startup
+function setupAliases(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    getAll(function (error, mailboxes) {
+        if (error) return callback(error);
+
+        async.each(mailboxes, function (mailbox, iteratorDone) {
+            getAliases(mailbox.name, function (error, aliases) {
+                if (error) return iteratorDone(error);
+
+                if (aliases.length === 0) return iteratorDone();
+
+                pushAlias(mailbox.name, aliases, iteratorDone);
+            });
+        }, callback);
+    });
+}
+

src/mailer.js

@@ -12,7 +12,7 @@ exports = module.exports = {
     appUpdateAvailable: appUpdateAvailable,
 
     sendInvite: sendInvite,
-    sendCrashNotification: sendCrashNotification,
+    unexpectedExit: unexpectedExit,
 
     appDied: appDied,
 
@@ -24,6 +24,7 @@ exports = module.exports = {
     FEEDBACK_TYPE_TICKET: 'ticket',
     FEEDBACK_TYPE_APP_MISSING: 'app_missing',
     FEEDBACK_TYPE_APP_ERROR: 'app_error',
+    FEEDBACK_TYPE_UPGRADE_REQUEST: 'upgrade_request',
     sendFeedback: sendFeedback,
 
     _getMailQueue: _getMailQueue,
@@ -40,6 +41,7 @@ var assert = require('assert'),
     ejs = require('ejs'),
     nodemailer = require('nodemailer'),
     path = require('path'),
+    platform = require('./platform.js'),
     safe = require('safetydance'),
     smtpTransport = require('nodemailer-smtp-transport'),
     users = require('./user.js'),
@@ -112,7 +114,7 @@ function getTxtRecords(callback) {
 function checkDns() {
     getTxtRecords(function (error, records) {
         if (error || !records) {
-            debug('checkDns: DNS error or no records looking up TXT records for %s %s', config.adminFqdn(), error, records);
+            debug('checkDns: DNS error or no records looking up TXT records for %s %s', config.fqdn(), error, records);
             gCheckDnsTimerId = setTimeout(checkDns, 60000);
             return;
         }
@@ -153,15 +155,19 @@ function sendMails(queue) {
     docker.getContainer('mail').inspect(function (error, data) {
         if (error) return console.error(error);
 
-        var mailServerIp = safe.query(data, 'NetworkSettings.IPAddress');
+        var mailServerIp = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
         if (!mailServerIp) return debug('Error querying mail server IP');
 
         var transport = nodemailer.createTransport(smtpTransport({
             host: mailServerIp,
-            port: 2500 // this value comes from mail container
+            port: config.get('smtpPort'),
+            auth: {
+                user: platform.mailConfig().username,
+                pass: platform.mailConfig().password
+            }
         }));
 
-        debug('Processing mail queue of size %d (through %s:2500)', queue.length, mailServerIp);
+        debug('Processing mail queue of size %d (through %s:2525)', queue.length, mailServerIp);
 
         async.mapSeries(queue, function iterator(mailOptions, callback) {
             transport.sendMail(mailOptions, function (error) {
@@ -217,7 +223,7 @@ function mailUserEventToAdmins(user, event) {
         adminEmails = _.difference(adminEmails, [ user.email ]);
 
         var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s %s in Cloudron %s', user.username || user.email, event, config.fqdn()),
             text: render('user_event.ejs', { fqdn: config.fqdn(), user: user, event: event, format: 'text' }),
@@ -243,7 +249,7 @@ function sendInvite(user, invitor) {
     };
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: user.email,
         subject: util.format('Welcome to Cloudron %s', config.fqdn()),
         text: render('welcome_user.ejs', templateData)
@@ -266,7 +272,7 @@ function userAdded(user, inviteSent) {
         var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken;
 
         var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s added in Cloudron %s', user.email, config.fqdn()),
             text: render('user_added.ejs', { fqdn: config.fqdn(), user: user, inviteLink: inviteLink, format: 'text' }),
@@ -301,7 +307,7 @@ function passwordReset(user) {
     var resetLink = config.adminOrigin() + '/api/v1/session/password/reset.html?reset_token=' + user.resetToken;
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: user.email,
         subject: 'Password Reset Request',
         text: render('password_reset.ejs', { fqdn: config.fqdn(), user: user, resetLink: resetLink, format: 'text' })
@@ -319,7 +325,7 @@ function appDied(app) {
         if (error) return console.log('Error getting admins', error);
 
         var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.concat('support@cloudron.io').join(', '),
             subject: util.format('App %s is down', app.location),
             text: render('app_down.ejs', { fqdn: config.fqdn(), title: app.manifest.title, appFqdn: config.appFqdn(app.location), format: 'text' })
@@ -337,7 +343,7 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', config.fqdn()),
             text: render('box_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), newBoxVersion: newBoxVersion, changelog: changelog, format: 'text' })
@@ -355,7 +361,7 @@ function appUpdateAvailable(app, updateInfo) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', app.fqdn),
             text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, updateInfo: updateInfo, format: 'text' })
@@ -369,7 +375,7 @@ function outOfDiskSpace(message) {
     assert.strictEqual(typeof message, 'string');
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: 'admin@cloudron.io',
         subject: util.format('[%s] Out of disk space alert', config.fqdn()),
         text: render('out_of_disk_space.ejs', { fqdn: config.fqdn(), message: message, format: 'text' })
@@ -383,7 +389,7 @@ function certificateRenewed(domain, message) {
     assert.strictEqual(typeof message, 'string');
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: 'admin@cloudron.io',
         subject: util.format('[%s] Certificate was %s renewed', domain, message ? 'not' : ''),
         text: render('certificate_renewed.ejs', { domain: domain, message: message, format: 'text' })
@@ -394,15 +400,15 @@ function certificateRenewed(domain, message) {
 
 // this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized
 // crashnotifier should be able to send mail when there is no db
-function sendCrashNotification(program, context) {
+function unexpectedExit(program, context) {
     assert.strictEqual(typeof program, 'string');
     assert.strictEqual(typeof context, 'string');
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: 'admin@cloudron.io',
         subject: util.format('[%s] %s exited unexpectedly', config.fqdn(), program),
-        text: render('crash_notification.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
+        text: render('unexpected_exit.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
     };
 
     sendMails([ mailOptions ]);
@@ -417,10 +423,11 @@ function sendFeedback(user, type, subject, description) {
     assert(type === exports.FEEDBACK_TYPE_TICKET ||
         type === exports.FEEDBACK_TYPE_FEEDBACK ||
         type === exports.FEEDBACK_TYPE_APP_MISSING ||
+        type === exports.FEEDBACK_TYPE_UPGRADE_REQUEST ||
         type === exports.FEEDBACK_TYPE_APP_ERROR);
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: 'support@cloudron.io',
         subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
         text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})

src/middleware/cors.js

@@ -20,6 +20,7 @@ module.exports = function cors(options) {
         if (!requestOrigin) return next();
 
         requestOrigin = url.parse(requestOrigin);
+        if (!requestOrigin.host) return res.status(405).send('CORS not allowed from this domain');
 
         var hostname = requestOrigin.host.split(':')[0]; // remove any port
         var originAllowed = origins.some(function (o) { return o === '*' || o === hostname; });

src/middleware/index.js

@@ -4,7 +4,6 @@ exports = module.exports = {
     cookieParser: require('cookie-parser'),
     cors: require('./cors'),
     csrf: require('csurf'),
-    favicon: require('serve-favicon'),
     json: require('body-parser').json,
     morgan: require('morgan'),
     proxy: require('proxy-middleware'),

src/nginx.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 var assert = require('assert'),
@@ -47,7 +45,8 @@ function configureAdmin(certFilePath, keyFilePath, callback) {
         vhost: config.adminFqdn(),
         endpoint: 'admin',
         certFilePath: certFilePath,
-        keyFilePath: keyFilePath
+        keyFilePath: keyFilePath,
+        xFrameOptions: 'SAMEORIGIN'
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, 'admin.conf');
@@ -66,7 +65,7 @@ function configureApp(app, certFilePath, keyFilePath, callback) {
     var sourceDir = path.resolve(__dirname, '..');
     var oauthProxy = requiresOAuthProxy(app);
     var endpoint = oauthProxy ? 'oauthproxy' : 'app';
-    var vhost = config.appFqdn(app.location);
+    var vhost = app.altDomain || config.appFqdn(app.location);
 
     var data = {
         sourceDir: sourceDir,
@@ -75,15 +74,16 @@ function configureApp(app, certFilePath, keyFilePath, callback) {
         port: app.httpPort,
         endpoint: endpoint,
         certFilePath: certFilePath,
-        keyFilePath: keyFilePath
+        keyFilePath: keyFilePath,
+        xFrameOptions: app.xFrameOptions || 'SAMEORIGIN'    // once all apps have been updated/
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-    debug('writing config for "%s" to %s', app.location, nginxConfigFilename);
+    debug('writing config for "%s" to %s', vhost, nginxConfigFilename);
 
     if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
-        debug('Error creating nginx config for "%s" : %s', app.location, safe.error.message);
+        debug('Error creating nginx config for "%s" : %s', vhost, safe.error.message);
         return callback(safe.error);
     }
 
@@ -94,9 +94,11 @@ function unconfigureApp(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    var vhost = app.altDomain || config.appFqdn(app.location);
+
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
     if (!safe.fs.unlinkSync(nginxConfigFilename)) {
-        debug('Error removing nginx configuration of "%s": %s', app.location, safe.error.message);
+        debug('Error removing nginx configuration of "%s": %s', vhost, safe.error.message);
         return callback(null);
     }
 

src/oauth2views/account_setup.ejs

@@ -8,7 +8,10 @@
 
 // very basic angular app
 var app = angular.module('Application', []);
-app.controller('Controller', [function () {}]);
+app.controller('Controller', ['$scope', function ($scope) {
+    $scope.username = '<%= (user && user.username) ? user.username : '' %>';
+    $scope.displayName = '<%= (user && user.displayName) ? user.displayName : '' %>';
+}]);
 
 </script>
 
@@ -28,6 +31,12 @@ app.controller('Controller', [function () {}]);
 
                 <center><p class="has-error"><%= error %></p></center>
 
+<% if (user && user.username) { %>
+                <div class="form-group"">
+                    <label class="control-label">Username</label>
+                    <input type="text" class="form-control" ng-model="username" name="username" readonly required>
+                </div>
+<% } else { %>
                 <div class="form-group" ng-class="{ 'has-error': (setupForm.username.$dirty && setupForm.username.$invalid) }">
                     <label class="control-label">Username</label>
                     <div class="control-label" ng-show="setupForm.username.$dirty && setupForm.username.$invalid">
@@ -37,6 +46,7 @@ app.controller('Controller', [function () {}]);
                     </div>
                     <input type="text" class="form-control" ng-model="username" name="username" ng-maxlength="512" ng-minlength="3" required autofocus>
                 </div>
+<% } %>
 
                 <div class="form-group">
                     <label class="control-label">Display Name</label>

src/oauthproxy.js

@@ -7,7 +7,7 @@ exports = module.exports = {
 
 var appdb = require('./appdb.js'),
     assert = require('assert'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:proxy'),
@@ -124,7 +124,7 @@ function authenticate(req, res, next) {
                 return res.send(500, 'Unknown app.');
             }
 
-            clientdb.getByAppIdAndType(result.id, clientdb.TYPE_PROXY, function (error, result) {
+            clients.getByAppIdAndType(result.id, clients.TYPE_PROXY, function (error, result) {
                 if (error) {
                     console.error('Unknown OAuth client.', error);
                     return res.send(500, 'Unknown OAuth client.');

src/paths.js

@@ -27,5 +27,7 @@ exports = module.exports = {
     UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json'),
 
     ACME_CHALLENGES_DIR: path.join(config.baseDir(), 'data/acme'),
-    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key')
+    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key'),
+
+    INFRA_VERSION_FILE: path.join(config.baseDir(), 'data/INFRA_VERSION')
 };

src/platform.js

@@ -0,0 +1,312 @@
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize,
+
+    events: new (require('events').EventEmitter)(),
+    EVENT_READY: 'ready',
+
+    isReadySync: isReadySync,
+
+    mailConfig: mailConfig
+};
+
+var apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    certificates = require('./certificates.js'),
+    debug = require('debug')('box:platform'),
+    fs = require('fs'),
+    hat = require('hat'),
+    infra = require('./infra_version.js'),
+    ini = require('ini'),
+    mailboxes = require('./mailboxes.js'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
+    util = require('util'),
+    _ = require('underscore');
+
+var gAddonVars = null,
+    gPlatformReadyTimer = null;
+
+function initialize(callback) {
+    if (process.env.BOX_ENV === 'test' && !process.env.CREATE_INFRA) return callback();
+
+    debug('initializing addon infrastructure');
+
+    var existingInfra = { version: 'none' };
+    if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
+        existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8'));
+        if (!existingInfra) existingInfra = { version: 'corrupt' };
+    }
+
+    // short-circuit for the restart case
+    if (_.isEqual(infra, existingInfra)) {
+        debug('platform is uptodate at version %s', infra.version);
+        process.nextTick(function () { exports.events.emit(exports.EVENT_READY); });
+        return loadAddonVars(callback);
+    }
+
+    debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version);
+
+    async.series([
+        stopContainers.bind(null, existingInfra),
+        createDockerNetwork,
+        startAddons.bind(null, existingInfra),
+        removeOldImages,
+        startApps.bind(null, existingInfra),
+        loadAddonVars,
+        fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra))
+    ], function (error) {
+        if (error) return callback(error);
+
+        // give 30 seconds for the platform to "settle". For example, mysql might still be initing the
+        // database dir and we cannot call service scripts until that's done.
+        // TODO: make this smarter to not wait for 30secs for the crash-restart case
+        gPlatformReadyTimer = setTimeout(function () {
+            debug('emitting platform ready');
+            gPlatformReadyTimer = null;
+            exports.events.emit(exports.EVENT_READY);
+        }, 30000);
+
+        callback();
+    });
+}
+
+function uninitialize(callback) {
+    clearTimeout(gPlatformReadyTimer);
+    gPlatformReadyTimer = null;
+    callback();
+}
+
+function isReadySync() {
+    return gPlatformReadyTimer === null;
+}
+
+function removeOldImages(callback) {
+    debug('removing old addon images');
+
+    for (var imageName in infra.images) {
+        var image = infra.images[imageName];
+        debug('cleaning up images of %j', image);
+        var cmd = 'docker images "%s" | tail -n +2 | awk \'{ print $1 ":" $2 }\' | grep -v "%s" | xargs --no-run-if-empty docker rmi';
+        shell.execSync('removeOldImagesSync', util.format(cmd, image.repo, image.tag));
+    }
+
+    callback();
+}
+
+function stopContainers(existingInfra, callback) {
+    // TODO: be nice and stop addons cleanly (example, shutdown commands)
+
+    if (existingInfra.version !== infra.version) { // infra upgrade
+        debug('stopping all containers for infra upgrade');
+        shell.execSync('stopContainers', 'docker ps -qa | xargs --no-run-if-empty docker rm -f');
+    } else {
+        assert(typeof infra.images, 'object');
+        var changedAddons = [ ];
+        for (var imageName in infra.images) {
+            if (infra.images[imageName].tag !== existingInfra.images[imageName].tag) changedAddons.push(imageName);
+        }
+
+        debug('stopping addons for incremental infra update: %j', changedAddons);
+        shell.execSync('stopContainers', 'docker rm -f ' + changedAddons.join(' '));
+    }
+
+    callback();
+}
+
+function createDockerNetwork(callback) {
+    shell.execSync('createDockerNetwork', 'docker network create --subnet=172.18.0.0/16 cloudron || true', callback);
+}
+
+function startGraphite(callback) {
+    const tag = infra.images.graphite.tag;
+    const dataDir = paths.DATA_DIR;
+
+    const cmd = `docker run --restart=always -d --name="graphite" \
+                --net cloudron \
+                --net-alias graphite \
+                -m 75m \
+                --memory-swap 150m \
+                -p 127.0.0.1:2003:2003 \
+                -p 127.0.0.1:2004:2004 \
+                -p 127.0.0.1:8000:8000 \
+                -v "${dataDir}/graphite:/app/data" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startGraphite', cmd);
+
+    callback();
+}
+
+function startMysql(callback) {
+    const tag = infra.images.mysql.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/mysql_vars.sh', 
+            'MYSQL_ROOT_PASSWORD=' + rootPassword +'\nMYSQL_ROOT_HOST=172.18.0.1', 'utf8')) {
+        return callback(new Error('Could not create mysql var file:' + safe.error.message));
+    }
+
+    const cmd = `docker run --restart=always -d --name="mysql" \
+                --net cloudron \
+                --net-alias mysql \
+                -m 256m \
+                --memory-swap 512m \
+                -v "${dataDir}/mysql:/var/lib/mysql" \
+                -v "${dataDir}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startMysql', cmd);
+
+    callback();
+}
+
+function startPostgresql(callback) {
+    const tag = infra.images.postgresql.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/postgresql_vars.sh', 'POSTGRESQL_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
+        return callback(new Error('Could not create postgresql var file:' + safe.error.message));
+    }
+
+    const cmd = `docker run --restart=always -d --name="postgresql" \
+                --net cloudron \
+                --net-alias postgresql \
+                -m 100m \
+                --memory-swap 200m \
+                -v "${dataDir}/postgresql:/var/lib/postgresql" \
+                -v "${dataDir}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startPostgresql', cmd);
+
+    callback();
+}
+
+function startMongodb(callback) {
+    const tag = infra.images.mongodb.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/mongodb_vars.sh', 'MONGODB_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
+        return callback(new Error('Could not create mongodb var file:' + safe.error.message));
+    }
+
+    const cmd = `docker run --restart=always -d --name="mongodb" \
+                --net cloudron \
+                --net-alias mongodb \
+                -m 100m \
+                --memory-swap 200m \
+                -v "${dataDir}/mongodb:/var/lib/mongodb" \
+                -v "${dataDir}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startMongodb', cmd);
+
+    callback();
+}
+
+function startMail(callback) {
+    // mail (note: 2525 is hardcoded in mail container and app use this port)
+    // MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
+    // MAIL_DOMAIN is the domain for which this server is relaying mails
+    // mail container uses /app/data for backed up data and /run for restart-able data
+
+    const tag = infra.images.mail.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+    const fqdn = config.fqdn();
+    const mailFqdn = config.adminFqdn();
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/mail_vars.sh',
+            'MAIL_ROOT_USERNAME=no-reply\nMAIL_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
+        return callback(new Error('Could not create mail var file:' + safe.error.message));
+    }
+
+    certificates.getAdminCertificatePath(function (error, certFilePath, keyFilePath) {
+        if (error) return callback(error);
+
+        const cmd = `docker run --restart=always -d --name="mail" \
+                    --net cloudron \
+                    --net-alias mail \
+                    -m 75m \
+                    --memory-swap 150m \
+                    -e "MAIL_DOMAIN=${fqdn}" \
+                    -e "MAIL_SERVER_NAME=${mailFqdn}" \
+                    -v "${dataDir}/box/mail:/app/data" \
+                    -v "${dataDir}/mail:/run" \
+                    -v "${dataDir}/addons/mail_vars.sh:/etc/mail/mail_vars.sh:ro" \
+                    -v "${certFilePath}:/etc/tls_cert.pem:ro" \
+                    -v "${keyFilePath}:/etc/tls_key.pem:ro" \
+                    -p 587:2525 \
+                    -p 993:9993 \
+                    -p 4190:4190 \
+                    -p 25:2525 \
+                    --read-only -v /tmp ${tag}`;
+
+        shell.execSync('startMail', cmd);
+
+        mailboxes.setupAliases(callback);
+    });
+}
+
+function startAddons(existingInfra, callback) {
+    var startFuncs = [ ];
+
+    if (existingInfra.version !== infra.version) {
+        debug('startAddons: no existing infra or infra upgrade. starting all addons');
+        startFuncs.push(startGraphite, startMysql, startPostgresql, startMongodb, startMail);
+    } else {
+        assert.strictEqual(typeof existingInfra.images, 'object');
+
+        if (infra.images.graphite.tag !== existingInfra.images.graphite.tag) startFuncs.push(startGraphite);
+        if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) startFuncs.push(startMysql);
+        if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) startFuncs.push(startPostgresql);
+        if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) startFuncs.push(startMongodb);
+        if (infra.images.mail.tag !== existingInfra.images.mail.tag) startFuncs.push(startMail);
+
+        debug('startAddons: existing infra. incremental addon create %j', startFuncs.map(function (f) { return f.name; }));
+    }
+
+    async.series(startFuncs, callback);
+}
+
+function startApps(existingInfra, callback) {
+    if (existingInfra.version === infra.version) {
+        debug('startApp: apps are already uptodate');
+        callback();
+    } else if (existingInfra.version === 'none') {
+        debug('startApps: restoring installed apps');
+        apps.restoreInstalledApps(callback);
+    } else {
+        debug('startApps: reconfiguring installed apps');
+        apps.configureInstalledApps(callback);
+    }
+}
+
+function loadAddonVars(callback) {
+    gAddonVars = {
+        mail: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mail_vars.sh', 'utf8')),
+        postgresql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/postgresql_vars.sh', 'utf8')),
+        mysql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mysql_vars.sh', 'utf8')),
+        mongodb: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mongodb_vars.sh', 'utf8'))
+    };
+    callback();
+}
+
+function mailConfig() {
+    if (!gAddonVars) return { username: 'no-reply', from: 'no-reply@' + config.fqdn(), password: 'doesnotwork' }; // for tests which don't run infra
+
+    return {
+        username: gAddonVars.mail.MAIL_ROOT_USERNAME,
+        from: '"Cloudron" <' + gAddonVars.mail.MAIL_ROOT_USERNAME + '@' + config.fqdn() + '>',
+        password: gAddonVars.mail.MAIL_ROOT_PASSWORD
+    };
+}

src/progress.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -8,7 +6,8 @@ exports = module.exports = {
     get: get,
 
     UPDATE: 'update',
-    BACKUP: 'backup'
+    BACKUP: 'backup',
+    MIGRATE: 'migrate'
 };
 
 var assert = require('assert'),
@@ -18,12 +17,13 @@ var assert = require('assert'),
 // otherwise no such operation is currently ongoing
 var progress = {
     update: null,
-    backup: null
+    backup: null,
+    migrate: null
 };
 
 // We use -1 for percentage to indicate errors
 function set(tag, percent, message) {
-    assert(tag === exports.UPDATE || tag === exports.BACKUP);
+    assert.strictEqual(typeof tag, 'string');
     assert.strictEqual(typeof percent, 'number');
     assert.strictEqual(typeof message, 'string');
 
@@ -36,7 +36,7 @@ function set(tag, percent, message) {
 }
 
 function clear(tag) {
-    assert(tag === exports.UPDATE || tag === exports.BACKUP);
+    assert.strictEqual(typeof tag, 'string');
 
     progress[tag] = null;
 

src/routes/apps.js

@@ -1,10 +1,7 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
     getApp: getApp,
-    getAppBySubdomain: getAppBySubdomain,
     getApps: getApps,
     getAppIcon: getAppIcon,
     installApp: installApp,
@@ -19,7 +16,9 @@ exports = module.exports = {
 
     stopApp: stopApp,
     startApp: startApp,
-    exec: exec
+    exec: exec,
+
+    cloneApp: cloneApp
 };
 
 var apps = require('../apps.js'),
@@ -31,8 +30,12 @@ var apps = require('../apps.js'),
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     paths = require('../paths.js'),
     safe = require('safetydance'),
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');
+
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
 
 function removeInternalAppFields(app) {
     return {
@@ -49,7 +52,9 @@ function removeInternalAppFields(app) {
         portBindings: app.portBindings,
         iconUrl: app.iconUrl,
         fqdn: app.fqdn,
-        memoryLimit: app.memoryLimit
+        memoryLimit: app.memoryLimit,
+        altDomain: app.altDomain,
+        xFrameOptions: app.xFrameOptions
     };
 }
 
@@ -64,17 +69,6 @@ function getApp(req, res, next) {
     });
 }
 
-function getAppBySubdomain(req, res, next) {
-    assert.strictEqual(typeof req.params.subdomain, 'string');
-
-    apps.getBySubdomain(req.params.subdomain, function (error, app) {
-        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such subdomain'));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(200, removeInternalAppFields(app)));
-    });
-}
-
 function getApps(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
@@ -98,80 +92,77 @@ function getAppIcon(req, res, next) {
     });
 }
 
-/*
- * Installs an app
- * @bodyparam {string} appStoreId The id of the app to be installed
- * @bodyparam {manifest} manifest The app manifest
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from environment variable name to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- * @bodyparam {icon} icon Base64 encoded image
- */
 function installApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId is required'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    // required
     if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
+
+    // optional
+    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+
     if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
 
-    // allow tests to provide an appId for testing
-    var appId = (process.env.BOX_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
+    // falsy value in altDomain unsets it
+    if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
 
-    debug('Installing app id:%s storeid:%s loc:%s port:%j accessRestriction:%j memoryLimit:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.memoryLimit, data.manifest);
+    if (data.xFrameOptions && typeof data.xFrameOptions !== 'string') return next(new HttpError(400, 'xFrameOptions must be a string'));
 
-    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.icon || null, data.cert || null, data.key || null, data.memoryLimit || 0, function (error) {
+    debug('Installing app id:%s data:%j', data);
+
+    apps.install(data, auditSource(req), function (error, app) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BILLING_REQUIRED) return next(new HttpError(402, 'Billing required'));
-        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_CERTIFICATE) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.USER_REQUIRED) return next(new HttpError(400, 'accessRestriction must specify one user'));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(202, { id: appId } ));
+        next(new HttpSuccess(202, app));
     });
 }
 
-/*
- * Configure an app
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from env to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- */
 function configureApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
-    if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+    if ('location' in data && typeof data.location !== 'string') return next(new HttpError(400, 'location must be string'));
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    if ('accessRestriction' in data && typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction must be an object'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+
     if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
+    if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
+    if (data.xFrameOptions && typeof data.xFrameOptions !== 'string') return next(new HttpError(400, 'xFrameOptions must be a string'));
 
-    debug('Configuring app id:%s location:%s bindings:%j accessRestriction:%j memoryLimit:%s', req.params.id, data.location, data.portBindings, data.accessRestriction, data.memoryLimit);
+    debug('Configuring app id:%s data:%j', req.params.id, data);
 
-    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.cert || null, data.key || null, data.memoryLimit || 0, function (error) {
+    apps.configure(req.params.id, data, auditSource(req), function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
@@ -186,20 +177,50 @@ function configureApp(req, res, next) {
 }
 
 function restoreApp(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
 
+    var data = req.body;
+
     debug('Restore app id:%s', req.params.id);
 
-    apps.restore(req.params.id, function (error) {
+    if (!('backupId' in req.body)) return next(new HttpError(400, 'backupId is required'));
+    if (data.backupId !== null && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
+
+    apps.restore(req.params.id, data, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(424, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, { }));
     });
 }
 
+function cloneApp(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    var data = req.body;
+
+    debug('Clone app id:%s', req.params.id);
+
+    if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be a string'));
+    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
+    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+
+    apps.clone(req.params.id, data, auditSource(req), function (error, result) {
+        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
+        if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(424, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(201, { id: result.id }));
+    });
+}
+
 function backupApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
@@ -215,16 +236,12 @@ function backupApp(req, res, next) {
     });
 }
 
-/*
- * Uninstalls an app
- * @bodyparam {string} id The id of the app to be uninstalled
- */
 function uninstallApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
     debug('Uninstalling app id:%s', req.params.id);
 
-    apps.uninstall(req.params.id, function (error) {
+    apps.uninstall(req.params.id, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error) return next(new HttpError(500, error));
 
@@ -266,15 +283,18 @@ function updateApp(req, res, next) {
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
     debug('Update app id:%s to manifest:%j with portBindings:%j', req.params.id, data.manifest, data.portBindings);
 
-    apps.update(req.params.id, data.force || false, data.manifest, data.portBindings, data.icon, function (error) {
+    apps.update(req.params.id, req.body, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
@@ -344,15 +364,37 @@ function getLogs(req, res, next) {
     });
 }
 
+function demuxStream(stream, stdin) {
+    var header = null;
+
+    stream.on('readable', function() {
+        header = header || stream.read(4);
+
+        while (header !== null) {
+            var length = header.readUInt32BE(0);
+            if (length === 0) {
+                header = null;
+                return stdin.end(); // EOF
+            }
+
+            var payload = stream.read(length);
+
+            if (payload === null) break;
+            stdin.write(payload);
+            header = stream.read(4);
+        }
+    });
+}
+
 function exec(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
-    debug('Execing into app id:%s', req.params.id);
+    debug('Execing into app id:%s and cmd:%s', req.params.id, req.query.cmd);
 
     var cmd = null;
     if (req.query.cmd) {
         cmd = safe.JSON.parse(req.query.cmd);
-        if (!util.isArray(cmd) && cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
+        if (!util.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
     }
 
     var columns = req.query.columns ? parseInt(req.query.columns, 10) : null;
@@ -373,8 +415,16 @@ function exec(req, res, next) {
         req.clearTimeout();
         res.sendUpgradeHandshake();
 
+        // When tty is disabled, the duplexStream has 2 separate streams. When enabled, it has stdout/stderr merged.
         duplexStream.pipe(res.socket);
-        res.socket.pipe(duplexStream);
+
+        if (tty) {
+            res.socket.pipe(duplexStream); // in tty mode, the client always waits for server to exit
+        } else {
+            demuxStream(res.socket, duplexStream);
+            res.socket.on('error', function () { duplexStream.end(); });
+            res.socket.on('end', function () { duplexStream.end(); });
+        }
     });
 }
 

src/routes/backups.js

@@ -1,21 +1,22 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
     get: get,
     create: create,
-    download: download
+    createDownloadUrl: createDownloadUrl
 };
 
 var assert = require('assert'),
     backups = require('../backups.js'),
     BackupsError = require('../backups.js').BackupsError,
-    cloudron = require('../cloudron.js'),
-    CloudronError = require('../cloudron.js').CloudronError,
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 function get(req, res, next) {
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
     if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
@@ -34,15 +35,15 @@ function get(req, res, next) {
 function create(req, res, next) {
     // note that cloudron.backup only waits for backup initiation and not for backup to complete
     // backup progress can be checked up ny polling the progress api call
-    cloudron.backup(function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+    backups.backup(auditSource(req), function (error) {
+        if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
     });
 }
 
-function download(req, res, next) {
+function createDownloadUrl(req, res, next) {
     assert.strictEqual(typeof req.params.backupId, 'string');
 
     backups.getRestoreUrl(req.params.backupId, function (error, result) {
@@ -51,4 +52,3 @@ function download(req, res, next) {
         next(new HttpSuccess(200, result));
     });
 }
-

src/routes/clients.js

@@ -1,21 +1,19 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
     add: add,
     get: get,
     del: del,
-    getAllByUserId: getAllByUserId,
+    getAll: getAll,
+    addClientToken: addClientToken,
     getClientTokens: getClientTokens,
-    delClientTokens: delClientTokens
+    delClientTokens: delClientTokens,
+    delToken: delToken
 };
 
 var assert = require('assert'),
-    clientdb = require('../clientdb.js'),
     clients = require('../clients.js'),
     ClientsError = clients.ClientsError,
-    DatabaseError = require('../databaseerror.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     validUrl = require('valid-url');
@@ -29,8 +27,9 @@ function add(req, res, next) {
     if (typeof data.scope !== 'string' || !data.scope) return next(new HttpError(400, 'scope is required'));
     if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));
 
-    clients.add(data.appId, clientdb.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
-        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, 'Invalid scope'));
+    clients.add(data.appId, clients.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
+        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, error.message));
+        if (error && error.reason === ClientsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(201, result));
     });
@@ -40,7 +39,7 @@ function get(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
 
     clients.get(req.params.clientId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'No such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, result));
     });
@@ -49,26 +48,49 @@ function get(req, res, next) {
 function del(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
 
-    clients.del(req.params.clientId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+    clients.get(req.params.clientId, function (error, result) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
-        next(new HttpSuccess(204, result));
+
+        // we do not allow to use the REST API to delete addon clients
+        if (result.type !== clients.TYPE_EXTERNAL) return next(new HttpError(405, 'Deleting app addon clients is not allowed.'));
+
+        clients.del(req.params.clientId, function (error, result) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+            if (error && error.reason === ClientsError.NOT_ALLOWED) return next(new HttpError(405, error.message));
+            if (error) return next(new HttpError(500, error));
+            next(new HttpSuccess(204, result));
+        });
     });
 }
 
-function getAllByUserId(req, res, next) {
-    clients.getAllWithDetailsByUserId(req.user.id, function (error, result) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return next(new HttpError(500, error));
+function getAll(req, res, next) {
+    clients.getAll(function (error, result) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { clients: result }));
     });
 }
 
+function addClientToken(req, res, next) {
+    assert.strictEqual(typeof req.params.clientId, 'string');
+    assert.strictEqual(typeof req.user, 'object');
+
+    var expiresAt = req.query.expiresAt ? parseInt(req.query.expiresAt, 10) : Date.now() + 24 * 60 * 60 * 1000; // default 1 day;
+    if (isNaN(expiresAt) || expiresAt <= Date.now()) return next(new HttpError(400, 'expiresAt must be a timestamp in the future'));
+
+    clients.addClientTokenByUserId(req.params.clientId, req.user.id, expiresAt, function (error, result) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+        next(new HttpSuccess(201, { token: result }));
+    });
+}
+
 function getClientTokens(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
     assert.strictEqual(typeof req.user, 'object');
 
     clients.getClientTokensByUserId(req.params.clientId, req.user.id, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { tokens: result }));
     });
@@ -79,8 +101,22 @@ function delClientTokens(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
     clients.delClientTokensByUserId(req.params.clientId, req.user.id, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(204));
     });
 }
+
+function delToken(req, res, next) {
+    assert.strictEqual(typeof req.params.clientId, 'string');
+    assert.strictEqual(typeof req.params.tokenId, 'string');
+    assert.strictEqual(typeof req.user, 'object');
+
+    clients.delToken(req.params.clientId, req.params.tokenId, function (error) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error && error.reason === ClientsError.INVALID_TOKEN) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}

src/routes/cloudron.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -7,22 +5,32 @@ exports = module.exports = {
     setupTokenAuth: setupTokenAuth,
     getStatus: getStatus,
     reboot: reboot,
+    migrate: migrate,
     getProgress: getProgress,
     getConfig: getConfig,
     update: update,
-    feedback: feedback
+    feedback: feedback,
+    checkForUpdates: checkForUpdates
 };
 
 var assert = require('assert'),
+    async = require('async'),
     cloudron = require('../cloudron.js'),
-    config = require('../config.js'),
-    progress = require('../progress.js'),
-    mailer = require('../mailer.js'),
     CloudronError = cloudron.CloudronError,
+    config = require('../config.js'),
     debug = require('debug')('box:routes/cloudron'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
-    superagent = require('superagent');
+    progress = require('../progress.js'),
+    mailer = require('../mailer.js'),
+    superagent = require('superagent'),
+    updateChecker = require('../updatechecker.js'),
+    _ = require('underscore');
+
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
 
 /**
  * Creating an admin user and activate the cloudron.
@@ -50,11 +58,9 @@ function activate(req, res, next) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
     debug('activate: username:%s ip:%s', username, ip);
 
-    cloudron.activate(username, password, email, displayName, ip, function (error, info) {
+    cloudron.activate(username, password, email, displayName, ip, auditSource(req), function (error, info) {
         if (error && error.reason === CloudronError.ALREADY_PROVISIONED) return next(new HttpError(409, 'Already setup'));
-        if (error && error.reason === CloudronError.BAD_USERNAME) return next(new HttpError(400, 'Bad username'));
-        if (error && error.reason === CloudronError.BAD_PASSWORD) return next(new HttpError(400, 'Bad password'));
-        if (error && error.reason === CloudronError.BAD_EMAIL) return next(new HttpError(400, 'Bad email'));
+        if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         // only in caas case do we have to notify the api server about activation
@@ -109,6 +115,31 @@ function reboot(req, res, next) {
     cloudron.reboot();
 }
 
+function migrate(req, res, next) {
+    if (config.provider() !== 'caas') return next(new HttpError(422, 'Cannot use migrate API with this provider'));
+
+    if ('size' in req.body && typeof req.body.size !== 'string') return next(new HttpError(400, 'size must be string'));
+    if ('region' in req.body && typeof req.body.region !== 'string') return next(new HttpError(400, 'region must be string'));
+
+    if ('domain' in req.body) {
+        if (typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be string'));
+        if (typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider must be string'));
+    }
+
+    debug('Migration requested domain:%s size:%s region:%s', req.body.domain, req.body.size, req.body.region);
+
+    var options = _.pick(req.body, 'domain', 'size', 'region');
+    if (Object.keys(options).length === 0) return next(new HttpError(400, 'no migrate option provided'));
+
+    cloudron.migrate(req.body, function (error) { // pass req.body because 'domain' can have arbitrary options
+        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
 function getConfig(req, res, next) {
     cloudron.getConfig(function (error, cloudronConfig) {
         if (error) return next(new HttpError(500, error));
@@ -119,22 +150,33 @@ function getConfig(req, res, next) {
 
 function update(req, res, next) {
     // this only initiates the update, progress can be checked via the progress route
-    cloudron.updateToLatest(function (error) {
+    cloudron.updateToLatest(auditSource(req), function (error) {
         if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
         if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === CloudronError.SELF_UPGRADE_NOT_SUPPORTED) return next(new HttpError(412, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
     });
 }
 
+function checkForUpdates(req, res, next) {
+    async.series([
+        updateChecker.checkAppUpdates,
+        updateChecker.checkBoxUpdates
+    ], function () {
+        next(new HttpSuccess(200, { update: updateChecker.getUpdateInfo() }));
+    });
+}
+
 function feedback(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
     if (req.body.type !== mailer.FEEDBACK_TYPE_FEEDBACK &&
         req.body.type !== mailer.FEEDBACK_TYPE_TICKET &&
         req.body.type !== mailer.FEEDBACK_TYPE_APP_MISSING &&
-        req.body.type !== mailer.FEEDBACK_TYPE_APP_ERROR) return next(new HttpError(400, 'type must be either "ticket", "feedback" or "app_missing" or "app_error"'));
+        req.body.type !== mailer.FEEDBACK_TYPE_UPGRADE_REQUEST &&
+        req.body.type !== mailer.FEEDBACK_TYPE_APP_ERROR) return next(new HttpError(400, 'type must be either "ticket", "feedback", "app_missing", "app_error" or "upgrade_request"'));
     if (typeof req.body.subject !== 'string' || !req.body.subject) return next(new HttpError(400, 'subject must be string'));
     if (typeof req.body.description !== 'string' || !req.body.description) return next(new HttpError(400, 'description must be string'));
 

src/routes/developer.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -15,8 +13,13 @@ var developer = require('../developer.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 function enabled(req, res, next) {
-    developer.enabled(function (error, enabled) {
+    developer.isEnabled(function (error, enabled) {
         if (enabled) return next();
         next(new HttpError(412, 'Developer mode not enabled'));
     });
@@ -25,8 +28,9 @@ function enabled(req, res, next) {
 function setEnabled(req, res, next) {
     if (typeof req.body.enabled !== 'boolean') return next(new HttpError(400, 'enabled must be boolean'));
 
-    developer.setEnabled(req.body.enabled, function (error) {
+    developer.setEnabled(req.body.enabled, auditSource(req), function (error) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200, {}));
     });
 }
@@ -40,7 +44,7 @@ function login(req, res, next) {
         if (error) return next(new HttpError(500, error));
         if (!user) return next(new HttpError(401, 'Invalid credentials'));
 
-        developer.issueDeveloperToken(user, function (error, result) {
+        developer.issueDeveloperToken(user, auditSource(req), function (error, result) {
             if (error) return next(new HttpError(500, error));
 
             next(new HttpSuccess(200, { token: result.token, expiresAt: result.expiresAt }));
@@ -53,4 +57,4 @@ function apps(req, res, next) {
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { apps: result }));
     });
-}
+}

src/routes/eventlog.js

@@ -0,0 +1,26 @@
+'use strict';
+
+exports = module.exports = {
+    get: get
+};
+
+var eventlog = require('../eventlog.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess;
+
+function get(req, res, next) {
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    if (req.query.action && typeof req.query.action !== 'string') return next(new HttpError(400, 'action must be a string'));
+    if (req.query.search && typeof req.query.search !== 'string') return next(new HttpError(400, 'search must be a string'));
+
+    eventlog.getAllPaged(req.query.action || null, req.query.search || null, page, perPage, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { eventlogs: result }));
+    });
+}

src/routes/groups.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -22,7 +20,7 @@ function create(req, res, next) {
     if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
 
     groups.create(req.body.name, function (error, group) {
-        if (error && error.reason === GroupError.BAD_NAME) return next(new HttpError(400, error.message));
+        if (error && error.reason === GroupError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === GroupError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
         if (error) return next(new HttpError(500, error));
 
@@ -47,7 +45,7 @@ function get(req, res, next) {
 }
 
 function list(req, res, next) {
-    groups.getAll(function (error, result) {
+    groups.getAllWithMembers(function (error, result) {
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(200, { groups: result }));

src/routes/index.js

@@ -6,10 +6,13 @@ exports = module.exports = {
     clients: require('./clients.js'),
     cloudron: require('./cloudron.js'),
     developer: require('./developer.js'),
+    eventlog: require('./eventlog.js'),
     graphs: require('./graphs.js'),
     groups: require('./groups.js'),
-    internal: require('./internal.js'),
+    mailboxes: require('./mailboxes.js'),
     oauth2: require('./oauth2.js'),
+    profile: require('./profile.js'),
+    sysadmin: require('./sysadmin.js'),
     settings: require('./settings.js'),
     user: require('./user.js')
 };

src/routes/mailboxes.js

@@ -0,0 +1,91 @@
+'use strict';
+
+exports = module.exports = {
+    list: list,
+    get: get,
+    remove: remove,
+    create: create,
+    setAliases: setAliases,
+    getAliases: getAliases
+};
+
+var assert = require('assert'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    mailboxes = require('../mailboxes.js'),
+    MailboxError = mailboxes.MailboxError,
+    util = require('util');
+
+function create(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+
+    mailboxes.add(req.body.name, function (error, mailbox) {
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(201, mailbox));
+    });
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.get(req.params.mailboxId, function (error, result) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function list(req, res, next) {
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { mailboxes: result }));
+    });
+}
+
+function remove(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.del(req.params.mailboxId, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'Mailbox not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function setAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    if (!util.isArray(req.body.aliases)) return next(new HttpError(400, 'aliases must be an array'));
+
+    for (var i = 0; i < req.body.aliases.length; i++) {
+        if (typeof req.body.aliases[i] !== 'string') return next(new HttpError(400, 'alias must be a string'));
+    }
+
+    mailboxes.setAliases(req.params.mailboxId, req.body.aliases, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'One or more alias already exist'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
+function getAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.getAliases(req.params.mailboxId, function (error, aliases) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { aliases: aliases }));
+    });
+}

src/routes/oauth2.js

@@ -1,33 +1,37 @@
-/* jslint  node:true */
-
 'use strict';
 
-var assert = require('assert'),
+var appdb = require('../appdb'),
     apps = require('../apps'),
+    assert = require('assert'),
     authcodedb = require('../authcodedb'),
-    clientdb = require('../clientdb'),
+    clients = require('../clients'),
+    ClientsError = clients.ClientsError,
     config = require('../config.js'),
-    constants = require('../constants.js'),
     DatabaseError = require('../databaseerror'),
     debug = require('debug')('box:routes/oauth2'),
+    eventlog = require('../eventlog.js'),
+    hat = require('hat'),
     HttpError = require('connect-lastmile').HttpError,
     middleware = require('../middleware/index.js'),
     oauth2orize = require('oauth2orize'),
     passport = require('passport'),
     querystring = require('querystring'),
-    util = require('util'),
     session = require('connect-ensure-login'),
     tokendb = require('../tokendb'),
-    appdb = require('../appdb'),
     url = require('url'),
     user = require('../user.js'),
     UserError = user.UserError,
-    hat = require('hat');
+    util = require('util'),
+    _ = require('underscore');
+
+function auditSource(req, appId) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { authType: 'oauth', ip: ip, appId: appId };
+}
 
 // create OAuth 2.0 server
 var gServer = oauth2orize.createServer();
 
-
 // Register serialialization and deserialization functions.
 //
 // The client id is stored in the session and can thus be retrieved for each
@@ -38,7 +42,7 @@ gServer.serializeClient(function (client, callback) {
 });
 
 gServer.deserializeClient(function (id, callback) {
-    clientdb.get(id, callback);
+    clients.get(id, callback);
 });
 
 
@@ -57,7 +61,7 @@ gServer.grant(oauth2orize.grant.code({ scopeSeparator: ',' }, function (client,
     var code = hat(256);
     var expiresAt = Date.now() + 60 * 60000; // 1 hour
 
-    authcodedb.add(code, client.id, user.username, expiresAt, function (error) {
+    authcodedb.add(code, client.id, user.id, expiresAt, function (error) {
         if (error) return callback(error);
 
         debug('grant code: new auth code for client %s code %s', client.id, code);
@@ -73,7 +77,7 @@ gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client,
     var token = tokendb.generateToken();
     var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-    tokendb.add(token, tokendb.PREFIX_USER + user.id, client.id, expires, client.scope, function (error) {
+    tokendb.add(token, user.id, client.id, expires, client.scope, function (error) {
         if (error) return callback(error);
 
         debug('grant token: new access token for client %s token %s', client.id, token);
@@ -103,7 +107,7 @@ gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI,
             var token = tokendb.generateToken();
             var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-            tokendb.add(token, tokendb.PREFIX_USER + authCode.userId, authCode.clientId, expires, client.scope, function (error) {
+            tokendb.add(token, authCode.userId, authCode.clientId, expires, client.scope, function (error) {
                 if (error) return callback(error);
 
                 debug('exchange: new access token for client %s token %s', client.id, token);
@@ -198,13 +202,13 @@ function loginForm(req, res) {
         });
     }
 
-    clientdb.get(u.query.client_id, function (error, result) {
+    clients.get(u.query.client_id, function (error, result) {
         if (error) return sendError(req, res, 'Unknown OAuth client');
 
         switch (result.type) {
-            case clientdb.TYPE_ADMIN: return render(constants.ADMIN_NAME, '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_EXTERNAL: return render('External Application', '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
+            case clients.TYPE_BUILT_IN: return render(result.appId, '/api/v1/cloudron/avatar');
+            case clients.TYPE_EXTERNAL: return render(result.appId, '/api/v1/cloudron/avatar');
+            case clients.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
             default: break;
         }
 
@@ -303,16 +307,20 @@ function accountSetup(req, res, next) {
     user.getByResetToken(req.body.resetToken, function (error, userObject) {
         if (error) return sendError(req, res, 'Invalid Reset Token');
 
-        userObject.username = req.body.username;
-        userObject.displayName = req.body.displayName;
-
-        user.update(userObject.id, userObject.username, userObject.email, userObject.displayName, function (error) {
+        var data = _.pick(req.body, 'username', 'displayName');
+        user.update(userObject.id, data, auditSource(req), function (error) {
             if (error && error.reason === UserError.ALREADY_EXISTS) return renderAccountSetupSite(res, req, userObject, 'Username already exists');
+            if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+            if (error && error.reason === UserError.NOT_FOUND) return renderAccountSetupSite(res, req, userObject, 'No such user');
             if (error) return next(new HttpError(500, error));
 
+            userObject.username = req.body.username;
+            userObject.displayName = req.body.displayName;
+
             // setPassword clears the resetToken
             user.setPassword(userObject.id, req.body.password, function (error, result) {
-                if (error && error.reason === UserError.BAD_PASSWORD) return renderAccountSetupSite(res, req, userObject, 'Password invalid');
+                if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+
                 if (error) return next(new HttpError(500, error));
 
                 res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
@@ -354,7 +362,7 @@ function passwordReset(req, res, next) {
 
         // setPassword clears the resetToken
         user.setPassword(userObject.id, req.body.password, function (error, result) {
-            if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(406, 'Password does not meet the requirements'));
+            if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(406, error.message));
             if (error) return next(new HttpError(500, error));
 
             res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
@@ -396,8 +404,8 @@ var authorization = [
     gServer.authorization({}, function (clientId, redirectURI, callback) {
         debug('authorization: client %s with callback to %s.', clientId, redirectURI);
 
-        clientdb.get(clientId, function (error, client) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+        clients.get(clientId, function (error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
             if (error) return callback(error);
 
             // ignore the origin passed into form the client, but use the one from the clientdb
@@ -411,9 +419,12 @@ var authorization = [
         // Handle our different types of oauth clients
         var type = req.oauth2.client.type;
 
-        if (type === clientdb.TYPE_ADMIN) return next();
-        if (type === clientdb.TYPE_EXTERNAL) return next();
-        if (type === clientdb.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unknown OAuth client.');
+        if (type === clients.TYPE_EXTERNAL || type === clients.TYPE_BUILT_IN) {
+            eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, req.oauth2.client.appId), { userId: req.oauth2.user.id });
+            return next();
+        } else if (type === clients.TYPE_SIMPLE_AUTH) {
+            return sendError(req, res, 'Unknown OAuth client.');
+        }
 
         appdb.get(req.oauth2.client.appId, function (error, appObject) {
             if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
@@ -422,6 +433,8 @@ var authorization = [
                 if (error) return sendError(req, res, 'Internal error');
                 if (!access) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
 
+                eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, appObject.id), { userId: req.oauth2.user.id });
+
                 next();
             });
         });
@@ -443,6 +456,31 @@ var token = [
     gServer.errorHandler()
 ];
 
+// tests if all requestedScopes are attached to the request
+function validateRequestedScopes(req, requestedScopes) {
+    assert.strictEqual(typeof req, 'object');
+    assert(Array.isArray(requestedScopes));
+
+    if (!req.authInfo || !req.authInfo.scope) return new Error('No scope found');
+
+    var scopes = req.authInfo.scope.split(',');
+
+    // check for roles separately
+    if (requestedScopes.indexOf(clients.SCOPE_ROLE_SDK) !== -1 && scopes.indexOf(clients.SCOPE_ROLE_SDK) === -1) {
+        return new Error('Missing required scope role "' + clients.SCOPE_ROLE_SDK + '"');
+    }
+
+    if (scopes.indexOf('*') !== -1) return null;
+
+    for (var i = 0; i < requestedScopes.length; ++i) {
+        if (scopes.indexOf(requestedScopes[i]) === -1) {
+            debug('scope: missing scope "%s".', requestedScopes[i]);
+            return new Error('Missing required scope "' + requestedScopes[i] + '"');
+        }
+    }
+
+    return null;
+}
 
 //  The scope middleware provides an auth middleware for routes.
 //
@@ -462,17 +500,8 @@ function scope(requestedScope) {
     return [
         passport.authenticate(['bearer'], { session: false }),
         function (req, res, next) {
-            if (!req.authInfo || !req.authInfo.scope) return next(new HttpError(401, 'No scope found'));
-            if (req.authInfo.scope === '*') return next();
-
-            var scopes = req.authInfo.scope.split(',');
-
-            for (var i = 0; i < requestedScopes.length; ++i) {
-                if (scopes.indexOf(requestedScopes[i]) === -1) {
-                    debug('scope: missing scope "%s".', requestedScopes[i]);
-                    return next(new HttpError(401, 'Missing required scope "' + requestedScopes[i] + '"'));
-                }
-            }
+            var error = validateRequestedScopes(req, requestedScopes);
+            if (error) return next(new HttpError(401, error.message));
 
             next();
         }
@@ -503,6 +532,7 @@ exports = module.exports = {
     accountSetup: accountSetup,
     authorization: authorization,
     token: token,
+    validateRequestedScopes: validateRequestedScopes,
     scope: scope,
     csrf: csrf
 };

src/routes/profile.js

@@ -0,0 +1,81 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    update: update,
+    changePassword: changePassword,
+    setShowTutorial: setShowTutorial
+};
+
+var assert = require('assert'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    user = require('../user.js'),
+    UserError = user.UserError,
+    _ = require('underscore');
+
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    next(new HttpSuccess(200, {
+        id: req.user.id,
+        username: req.user.username,
+        email: req.user.email,
+        admin: req.user.admin,
+        displayName: req.user.displayName,
+        showTutorial: req.user.showTutorial
+    }));
+}
+
+function update(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
+    if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
+
+    var data = _.pick(req.body, 'email', 'displayName');
+
+    user.update(req.user.id, data, auditSource(req), function (error) {
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function changePassword(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.user, 'object');
+
+    if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'newPassword must be a string'));
+
+    user.setPassword(req.user.id, req.body.newPassword, function (error) {
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Wrong password'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function setShowTutorial(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.showTutorial !== 'boolean') return next(new HttpError(400, 'showTutorial must be a boolean.'));
+
+    user.setShowTutorial(req.user.id, req.body.showTutorial, function (error) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Wrong password'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}

src/routes/settings.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -18,6 +16,9 @@ exports = module.exports = {
     getBackupConfig: getBackupConfig,
     setBackupConfig: setBackupConfig,
 
+    getTimeZone: getTimeZone,
+    setTimeZone: setTimeZone,
+
     setCertificate: setCertificate,
     setAdminCertificate: setAdminCertificate
 };
@@ -45,7 +46,7 @@ function setAutoupdatePattern(req, res, next) {
     if (typeof req.body.pattern !== 'string') return next(new HttpError(400, 'pattern is required'));
 
     settings.setAutoupdatePattern(req.body.pattern, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, 'Invalid pattern'));
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(200));
@@ -58,8 +59,9 @@ function setCloudronName(req, res, next) {
     if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name is required'));
 
     settings.setCloudronName(req.body.name, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, 'Invalid name'));
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200));
     });
 }
@@ -67,10 +69,32 @@ function setCloudronName(req, res, next) {
 function getCloudronName(req, res, next) {
     settings.getCloudronName(function (error, name) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200, { name: name }));
     });
 }
 
+function getTimeZone(req, res, next) {
+    settings.getTimeZone(function (error, tz) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { timeZone: tz }));
+    });
+}
+
+function setTimeZone(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.timeZone !== 'string') return next(new HttpError(400, 'timeZone is required'));
+
+    settings.setTimeZone(req.body.timeZone, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
 function setCloudronAvatar(req, res, next) {
     assert.strictEqual(typeof req.files, 'object');
 
@@ -79,6 +103,7 @@ function setCloudronAvatar(req, res, next) {
 
     settings.setCloudronAvatar(avatar, function (error) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(202, {}));
     });
 }

src/routes/sysadmin.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -8,9 +6,11 @@ exports = module.exports = {
     retire: retire
 };
 
-var cloudron = require('../cloudron.js'),
+var backups = require('../backups.js'),
+    BackupsError = require('../backups.js').BackupsError,
+    cloudron = require('../cloudron.js'),
     CloudronError = require('../cloudron.js').CloudronError,
-    debug = require('debug')('box:routes/internal'),
+    debug = require('debug')('box:routes/sysadmin'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
@@ -19,8 +19,9 @@ function backup(req, res, next) {
 
     // note that cloudron.backup only waits for backup initiation and not for backup to complete
     // backup progress can be checked up ny polling the progress api call
-    cloudron.backup(function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+    var auditSource = { userId: null, username: 'sysadmin' };
+    backups.backup(auditSource, function (error) {
+        if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
@@ -31,9 +32,11 @@ function update(req, res, next) {
     debug('triggering update');
 
     // this only initiates the update, progress can be checked via the progress route
-    cloudron.updateToLatest(function (error) {
+    var auditSource = { userId: null, username: 'sysadmin' };
+    cloudron.updateToLatest(auditSource, function (error) {
         if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
         if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === CloudronError.SELF_UPGRADE_NOT_SUPPORTED) return next(new HttpError(412, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
@@ -43,12 +46,9 @@ function update(req, res, next) {
 function retire(req, res, next) {
     debug('triggering retire');
 
-    // note that cloudron.backup only waits for backup initiation and not for backup to complete
-    // backup progress can be checked up ny polling the progress api call
-    cloudron.retire(function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(202, {}));
+    cloudron.retire('migrate', { }, function (error) {
+        if (error) console.error('Retire failed.', error);
     });
+
+    next(new HttpSuccess(202, {}));
 }

src/routes/test/apps-test.js

@@ -1,7 +1,7 @@
 'use strict';
 
-/* jslint node:true */
 /* global it:false */
+/* global xit:false */
 /* global describe:false */
 /* global before:false */
 /* global after:false */
@@ -12,7 +12,7 @@ var appdb = require('../../appdb.js'),
     path = require('path'),
     async = require('async'),
     child_process = require('child_process'),
-    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     config = require('../../config.js'),
     constants = require('../../constants.js'),
     database = require('../../database.js'),
@@ -23,18 +23,18 @@ var appdb = require('../../appdb.js'),
     http = require('http'),
     https = require('https'),
     js2xml = require('js2xmlparser'),
+    ldap = require('../../ldap.js'),
     net = require('net'),
     nock = require('nock'),
     paths = require('../../paths.js'),
-    redis = require('redis'),
-    superagent = require('superagent'),
     safe = require('safetydance'),
     server = require('../../server.js'),
     settings = require('../../settings.js'),
+    simpleauth = require('../../simpleauth.js'),
+    superagent = require('superagent'),
     taskmanager = require('../../taskmanager.js'),
     tokendb = require('../../tokendb.js'),
     url = require('url'),
-    util = require('util'),
     uuid = require('node-uuid'),
     _ = require('underscore');
 
@@ -42,9 +42,9 @@ var SERVER_URL = 'http://localhost:' + config.get('port');
 
 // Test image information
 var TEST_IMAGE_REPO = 'cloudron/test';
-var TEST_IMAGE_TAG = '10.0.0';
+var TEST_IMAGE_TAG = '16.0.0';
 var TEST_IMAGE = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
-var TEST_IMAGE_ID = child_process.execSync('docker inspect --format={{.Id}} ' + TEST_IMAGE).toString('utf8').trim();
+// var TEST_IMAGE_ID = child_process.execSync('docker inspect --format={{.Id}} ' + TEST_IMAGE).toString('utf8').trim();
 
 var APP_STORE_ID = 'test', APP_ID;
 var APP_LOCATION = 'appslocation';
@@ -58,8 +58,8 @@ var APP_MANIFEST_1 = JSON.parse(fs.readFileSync(__dirname + '/../../../../test-a
 APP_MANIFEST_1.dockerImage = TEST_IMAGE;
 APP_MANIFEST_1.singleUser = true;
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='admin@me.com';
-var USER_1_ID = null, USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1338', EMAIL_1 ='user@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='admin@me.com';
+var USER_1_ID = null, USERNAME_1 = 'user', EMAIL_1 ='user@me.com';
 var token = null; // authentication token
 var token_1 = null;
 
@@ -104,56 +104,74 @@ function startDockerProxy(interceptor, callback) {
     }).listen(5687, callback);
 }
 
+function checkAddons(appEntry, done) {
+    async.retry({ times: 15, interval: 3000 }, function (callback) {
+        // this was previously written with superagent but it was getting sporadic EPIPE
+        var req = http.get({ hostname: 'localhost', port: appEntry.httpPort, path: '/check_addons?username=' + USERNAME + '&password=' + PASSWORD });
+        req.on('error', callback);
+        req.on('response', function (res) {
+            if (res.statusCode !== 200) return callback('app returned non-200 status : ' + res.statusCode);
+
+            var d = '';
+            res.on('data', function (chunk) { d += chunk.toString('utf8'); });
+            res.on('end', function () {
+                var body = JSON.parse(d);
+
+                delete body.recvmail; // unclear why dovecot mail delivery won't work
+                delete body.stdenv; // cannot access APP_ORIGIN
+
+                for (var key in body) {
+                    if (body[key] !== 'OK') return callback('Not done yet: ' + JSON.stringify(body));
+                }
+
+                callback();
+            });
+        });
+
+        req.end();
+    }, done);
+}
+
+function checkRedis(containerId, done) {
+    var redisIp, exportedRedisPort;
+
+    docker.getContainer(containerId).inspect(function (error, data) {
+        expect(error).to.not.be.ok();
+        expect(data).to.be.ok();
+
+        redisIp = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
+        expect(redisIp).to.be.ok();
+
+        exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp');
+        expect(exportedRedisPort).to.be(null);
+
+        done();
+    });
+}
+
 describe('Apps', function () {
-    this.timeout(50000);
+    this.timeout(100000);
 
     var dockerProxy;
     var imageDeleted = false;
     var imageCreated = false;
 
     before(function (done) {
-        console.log('Starting addons, this can take 10 seconds');
-
-        child_process.exec(__dirname + '/start_addons.sh', function (error) {
-            if (error) return done(error);
-
-            dockerProxy = startDockerProxy(function interceptor(req, res) {
-                if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
-                    imageCreated = true;
-                    res.writeHead(200);
-                    res.end();
-                    return true;
-                } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE + '?force=false&noprune=false') {
-                    imageDeleted = true;
-                    res.writeHead(200);
-                    res.end();
-                    return true;
-                }
-                return false;
-            }, done);
-        });
-    });
-
-    after(function (done) {
-        // child_process.exec('docker rm -f mysql; docker rm -f postgresql; docker rm -f mongodb; docker rm -f mail', function (error) {
-        child_process.exec('docker ps | awk \'{print $1}\' | xargs docker rm -f', function () {
-            dockerProxy.close(done);
-        });
-    });
-
-
-    /*
-        Individual sub category setup and cleanup
-    */
-    function setup(done) {
         config._reset();
 
+        process.env.CREATE_INFRA = 1;
+
+        safe.fs.unlinkSync(paths.INFRA_VERSION_FILE);
+        child_process.execSync('docker ps -qa | xargs --no-run-if-empty docker rm -f');
+
         async.series([
             // first clear, then start server. otherwise, taskmanager spins up tasks for obsolete appIds
             database.initialize,
             database._clear,
 
             server.start.bind(server),
+            ldap.start,
+            simpleauth.start,
 
             function (callback) {
                 var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
@@ -192,58 +210,104 @@ describe('Apps', function () {
                 token_1 = tokendb.generateToken();
 
                 // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                tokendb.add(token_1, tokendb.PREFIX_USER + USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+                tokendb.add(token_1, USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+            },
+
+            function (callback) {
+                dockerProxy = startDockerProxy(function interceptor(req, res) {
+                    if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
+                        imageCreated = true;
+                        res.writeHead(200);
+                        res.end();
+                        return true;
+                    } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE + '?force=false&noprune=false') {
+                        imageDeleted = true;
+                        res.writeHead(200);
+                        res.end();
+                        return true;
+                    }
+                    return false;
+                }, callback);
             },
 
             settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
             settings.setTlsConfig.bind(null, { provider: 'caas' }),
             settings.setBackupConfig.bind(null, { provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
-        ], done);
-    }
+        ], function (error) {
+            if (error) return done(error);
+
+            console.log('This test can take ~40 seconds to start as it waits for infra to be ready');
+            setTimeout(done, 40000);
+        });
+    });
+
+    after(function (done) {
+        delete process.env.CREATE_INFRA;
+
+        // child_process.execSync('docker ps -qa | xargs --no-run-if-empty docker rm -f');
+        dockerProxy.close(function () { });
 
-    function cleanup(done) {
         // db is not cleaned up here since it's too late to call it after server.stop. if called before server.stop taskmanager apptasks are unhappy :/
         async.series([
             taskmanager.stopPendingTasks,
             taskmanager.waitForPendingTasks,
             server.stop,
+            ldap.stop,
+            simpleauth.stop,
             config._reset,
         ], done);
-    }
+    });
 
     describe('App API', function () {
-        this.timeout(50000);
-
-        before(setup);
-
         after(function (done) {
-            APP_ID = null;
-            cleanup(done);
+            appdb._clear(done); // TODO: test proper uninstall (requires mock for aws)
         });
 
         it('app install fails - missing manifest', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD })
+                   .send({ password: PASSWORD })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
-                expect(res.body.message).to.eql('manifest is required');
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
                 done();
             });
         });
 
-        it('app install fails - missing appId', function (done) {
+        it('app install fails - null manifest', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ manifest: APP_MANIFEST, password: PASSWORD })
+                   .send({ manifest: null, password: PASSWORD })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
-                expect(res.body.message).to.eql('appStoreId is required');
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
                 done();
             });
         });
 
-        it('app install fails - invalid json', function (done) {
+        it('app install fails - bad manifest format', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ manifest: 'epic', password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(res.body.message).to.eql('manifest must be an object');
+                done();
+            });
+        });
+
+        it('app install fails - empty appStoreId format', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ manifest: null, appStoreId: '', password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
+                done();
+            });
+        });
+
+       it('app install fails - invalid json', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
                    .send('garbage')
@@ -256,7 +320,7 @@ describe('Apps', function () {
         it('app install fails - invalid location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('Hostname can only contain alphanumerics and hyphen');
@@ -267,7 +331,7 @@ describe('Apps', function () {
         it('app install fails - invalid location type', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('location is required');
@@ -278,7 +342,7 @@ describe('Apps', function () {
         it('app install fails - reserved admin location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql(constants.ADMIN_LOCATION + ' is reserved');
@@ -289,7 +353,7 @@ describe('Apps', function () {
         it('app install fails - reserved api location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql(constants.API_LOCATION + ' is reserved');
@@ -300,7 +364,7 @@ describe('Apps', function () {
         it('app install fails - portBindings must be object', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('portBindings must be an object');
@@ -311,7 +375,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction is required', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {} })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {} })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction is required');
@@ -322,7 +386,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction type is wrong', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: '' })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: '' })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction is required');
@@ -333,7 +397,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction no users not allowed', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction must specify one user');
@@ -344,7 +408,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction too many users not allowed', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: { users: [ 'one', 'two' ] } })
+                   .send({ manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: { users: [ 'one', 'two' ] } })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction must specify one user');
@@ -355,50 +419,64 @@ describe('Apps', function () {
         it('app install fails for non admin', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token_1 })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(403);
                 done();
             });
         });
 
-        it('app install fails due to purchase failure', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(402, {});
+        it('app install fails because manifest download fails', function (done) {
+            var fake = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(404, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(fake.isDone()).to.be.ok();
+                done();
+            });
+        });
+
+        it('app install fails due to purchase failure', function (done) {
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(402, {});
+
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(402);
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 done();
             });
         });
 
         it('app install succeeds with purchase', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 expect(res.body.id).to.be.a('string');
                 APP_ID = res.body.id;
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 done();
             });
         });
 
         it('app install fails because of conflicting location', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
-
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(409);
-                expect(fake.isDone()).to.be.ok();
                 done();
             });
         });
@@ -446,24 +524,6 @@ describe('Apps', function () {
              });
         });
 
-        it('can get appBySubdomain', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/subdomains/' + APP_LOCATION)
-                   .end(function (err, res) {
-                expect(res.statusCode).to.equal(200);
-                expect(res.body.id).to.eql(APP_ID);
-                expect(res.body.installationState).to.be.ok();
-                done();
-            });
-        });
-
-        it('cannot get invalid app by Subdomain', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/subdomains/tikaloma')
-                   .end(function (err, res) {
-                expect(res.statusCode).to.equal(404);
-                done();
-            });
-        });
-
         it('cannot uninstall invalid app', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/whatever/uninstall')
                 .send({ password: PASSWORD })
@@ -514,23 +574,23 @@ describe('Apps', function () {
         });
 
         it('app install succeeds already purchased', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: null })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 expect(res.body.id).to.be.a('string');
                 APP_ID = res.body.id;
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 done();
             });
         });
 
         it('app install succeeds without password but developer token', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
-
             settings.setDeveloperMode(true, function (error) {
                 expect(error).to.be(null);
 
@@ -539,7 +599,7 @@ describe('Apps', function () {
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(200);
-                    expect(result.body.expiresAt).to.be.a('number');
+                    expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                     expect(result.body.token).to.be.a('string');
 
                     // overwrite non dev token
@@ -547,11 +607,10 @@ describe('Apps', function () {
 
                     superagent.post(SERVER_URL + '/api/v1/apps/install')
                            .query({ access_token: token })
-                           .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: null })
+                           .send({ manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: null })
                            .end(function (err, res) {
                         expect(res.statusCode).to.equal(202);
                         expect(res.body.id).to.be.a('string');
-                        expect(fake.isDone()).to.be.ok();
                         APP_ID = res.body.id;
                         done();
                     });
@@ -582,8 +641,6 @@ describe('Apps', function () {
             imageCreated = false;
 
             async.series([
-                setup,
-
                 function (callback) {
                     apiHockInstance
                         .get('/api/v1/apps/' + APP_STORE_ID + '/versions/' + APP_MANIFEST.version + '/icon')
@@ -612,7 +669,6 @@ describe('Apps', function () {
             APP_ID = null;
 
             async.series([
-                cleanup,
                 apiHockServer.close.bind(apiHockServer),
                 awsHockServer.close.bind(awsHockServer)
             ], done);
@@ -621,7 +677,8 @@ describe('Apps', function () {
         var appResult = null /* the json response */, appEntry = null /* entry from database */;
 
         it('can install test app', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
 
             var count = 0;
             function checkInstallStatus() {
@@ -638,12 +695,13 @@ describe('Apps', function () {
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                  .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 expect(res.body.id).to.be.a('string');
-                expect(res.body.id).to.be.eql(APP_ID);
+                APP_ID = res.body.id;
                 checkInstallStatus();
             });
         });
@@ -672,15 +730,9 @@ describe('Apps', function () {
                 expect(data.Config.Env).to.contain('CLOUDRON=1');
                 expect(data.Config.Env).to.contain('APP_ORIGIN=https://' + config.appFqdn(APP_LOCATION));
                 expect(data.Config.Env).to.contain('APP_DOMAIN=' + config.appFqdn(APP_LOCATION));
-                expect(data.Config.Hostname).to.be(APP_LOCATION);
-                clientdb.getByAppIdAndType(appResult.id, clientdb.TYPE_OAUTH, function (error, client) {
-                    expect(error).to.not.be.ok();
-                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
-                    expect(client.clientSecret.length).to.be(64); // 32 hex chars (256 bits)
-                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_ID=' + client.id);
-                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_SECRET=' + client.clientSecret);
-                    done();
-                });
+                // Hostname must not be set of app fqdn or app location!
+                expect(data.Config.Hostname).to.not.contain(APP_LOCATION);
+                done();
             });
         });
 
@@ -726,116 +778,50 @@ describe('Apps', function () {
             });
         });
 
-        var redisIp, exportedRedisPort;
-
-        it('installation - redis addon created', function (done) {
-            docker.getContainer('redis-' + APP_ID).inspect(function (error, data) {
-                expect(error).to.not.be.ok();
-                expect(data).to.be.ok();
-
-                redisIp = safe.query(data, 'NetworkSettings.IPAddress');
-                expect(redisIp).to.be.ok();
-
-                exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
-                expect(exportedRedisPort).to.be.ok();
-
+        it('installation - app responnds to http request', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort).end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.status).to.be('OK');
                 done();
             });
         });
 
-        it('installation - redis addon config', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
-
-                var urlp = url.parse(redisUrl);
-                var password = urlp.auth.split(':')[1];
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                client.on('error', done);
-                client.set('key', 'value');
-                client.get('key', function (err, reply) {
-                    expect(err).to.not.be.ok();
-                    expect(reply.toString()).to.be('value');
-                    client.end();
-                    done();
-                });
-            });
-        });
-
-        it('installation - mysql addon config', function (done) {
+        it('installation - oauth addon config', function (done) {
             var appContainer = docker.getContainer(appEntry.containerId);
             appContainer.inspect(function (error, data) {
-                var mysqlUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('MYSQL_URL=') === 0) mysqlUrl = env.split('=')[1]; });
-                expect(mysqlUrl).to.be.ok();
+                expect(error).to.not.be.ok();
 
-                var urlp = url.parse(mysqlUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('MYSQL_PORT=3306');
-                expect(data.Config.Env).to.contain('MYSQL_HOST=mysql');
-                expect(data.Config.Env).to.contain('MYSQL_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('MYSQL_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('MYSQL_DATABASE=' + dbname);
-
-                var cmd = util.format('mysql -h %s -u%s -p%s --database=%s -e "CREATE TABLE IF NOT EXISTS foo (id INT);"',
-                    'mysql', username, password, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error, stdout, stderr) {
-                    expect(!error).to.be.ok();
-                    expect(stdout.length).to.be(0);
-                    // expect(stderr.length).to.be(0); // "Warning: Using a password on the command line interface can be insecure."
+                clients.getByAppIdAndType(APP_ID, clients.TYPE_OAUTH, function (error, client) {
+                    expect(error).to.not.be.ok();
+                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
+                    expect(client.clientSecret.length).to.be(256); // 32 hex chars (8 * 256 bits)
+                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_ID=' + client.id);
+                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_SECRET=' + client.clientSecret);
                     done();
                 });
             });
         });
 
-        it('installation - postgresql addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                var postgresqlUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('POSTGRESQL_URL=') === 0) postgresqlUrl = env.split('=')[1]; });
-                expect(postgresqlUrl).to.be.ok();
-
-                var urlp = url.parse(postgresqlUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('POSTGRESQL_PORT=5432');
-                expect(data.Config.Env).to.contain('POSTGRESQL_HOST=postgresql');
-                expect(data.Config.Env).to.contain('POSTGRESQL_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('POSTGRESQL_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('POSTGRESQL_DATABASE=' + dbname);
-
-                var cmd = util.format('bash -c "PGPASSWORD=%s psql -q -h %s -U%s --dbname=%s -e \'CREATE TABLE IF NOT EXISTS foo (id INT);\'"',
-                    password, 'postgresql', username, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error, stdout, stderr) {
-                    expect(!error).to.be.ok();
-                    expect(stdout.length).to.be(0);
-                    expect(stderr.length).to.be(0);
-                    done();
-                });
+        it('installation - app can populate addons', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort + '/populate_addons').end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                for (var key in res.body) {
+                    expect(res.body[key]).to.be('OK');
+                }
+                done();
             });
         });
 
-        it('installation - scheduler', function (done) {
-            async.retry({ times: 100, interval: 1000 }, function (retryCallback) {
-                if (fs.existsSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env')) return retryCallback();
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 1');
+            checkAddons(appEntry, done);
+        });
 
-                retryCallback(new Error('not run yet'));
-            }, done);
+        it('installation - redis addon created', function (done) {
+            checkRedis('redis-' + APP_ID, done);
         });
 
         xit('logs - stdout and stderr', function (done) {
@@ -957,6 +943,12 @@ describe('Apps', function () {
             checkStartState();
         });
 
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 2');
+            checkAddons(appEntry, done);
+        });
+
         it('can uninstall app', function (done) {
             var count = 0;
             function checkUninstallStatus() {
@@ -1040,8 +1032,6 @@ describe('Apps', function () {
             APP_ID = uuid.v4();
 
             async.series([
-                setup,
-
                 function (callback) {
                     config.set('fqdn', 'test.foobar.com');
                     callback();
@@ -1078,7 +1068,6 @@ describe('Apps', function () {
         after(function (done) {
             APP_ID = null;
             async.series([
-                cleanup,
                 apiHockServer.close.bind(apiHockServer),
                 awsHockServer.close.bind(awsHockServer)
             ], done);
@@ -1087,7 +1076,8 @@ describe('Apps', function () {
         var appResult = null, appEntry = null;
 
         it('can install test app', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
 
             var count = 0;
             function checkInstallStatus() {
@@ -1104,11 +1094,12 @@ describe('Apps', function () {
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: null })
+                  .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
-                expect(fake.isDone()).to.be.ok();
-                expect(res.body.id).to.equal(APP_ID);
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
+                APP_ID = res.body.id;
                 checkInstallStatus();
             });
         });
@@ -1196,52 +1187,26 @@ describe('Apps', function () {
             });
         });
 
-        var redisIp, exportedRedisPort;
-
-        it('installation - redis addon created', function (done) {
-            docker.getContainer('redis-' + APP_ID).inspect(function (error, data) {
-                expect(error).to.not.be.ok();
-                expect(data).to.be.ok();
-
-                redisIp = safe.query(data, 'NetworkSettings.IPAddress');
-                expect(redisIp).to.be.ok();
-
-                exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
-                expect(exportedRedisPort).to.be.ok();
 
+        it('installation - app can populate addons', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort + '/populate_addons').end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                for (var key in res.body) {
+                    expect(res.body[key]).to.be('OK');
+                }
                 done();
             });
         });
 
-        it('installation - redis addon config', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 3');
+            checkAddons(appEntry, done);
+        });
 
-                var urlp = url.parse(redisUrl);
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                var password = urlp.auth.split(':')[1];
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                function checkRedis() {
-                    var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                    client.on('error', done);
-                    client.set('key', 'value');
-                    client.get('key', function (err, reply) {
-                        expect(err).to.not.be.ok();
-                        expect(reply.toString()).to.be('value');
-                        client.end();
-                        done();
-                    });
-                }
-
-                setTimeout(checkRedis, 1000); // the bridge network takes time to come up?
-            });
+        it('installation - redis addon created', function (done) {
+            checkRedis('redis-' + APP_ID, done);
         });
 
         function checkConfigureStatus(count, done) {
@@ -1259,20 +1224,20 @@ describe('Apps', function () {
             });
         }
 
-        it('cannot reconfigure app with missing location', function (done) {
+        it('cannot reconfigure app with bad location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: 1234, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
             });
         });
 
-        it('cannot reconfigure app with missing accessRestriction', function (done) {
+        it('cannot reconfigure app with bad accessRestriction', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 } })
+                  .send({ password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: false })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -1282,7 +1247,7 @@ describe('Apps', function () {
         it('cannot reconfigure app with only the cert, no key', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -1292,27 +1257,27 @@ describe('Apps', function () {
         it('cannot reconfigure app with only the key, no cert', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, key: validKey1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
             });
         });
 
-        it('cannot reconfigure app with cert not bein a string', function (done) {
+        it('cannot reconfigure app with cert not being a string', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: 1234, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: 1234, key: validKey1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
             });
         });
 
-        it('cannot reconfigure app with key not bein a string', function (done) {
+        it('cannot reconfigure app with key not being a string', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: 1234 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, cert: validCert1, key: 1234 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -1322,7 +1287,7 @@ describe('Apps', function () {
         it('non admin cannot reconfigure app', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token_1 })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(403);
                 done();
@@ -1332,7 +1297,7 @@ describe('Apps', function () {
         it('can reconfigure app', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 } })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 checkConfigureStatus(0, done);
@@ -1345,7 +1310,7 @@ describe('Apps', function () {
                 expect(!error).to.be.ok();
                 expect(app).to.be.an('object');
                 appEntry = app;
-                expect(appEntry.containerid).to.not.be(oldContainerId);
+                expect(appEntry.containerId).to.not.be(oldContainerId);
                 done();
             });
         });
@@ -1362,61 +1327,19 @@ describe('Apps', function () {
         });
 
         it('reconfiguration - redis addon recreated', function (done) {
-            docker.getContainer('redis-' + APP_ID).inspect(function (error, data) {
-                expect(error).to.not.be.ok();
-                expect(data).to.be.ok();
-
-                redisIp = safe.query(data, 'NetworkSettings.IPAddress');
-                expect(redisIp).to.be.ok();
-
-                exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
-                expect(exportedRedisPort).to.be.ok();
-
-                done();
-            });
+            checkRedis('redis-' + APP_ID, done);
         });
 
-        it('redis addon works after reconfiguration', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
-
-                var urlp = url.parse(redisUrl);
-                var password = urlp.auth.split(':')[1];
-
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                client.on('error', done);
-                client.set('key', 'value');
-                client.get('key', function (err, reply) {
-                    expect(err).to.not.be.ok();
-                    expect(reply.toString()).to.be('value');
-                    client.end();
-                    done();
-                });
-            });
-        });
-
-        it('scheduler works after reconfiguration', function (done) {
-            async.retry({ times: 100, interval: 1000 }, function (callback) {
-                var data = safe.fs.readFileSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env', 'utf8');
-
-                if (data && data.indexOf('ECHO_SERVER_PORT=7172') !== -1) return callback();
-
-                callback(new Error('not run yet'));
-            }, done);
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 4');
+            checkAddons(appEntry, done);
         });
 
         it('can reconfigure app with custom certificate', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: validKey1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 checkConfigureStatus(0, done);

src/routes/test/backups-test.js

@@ -19,7 +19,7 @@ var appdb = require('../../appdb.js'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null;
 
 var server;
@@ -53,7 +53,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, { }, callback);
         },
 
         function createSettings(callback) {

src/routes/test/clients-test.js

@@ -8,7 +8,7 @@
 
 var async = require('async'),
     config = require('../../config.js'),
-    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     database = require('../../database.js'),
     oauth2 = require('../oauth2.js'),
     expect = require('expect.js'),
@@ -21,7 +21,7 @@ var async = require('async'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token
 
 function cleanup(done) {
@@ -163,6 +163,26 @@ describe('OAuth Clients API', function () {
                 });
             });
 
+            it('fails with invalid name', function (done) {
+                superagent.post(SERVER_URL + '/api/v1/oauth/clients')
+                       .query({ access_token: token })
+                       .send({ appId: '$"$%^45asdfasdfadf.adf.', redirectURI: 'http://foobar.com', scope: 'profile' })
+                       .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+            });
+
+            it('succeeds with dash', function (done) {
+                superagent.post(SERVER_URL + '/api/v1/oauth/clients')
+                       .query({ access_token: token })
+                       .send({ appId: 'fo-1234-bar', redirectURI: 'http://foobar.com', scope: 'profile' })
+                       .end(function (error, result) {
+                    expect(result.statusCode).to.equal(201);
+                    done();
+                });
+            });
+
             it('succeeds', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
                        .query({ access_token: token })
@@ -174,7 +194,7 @@ describe('OAuth Clients API', function () {
                     expect(result.body.redirectURI).to.be.a('string');
                     expect(result.body.clientSecret).to.be.a('string');
                     expect(result.body.scope).to.be.a('string');
-                    expect(result.body.type).to.equal(clientdb.TYPE_EXTERNAL);
+                    expect(result.body.type).to.equal(clients.TYPE_EXTERNAL);
 
                     done();
                 });
@@ -291,6 +311,14 @@ describe('OAuth Clients API', function () {
             scope: 'profile'
         };
 
+        var CLIENT_1 = {
+            id: '',
+            appId: 'someAppId-1',
+            redirectURI: 'http://some.callback1',
+            scope: 'profile',
+            type: clients.TYPE_OAUTH
+        };
+
         before(function (done) {
             async.series([
                 server.start.bind(null),
@@ -387,6 +415,44 @@ describe('OAuth Clients API', function () {
                    });
                 });
             });
+
+            it('fails for cid-webadmin', function (done) {
+                superagent.del(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin')
+                       .query({ access_token: token })
+                       .end(function (error, result) {
+                    expect(result.statusCode).to.equal(405);
+
+                    superagent.get(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin')
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(result.statusCode).to.equal(200);
+
+                        done();
+                   });
+                });
+            });
+
+            it('fails for addon auth client', function (done) {
+                clients.add(CLIENT_1.appId, CLIENT_1.type, CLIENT_1.redirectURI, CLIENT_1.scope, function (error, result) {
+                    expect(error).to.equal(null);
+
+                    CLIENT_1.id = result.id;
+
+                    superagent.del(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_1.id)
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(result.statusCode).to.equal(405);
+
+                        superagent.get(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_1.id)
+                               .query({ access_token: token })
+                               .end(function (error, result) {
+                            expect(result.statusCode).to.equal(200);
+
+                            done();
+                       });
+                    });
+                });
+            });
         });
     });
 });
@@ -489,8 +555,7 @@ describe('Clients', function () {
             .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
 
-                expect(result.body.clients.length).to.eql(1);
-                expect(result.body.clients[0].tokenCount).to.eql(1);
+                expect(result.body.clients.length).to.eql(3);
 
                 done();
             });
@@ -543,7 +608,7 @@ describe('Clients', function () {
                 expect(result.statusCode).to.equal(200);
 
                 expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.id);
+                expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
 
                 done();
             });
@@ -596,7 +661,7 @@ describe('Clients', function () {
                 expect(result.statusCode).to.equal(200);
 
                 expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.id);
+                expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
 
                 superagent.del(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin/tokens')
                 .query({ access_token: token })

src/routes/test/cloudron-test.js

@@ -1,6 +1,5 @@
 'use strict';
 
-/* jslint node:true */
 /* global it:false */
 /* global describe:false */
 /* global before:false */
@@ -10,6 +9,7 @@ var async = require('async'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
+    locker = require('../../locker.js'),
     nock = require('nock'),
     os = require('os'),
     superagent = require('superagent'),
@@ -18,7 +18,7 @@ var async = require('async'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token
 
 var server;
@@ -26,6 +26,7 @@ function setup(done) {
     nock.cleanAll();
     config._reset();
     config.set('version', '0.5.0');
+    config.set('fqdn', 'localhost');
     server.start(done);
 }
 
@@ -233,7 +234,9 @@ describe('Cloudron', function () {
         });
 
         it('succeeds', function (done) {
-            var scope = nock(config.apiServerOrigin()).get('/api/v1/boxes/localhost?token=' + config.token()).reply(200, { box: { region: 'sfo', size: '1gb' }});
+            var scope = nock(config.apiServerOrigin())
+                  .get('/api/v1/boxes/localhost?token=' + config.token())
+                  .reply(200, { box: { region: 'sfo', size: '1gb' }, user: { }});
 
             superagent.get(SERVER_URL + '/api/v1/cloudron/config')
                    .query({ access_token: token })
@@ -260,6 +263,182 @@ describe('Cloudron', function () {
 
     });
 
+    describe('migrate', function () {
+        before(function (done) {
+            async.series([
+                setup,
+
+                function (callback) {
+                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                           .query({ setupToken: 'somesetuptoken' })
+                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                           .end(function (error, result) {
+                        expect(result).to.be.ok();
+                        expect(scope1.isDone()).to.be.ok();
+                        expect(scope2.isDone()).to.be.ok();
+
+                        // stash token for further use
+                        token = result.body.token;
+
+                        callback();
+                    });
+                },
+
+                function setupBackupConfig(callback) {
+                    superagent.post(SERVER_URL + '/api/v1/settings/backup_config')
+                           .send({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(result.statusCode).to.equal(200);
+
+                        callback();
+                    });
+                }
+
+            ], done);
+        });
+
+        after(function (done) {
+            locker.unlock(locker._operation); // migrate never unlocks
+            cleanup(done);
+        });
+
+        it('fails without token', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 'sfo'})
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails without password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 'sfo'})
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds without size', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ region: 'sfo', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
+                done();
+            });
+        });
+
+        it('fails with wrong size type', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 4, region: 'sfo', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds without region', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
+                done();
+            });
+        });
+
+        it('fails with wrong region type', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 4, password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails when in wrong state', function (done) {
+            var scope2 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) {
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope1 = nock(config.apiServerOrigin())
+                .post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
+                    return body.size && body.region && body.restoreKey;
+                }).reply(409, {});
+
+            injectShellMock();
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 'sfo', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
+
+                function checkAppstoreServerCalled() {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
+                        restoreShellMock();
+                        return done();
+                    }
+
+                    setTimeout(checkAppstoreServerCalled, 100);
+                }
+
+                checkAppstoreServerCalled();
+            });
+        });
+
+        it('succeeds', function (done) {
+            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
+                return body.size && body.region && body.restoreKey;
+            }).reply(202, {});
+
+            var scope2 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) {
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+
+            injectShellMock();
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 'sfo', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
+
+                function checkAppstoreServerCalled() {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
+                        restoreShellMock();
+                        return done();
+                    }
+
+                    setTimeout(checkAppstoreServerCalled, 100);
+                }
+
+                checkAppstoreServerCalled();
+            });
+        });
+    });
+
     describe('feedback', function () {
         before(function (done) {
             async.series([

src/routes/test/developer-test.js

@@ -17,7 +17,7 @@ var async = require('async'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token
 
 var server;
@@ -297,7 +297,16 @@ describe('Developer API', function () {
 
         it('fails with unknown username', function (done) {
             superagent.post(SERVER_URL + '/api/v1/developer/login')
-                   .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
+                   .send({ username: USERNAME + USERNAME, password: PASSWORD })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails with unknown email', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/developer/login')
+                   .send({ username: USERNAME + EMAIL, password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(401);
                 done();
@@ -318,7 +327,18 @@ describe('Developer API', function () {
                    .send({ username: USERNAME, password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
+                expect(result.body.token).to.be.a('string');
+                done();
+            });
+        });
+
+        it('with uppercase username succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/developer/login')
+                   .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
@@ -329,10 +349,77 @@ describe('Developer API', function () {
                    .send({ username: EMAIL, password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
+                expect(result.body.token).to.be.a('string');
+                done();
+            });
+        });
+
+        it('with uppercase email succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/developer/login')
+                   .send({ username: EMAIL.toUpperCase(), password: PASSWORD })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
         });
     });
+
+    describe('sdk tokens are valid without password checks', function () {
+        var token_normal, token_sdk;
+
+        before(function (done) {
+            async.series([
+                setup,
+
+                settings.setDeveloperMode.bind(null, true),
+
+                function (callback) {
+                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                           .query({ setupToken: 'somesetuptoken' })
+                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                           .end(function (error, result) {
+                        expect(result).to.be.ok();
+                        expect(scope1.isDone()).to.be.ok();
+                        expect(scope2.isDone()).to.be.ok();
+
+                        token_normal = result.body.token;
+
+                        superagent.post(SERVER_URL + '/api/v1/developer/login')
+                          .send({ username: USERNAME, password: PASSWORD })
+                          .end(function (error, result) {
+                            expect(result.statusCode).to.equal(200);
+                            expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
+                            expect(result.body.token).to.be.a('string');
+
+                            token_sdk = result.body.token;
+
+                            callback();
+                        });
+                    });
+                },
+            ], done);
+        });
+
+        after(cleanup);
+
+        it('fails with non sdk token', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password').query({ access_token: token_normal }).send({ newPassword: 'Some?$123' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password').query({ access_token: token_sdk }).send({ newPassword: 'Some?$123' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+    });
 });

src/routes/test/eventlog-test.js

@@ -0,0 +1,152 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    nock = require('nock'),
+    superagent = require('superagent'),
+    server = require('../../server.js'),
+    tokendb = require('../../tokendb.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var USER_1_ID = null, token_1;
+
+function setup(done) {
+    config.setVersion('1.2.3');
+
+    async.series([
+        server.start.bind(server),
+
+        database._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        },
+
+        function (callback) {
+            superagent.post(SERVER_URL + '/api/v1/users')
+                   .query({ access_token: token })
+                   .send({ username: 'nonadmin', email: 'notadmin@server.test', invite: false })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(201);
+
+                USER_1_ID = res.body.id;
+
+                callback(null);
+            });
+        },
+
+        function (callback) {
+            token_1 = tokendb.generateToken();
+
+            // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+            tokendb.add(token_1, USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+        }
+
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('Eventlog API', function () {
+    before(setup);
+    after(cleanup);
+
+    describe('get', function () {
+        it('fails due to wrong token', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token.toUpperCase() })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails for non-admin', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token_1, page: 1, per_page: 10 })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(403);
+
+                done();
+            });
+        });
+
+        it('succeeds for admin', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token, page: 1, per_page: 10 })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.eventlogs.length >= 2).to.be.ok(); // activate, user.add
+
+                done();
+            });
+        });
+
+        it('succeeds with action', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token, page: 1, per_page: 10, action: 'cloudron.activate' })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.eventlogs.length).to.equal(1);
+
+                done();
+            });
+        });
+
+        it('succeeds with search', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token, page: 1, per_page: 10, search: EMAIL })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.eventlogs.length).to.equal(1);
+
+                done();
+            });
+        });
+
+        it('succeeds with search', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token, page: 1, per_page: 10, search: EMAIL, action: 'cloudron.activate' })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.eventlogs.length).to.equal(0);
+
+                done();
+            });
+        });
+    });
+});

src/routes/test/groups-test.js

@@ -6,22 +6,19 @@
 
 'use strict';
 
-var appdb = require('../../appdb.js'),
-    async = require('async'),
+var async = require('async'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
     groups = require('../../groups.js'),
     superagent = require('superagent'),
     server = require('../../server.js'),
-    settings = require('../../settings.js'),
     tokendb = require('../../tokendb.js'),
-    nock = require('nock'),
-    userdb = require('../../userdb.js');
+    nock = require('nock');
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1337', EMAIL_1 ='happy@me.com';
 var token, token_1 = null;
 var userId, userId_1 = null;
@@ -117,6 +114,9 @@ describe('Groups API', function () {
                 expect(res.body.groups).to.be.an(Array);
                 expect(res.body.groups.length).to.be(1);
                 expect(res.body.groups[0].name).to.eql('admin');
+                expect(res.body.groups[0].userIds).to.be.an(Array);
+                expect(res.body.groups[0].userIds.length).to.be(1);
+                expect(res.body.groups[0].userIds[0]).to.be(userId);
                 done();
             });
         });
@@ -224,7 +224,7 @@ describe('Groups API', function () {
         });
 
         it('cannot add user to invalid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'admin', 'something' ]})
                   .end(function (error, result) {
@@ -234,7 +234,7 @@ describe('Groups API', function () {
         });
 
         it('can add user to valid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'admin', 'group0', 'group1' ]})
                   .end(function (error, result) {
@@ -243,8 +243,8 @@ describe('Groups API', function () {
             });
         });
 
-        it('can remove last user from admin', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
+        it('cannot remove self from admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'group0', 'group1' ]})
                   .end(function (error, result) {
@@ -252,5 +252,37 @@ describe('Groups API', function () {
                 done();
             });
         });
+
+        it('can add another user to admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId_1 + '/groups')
+                  .query({ access_token: token })
+                  .send({ groupIds: [ 'admin' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+
+        it('lists members of admin group', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups/admin')
+                  .query({ access_token: token })
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.userIds.length).to.be(2);
+                expect(result.body.userIds[0]).to.be(userId);
+                expect(result.body.userIds[1]).to.be(userId_1);
+                done();
+            });
+        });
+
+        it('remove activation user from admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
+                  .query({ access_token: token_1 })
+                  .send({ groupIds: [ 'group0', 'group1' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204); // user_1 is still admin, so we can remove the other person
+                done();
+            });
+        });
     });
 });

src/routes/test/mailboxes-test.js

@@ -0,0 +1,211 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    superagent = require('superagent'),
+    server = require('../../server.js'),
+    nock = require('nock'),
+    userdb = require('../../userdb.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var MAILBOX_ID = 'mailbox';
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var server;
+function setup(done) {
+    config.set('fqdn', 'foobar.com');
+
+    async.series([
+        server.start.bind(server),
+
+        userdb._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        }
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('Mailbox API', function () {
+    this.timeout(10000);
+
+    before(setup);
+    after(cleanup);
+
+    it('cannot create a mailbox without name param', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot create a mailbox without token', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(401);
+            done();
+        });
+    });
+
+    it('cannot create invalid mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: 'no-reply' })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('can create mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(201);
+            done();
+        });
+    });
+
+    it('can get mailbox', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.name).to.equal(MAILBOX_ID);
+            expect(res.body.creationTime).to.be.ok();
+            done();
+        });
+    });
+
+    it('cannot set with invalid alias', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'a' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot set with invalid type', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'apple', 34 ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('can set aliases of mailbox', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'alias1', 'alias2' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            done();
+        });
+    });
+
+    it('can list mailboxes', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.mailboxes).to.be.an(Array);
+            expect(res.body.mailboxes[0].name).to.be(MAILBOX_ID);
+            done();
+        });
+    });
+
+    it('can get aliases', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.aliases).to.be.an(Array);
+            expect(res.body.aliases[0]).to.be('alias1');
+            expect(res.body.aliases[1]).to.be('alias2');
+            done();
+        });
+    });
+
+    it('can add another mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID + '2' })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(201);
+            done();
+        });
+    });
+
+    it('cannot alias existing mailbox', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ MAILBOX_ID + '2' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(409);
+            done();
+        });
+    });
+
+    it('can delete mailbox', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(204);
+            done();
+        });
+    });
+
+    it('cannot delete random mailbox', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(404);
+            done();
+        });
+    });
+});

src/routes/test/oauth2-test.js

@@ -18,6 +18,7 @@ var expect = require('expect.js'),
     querystring = require('querystring'),
     database = require('../../database.js'),
     clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     userdb = require('../../userdb.js'),
     user = require('../../user.js'),
     appdb = require('../../appdb.js'),
@@ -138,14 +139,15 @@ describe('OAuth2', function () {
     describe('flow', function () {
         var USER_0 = {
             id: uuid.v4(),
-            username: 'someusername',
+            username: 'someUSERname',
             password: '@#45Strongpassword',
-            email: 'some@email.com',
+            email: 'some@EMAIL.com',
             salt: 'somesalt',
             createdAt: (new Date()).toUTCString(),
             modifiedAt: (new Date()).toUTCString(),
             resetToken: hat(256),
-            displayName: ''
+            displayName: '',
+            showTutorial: false
         };
 
         var APP_0 = {
@@ -155,7 +157,8 @@ describe('OAuth2', function () {
             location: 'test',
             portBindings: {},
             accessRestriction: null,
-            memoryLimit: 0
+            memoryLimit: 0,
+            altDomain: null
         };
 
         var APP_1 = {
@@ -165,7 +168,8 @@ describe('OAuth2', function () {
             location: 'test1',
             portBindings: {},
             accessRestriction: { users: [ 'foobar' ] },
-            memoryLimit: 0
+            memoryLimit: 0,
+            altDomain: null
         };
 
         var APP_2 = {
@@ -175,7 +179,8 @@ describe('OAuth2', function () {
             location: 'test2',
             portBindings: {},
             accessRestriction: { users: [ USER_0.id ] },
-            memoryLimit: 0
+            memoryLimit: 0,
+            altDomain: null
         };
 
         var APP_3 = {
@@ -185,14 +190,15 @@ describe('OAuth2', function () {
             location: 'test3',
             portBindings: {},
             accessRestriction: { groups: [ 'someothergroup', 'admin', 'anothergroup' ] },
-            memoryLimit: 0
+            memoryLimit: 0,
+            altDomain: null
         };
 
         // unknown app
         var CLIENT_0 = {
             id: 'cid-client0',
             appId: 'appid-app0',
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret0',
             redirectURI: 'http://redirect0',
             scope: 'profile'
@@ -202,7 +208,7 @@ describe('OAuth2', function () {
         var CLIENT_1 = {
             id: 'cid-client1',
             appId: 'appid-app1',
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret1',
             redirectURI: 'http://redirect1',
             scope: 'profile'
@@ -212,7 +218,7 @@ describe('OAuth2', function () {
         var CLIENT_2 = {
             id: 'cid-client2',
             appId: APP_0.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret2',
             redirectURI: 'http://redirect2',
             scope: 'profile'
@@ -222,7 +228,7 @@ describe('OAuth2', function () {
         var CLIENT_3 = {
             id: 'cid-client3',
             appId: APP_0.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret3',
             redirectURI: 'http://redirect1',
             scope: 'profile'
@@ -232,7 +238,7 @@ describe('OAuth2', function () {
         var CLIENT_4 = {
             id: 'cid-client4',
             appId: 'appid-app4',
-            type: clientdb.TYPE_PROXY,
+            type: clients.TYPE_PROXY,
             clientSecret: 'secret4',
             redirectURI: 'http://redirect4',
             scope: 'profile'
@@ -242,7 +248,7 @@ describe('OAuth2', function () {
         var CLIENT_5 = {
             id: 'cid-client5',
             appId: APP_0.id,
-            type: clientdb.TYPE_PROXY,
+            type: clients.TYPE_PROXY,
             clientSecret: 'secret5',
             redirectURI: 'http://redirect5',
             scope: 'profile'
@@ -252,7 +258,7 @@ describe('OAuth2', function () {
         var CLIENT_6 = {
             id: 'cid-client6',
             appId: APP_1.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret6',
             redirectURI: 'http://redirect6',
             scope: 'profile'
@@ -262,7 +268,7 @@ describe('OAuth2', function () {
         var CLIENT_7 = {
             id: 'cid-client7',
             appId: APP_2.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret7',
             redirectURI: 'http://redirect7',
             scope: 'profile'
@@ -272,7 +278,7 @@ describe('OAuth2', function () {
         var CLIENT_8 = {
             id: 'cid-client8',
             appId: APP_2.id,
-            type: clientdb.TYPE_SIMPLE_AUTH,
+            type: clients.TYPE_SIMPLE_AUTH,
             clientSecret: 'secret8',
             redirectURI: 'http://redirect8',
             scope: 'profile'
@@ -282,7 +288,7 @@ describe('OAuth2', function () {
         var CLIENT_9 = {
             id: 'cid-client9',
             appId: APP_3.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret9',
             redirectURI: 'http://redirect9',
             scope: 'profile'
@@ -308,12 +314,12 @@ describe('OAuth2', function () {
                 clientdb.add.bind(null, CLIENT_7.id, CLIENT_7.appId, CLIENT_7.type, CLIENT_7.clientSecret, CLIENT_7.redirectURI, CLIENT_7.scope),
                 clientdb.add.bind(null, CLIENT_8.id, CLIENT_8.appId, CLIENT_8.type, CLIENT_8.clientSecret, CLIENT_8.redirectURI, CLIENT_8.scope),
                 clientdb.add.bind(null, CLIENT_9.id, CLIENT_9.appId, CLIENT_9.type, CLIENT_9.clientSecret, CLIENT_9.redirectURI, CLIENT_9.scope),
-                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit),
-                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit),
-                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit),
-                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit),
+                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0),
+                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1),
+                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2),
+                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3),
                 function (callback) {
-                    user.create(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, function (error, userObject) {
+                    user.create(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, null /* source */, function (error, userObject) {
                         expect(error).to.not.be.ok();
 
                         // update the global objects to reflect the new user id
@@ -883,7 +889,14 @@ describe('OAuth2', function () {
                         expect(foo.access_token).to.be.a('string');
                         expect(foo.token_type).to.eql('Bearer');
 
-                        done();
+                        // Ensure the token is also usable
+                        superagent.get(SERVER_URL + '/api/v1/profile?access_token=' + foo.access_token, function (error, result) {
+                            expect(error).to.not.be.ok();
+                            expect(result.status).to.eql(200);
+                            expect(result.body.username).to.equal(USER_0.username.toLowerCase());
+
+                            done();
+                        });
                     });
                 });
             });
@@ -1263,7 +1276,14 @@ describe('OAuth2', function () {
                             expect(body.access_token).to.be.a('string');
                             expect(body.token_type).to.eql('Bearer');
 
-                            done();
+                            // Ensure the token is also usable
+                            superagent.get(SERVER_URL + '/api/v1/profile?access_token=' + body.access_token, function (error, result) {
+                                expect(error).to.not.be.ok();
+                                expect(result.status).to.eql(200);
+                                expect(result.body.username).to.equal(USER_0.username.toLowerCase());
+
+                                done();
+                            });
                         });
                     });
                 });
@@ -1283,7 +1303,8 @@ describe('Password', function () {
         createdAt: (new Date()).toUTCString(),
         modifiedAt: (new Date()).toUTCString(),
         resetToken: hat(256),
-        displayName: ''
+        displayName: '',
+        showTutorial: false
     };
 
     // make csrf always succeed for testing

src/routes/test/profile-test.js

@@ -0,0 +1,322 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var config = require('../../config.js'),
+    database = require('../../database.js'),
+    tokendb = require('../../tokendb.js'),
+    expect = require('expect.js'),
+    mailer = require('../../mailer.js'),
+    superagent = require('superagent'),
+    nock = require('nock'),
+    server = require('../../server.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var USERNAME_0 = 'superaDmIn', PASSWORD = 'Foobar?1337', EMAIL_0 = 'silLY@me.com', EMAIL_0_NEW = 'stupID@me.com', DISPLAY_NAME_0_NEW = 'New Name';
+
+describe('Profile API', function () {
+    this.timeout(5000);
+
+    var user_0 = null;
+    var token_0;
+
+    function setup(done) {
+        server.start(function (error) {
+            expect(!error).to.be.ok();
+
+            mailer._clearMailQueue();
+
+            database._clear(function (error) {
+                expect(error).to.eql(null);
+
+                var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                       .query({ setupToken: 'somesetuptoken' })
+                       .send({ username: USERNAME_0, password: PASSWORD, email: EMAIL_0 })
+                       .end(function (err, res) {
+                    expect(err).to.eql(null);
+                    expect(res.statusCode).to.equal(201);
+
+                    // stash for later use
+                    token_0 = res.body.token;
+
+                    expect(scope1.isDone()).to.be.ok();
+                    expect(scope2.isDone()).to.be.ok();
+
+                    done();
+                });
+            });
+        });
+    }
+
+    function cleanup(done) {
+        database._clear(function (error) {
+            expect(!error).to.be.ok();
+
+            mailer._clearMailQueue();
+
+            server.stop(done);
+        });
+    }
+
+    describe('get profile', function () {
+        before(setup);
+        after(cleanup);
+
+        it('fails without token', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile/').end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+
+                done();
+            });
+        });
+
+        it('fails with empty token', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile/').query({ access_token: '' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+
+                done();
+            });
+        });
+
+        it('fails with invalid token', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile/').query({ access_token: 'some token' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile/').query({ access_token: token_0 }).end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.username).to.equal(USERNAME_0.toLowerCase());
+                expect(result.body.email).to.equal(EMAIL_0.toLowerCase());
+                expect(result.body.admin).to.be.ok();
+                expect(result.body.showTutorial).to.be.ok();
+                expect(result.body.displayName).to.be.a('string');
+                expect(result.body.password).to.not.be.ok();
+                expect(result.body.salt).to.not.be.ok();
+
+                user_0 = result.body;
+
+                done();
+            });
+        });
+
+        it('fails with expired token', function (done) {
+            var token = tokendb.generateToken();
+            var expires = Date.now() - 2000; // 1 sec
+
+            tokendb.add(token, user_0.id, null, expires, '*', function (error) {
+                expect(error).to.not.be.ok();
+
+                superagent.get(SERVER_URL + '/api/v1/profile').query({ access_token: token }).end(function (error, result) {
+                    expect(result.statusCode).to.equal(401);
+
+                    done();
+                });
+            });
+        });
+
+        it('fails with invalid token in auth header', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile').set('Authorization', 'Bearer ' + 'x' + token_0).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('succeeds with token in auth header', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile').set('Authorization', 'Bearer ' + token_0).end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.username).to.equal(USERNAME_0.toLowerCase());
+                expect(result.body.email).to.equal(EMAIL_0.toLowerCase());
+                expect(result.body.admin).to.be.ok();
+                expect(result.body.showTutorial).to.be.ok();
+                expect(result.body.displayName).to.be.a('string');
+                expect(result.body.password).to.not.be.ok();
+                expect(result.body.salt).to.not.be.ok();
+                done();
+            });
+        });
+    });
+
+    describe('update', function () {
+        before(setup);
+        after(cleanup);
+
+        it('change email fails due to missing token', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile')
+                   .send({ email: EMAIL_0_NEW })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('change email fails due to invalid email', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile')
+                   .query({ access_token: token_0 })
+                   .send({ email: 'foo@bar' })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('change user succeeds without email nor displayName', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile')
+                   .query({ access_token: token_0 })
+                   .send({})
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+
+        it('change email succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile')
+                   .query({ access_token: token_0 })
+                   .send({ email: EMAIL_0_NEW })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+
+                superagent.get(SERVER_URL + '/api/v1/profile')
+                      .query({ access_token: token_0 })
+                      .end(function (err, res) {
+                    expect(res.statusCode).to.equal(200);
+                    expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
+                    expect(res.body.email).to.equal(EMAIL_0_NEW.toLowerCase());
+                    expect(res.body.admin).to.equal(true);
+                    expect(res.body.displayName).to.equal('');
+
+                    done();
+                });
+            });
+        });
+
+        it('change displayName succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile')
+                   .query({ access_token: token_0 })
+                   .send({ displayName: DISPLAY_NAME_0_NEW })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+
+                superagent.get(SERVER_URL + '/api/v1/profile')
+                      .query({ access_token: token_0 })
+                      .end(function (err, res) {
+                    expect(res.statusCode).to.equal(200);
+                    expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
+                    expect(res.body.email).to.equal(EMAIL_0_NEW.toLowerCase());
+                    expect(res.body.admin).to.be.ok();
+                    expect(res.body.displayName).to.equal(DISPLAY_NAME_0_NEW);
+
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('password change', function () {
+        before(setup);
+        after(cleanup);
+
+        it('fails due to missing current password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ newPassword: 'some wrong password' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails due to missing new password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails due to wrong password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ password: 'some wrong password', newPassword: 'MOre#$%34' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(403);
+                done();
+            });
+        });
+
+        it('fails due to invalid password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ password: PASSWORD, newPassword: 'five' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ password: PASSWORD, newPassword: 'MOre#$%34' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(204);
+                done();
+            });
+        });
+    });
+
+    describe('showTutorial change', function () {
+        before(setup);
+        after(cleanup);
+
+        it('fails due to missing showTutorial', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/tutorial')
+                   .query({ access_token: token_0 })
+                   .send({})
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails due to wrong showTutorial type', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/tutorial')
+                   .query({ access_token: token_0 })
+                   .send({ showTutorial: 'true' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/tutorial')
+                   .query({ access_token: token_0 })
+                   .send({ showTutorial: false })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(204);
+
+                superagent.get(SERVER_URL + '/api/v1/profile/').query({ access_token: token_0 }).end(function (error, result) {
+                    expect(result.statusCode).to.equal(200);
+                    expect(result.body.showTutorial).to.not.be.ok();
+
+                    done();
+                });
+            });
+        });
+    });
+});

src/routes/test/server-test.js

@@ -0,0 +1,88 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    net = require('net'),
+    nock = require('nock'),
+    superagent = require('superagent'),
+    server = require('../../server.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var server;
+function setup(done) {
+    config.setVersion('1.2.3');
+
+    async.series([
+        server.start.bind(server),
+
+        database._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        }
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('REST API', function () {
+    before(setup);
+    after(cleanup);
+
+    it('does not crash with invalid JSON', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+            .query({ access_token: token })
+            .set('content-type', 'application/json')
+            .send("some invalid non-strict json")
+           .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            expect(result.body.message).to.be('Bad JSON');
+            done();
+        });
+    });
+
+    it('does not crash with invalid string', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+            .query({ access_token: token })
+            .set('content-type', 'application/x-www-form-urlencoded')
+            .send("some string")
+           .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            done();
+        });
+    });
+});

src/routes/test/settings-test.js

@@ -22,7 +22,7 @@ var appdb = require('../../appdb.js'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null;
 
 var server;
@@ -56,7 +56,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok' };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, { }, callback);
         }
     ], done);
 }
@@ -360,5 +360,17 @@ describe('Settings API', function () {
             done();
         });
     });
+
+    describe('time_zone', function () {
+        it('succeeds', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/settings/time_zone')
+                   .query({ access_token: token })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.timeZone).to.be('America/Los_Angeles');
+                done();
+            });
+        });
+    });
 });
 

src/routes/test/simpleauth-test.js

@@ -6,9 +6,10 @@
 
 'use strict';
 
-var clientdb = require('../../clientdb.js'),
-    appdb = require('../../appdb.js'),
+var appdb = require('../../appdb.js'),
     async = require('async'),
+    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
@@ -21,7 +22,7 @@ describe('SimpleAuth API', function () {
     var SERVER_URL = 'http://localhost:' + config.get('port');
     var SIMPLE_AUTH_ORIGIN = 'http://localhost:' + config.get('simpleAuthPort');
 
-    var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+    var USERNAME = 'superaDMin', PASSWORD = 'Foobar?1337', EMAIL ='silly@ME.com';
 
     var APP_0 = {
         id: 'app0',
@@ -30,7 +31,8 @@ describe('SimpleAuth API', function () {
         location: 'test0',
         portBindings: {},
         accessRestriction: { users: [ 'foobar', 'someone'] },
-        memoryLimit: 0
+        memoryLimit: 0,
+        altDomain: null
     };
 
     var APP_1 = {
@@ -40,7 +42,8 @@ describe('SimpleAuth API', function () {
         location: 'test1',
         portBindings: {},
         accessRestriction: { users: [ 'foobar', 'someone' ] },
-        memoryLimit: 0
+        memoryLimit: 0,
+        altDomain: null
     };
 
     var APP_2 = {
@@ -50,7 +53,8 @@ describe('SimpleAuth API', function () {
         location: 'test2',
         portBindings: {},
         accessRestriction: null,
-        memoryLimit: 0
+        memoryLimit: 0,
+        altDomain: null
     };
 
     var APP_3 = {
@@ -60,13 +64,14 @@ describe('SimpleAuth API', function () {
         location: 'test3',
         portBindings: {},
         accessRestriction: { groups: [ 'someothergroup', 'admin', 'anothergroup' ] },
-        memoryLimit: 0
+        memoryLimit: 0,
+        altDomain: null
     };
 
     var CLIENT_0 = {
         id: 'someclientid',
         appId: 'someappid',
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret',
         redirectURI: '',
         scope: 'user,profile'
@@ -75,7 +80,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_1 = {
         id: 'someclientid1',
         appId: APP_0.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret1',
         redirectURI: '',
         scope: 'user,profile'
@@ -84,7 +89,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_2 = {
         id: 'someclientid2',
         appId: APP_1.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret2',
         redirectURI: '',
         scope: 'user,profile'
@@ -93,7 +98,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_3 = {
         id: 'someclientid3',
         appId: APP_2.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret3',
         redirectURI: '',
         scope: 'user,profile'
@@ -102,7 +107,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_4 = {
         id: 'someclientid4',
         appId: APP_2.id,
-        type: clientdb.TYPE_OAUTH,
+        type: clients.TYPE_OAUTH,
         clientSecret: 'someclientsecret4',
         redirectURI: '',
         scope: 'user,profile'
@@ -111,7 +116,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_5 = {
         id: 'someclientid5',
         appId: APP_3.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret5',
         redirectURI: '',
         scope: 'user,profile'
@@ -155,10 +160,10 @@ describe('SimpleAuth API', function () {
             clientdb.add.bind(null, CLIENT_3.id, CLIENT_3.appId, CLIENT_3.type, CLIENT_3.clientSecret, CLIENT_3.redirectURI, CLIENT_3.scope),
             clientdb.add.bind(null, CLIENT_4.id, CLIENT_4.appId, CLIENT_4.type, CLIENT_4.clientSecret, CLIENT_4.redirectURI, CLIENT_4.scope),
             clientdb.add.bind(null, CLIENT_5.id, CLIENT_5.appId, CLIENT_5.type, CLIENT_5.clientSecret, CLIENT_5.redirectURI, CLIENT_5.scope),
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit),
-            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit),
-            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit),
-            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit)
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2),
+            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3)
         ], done);
     });
 
@@ -324,7 +329,8 @@ describe('SimpleAuth API', function () {
                 .end(function (error, result) {
                     expect(error).to.be(null);
                     expect(result.body).to.be.an('object');
-                    expect(result.body.username).to.eql(USERNAME);
+                    expect(result.body.username).to.eql(USERNAME.toLowerCase());
+                    expect(result.body.email).to.eql(EMAIL.toLowerCase());
 
                     done();
                 });
@@ -356,7 +362,8 @@ describe('SimpleAuth API', function () {
                 .end(function (error, result) {
                     expect(error).to.be(null);
                     expect(result.body).to.be.an('object');
-                    expect(result.body.username).to.eql(USERNAME);
+                    expect(result.body.username).to.eql(USERNAME.toLowerCase());
+                    expect(result.body.email).to.eql(EMAIL.toLowerCase());
 
                     done();
                 });
@@ -388,7 +395,8 @@ describe('SimpleAuth API', function () {
                 .end(function (error, result) {
                     expect(error).to.be(null);
                     expect(result.body).to.be.an('object');
-                    expect(result.body.username).to.eql(USERNAME);
+                    expect(result.body.username).to.eql(USERNAME.toLowerCase());
+                    expect(result.body.email).to.eql(EMAIL.toLowerCase());
 
                     done();
                 });
@@ -420,7 +428,8 @@ describe('SimpleAuth API', function () {
                 .end(function (error, result) {
                     expect(error).to.be(null);
                     expect(result.body).to.be.an('object');
-                    expect(result.body.username).to.eql(USERNAME);
+                    expect(result.body.username).to.eql(USERNAME.toLowerCase());
+                    expect(result.body.email).to.eql(EMAIL.toLowerCase());
 
                     done();
                 });