Hide the appstore account setup for now

This fixes the dependency issues with bootstrap

.jshintrc

@@ -1,7 +1,8 @@
 {
-	"node": true,
-	"browser": true,
-	"unused": true,
-	"globalstrict": true,
-	"predef": [ "angular", "$" ]
+    "node": true,
+    "browser": true,
+    "unused": true,
+    "globalstrict": true,
+    "predef": [ "angular", "$" ],
+    "esnext": true
 }

CHANGES

@@ -510,3 +510,81 @@
 [0.13.4]
 - Fix mail addon restart issue
 
+[0.14.0]
+- You have mail :-)
+
+[0.14.1]
+- 2-character usernames are now allowed
+- Make cloudron CLI push/pull more robust
+
+[0.14.2]
+- Update mail addon
+
+[0.15.0]
+- [REST API](https://cloudron.io/references/api.html) is now in public beta
+- Enable Developer mode by default for new Cloudrons
+- Reverse proxy fixes for apps exposing a WebDav server
+- Allow admins to optionally set the username and displayName on user creation
+- Fix app autoupdate logic to detect if one or more in-use port bindings was removed
+
+[0.15.1]
+- Fix mail connectivity from IPv6 clients
+- Add API token management UI
+- Improved UI to enter email aliases
+
+[0.15.2]
+- Allow restoring apps from any previous backup
+
+[0.15.3]
+- Show installation progress in a tooltip
+
+[0.16.0]
+- Allow apps to be configured in configuring state
+- Improved platform architecture that allows incremental infrastructure updates
+- Implement app clone
+
+[0.16.1]
+- Fix UI layout issue in tokens page
+- Resume app tasks only when configured and platform ready
+- Allow errored apps to be reconfigured
+
+[0.16.2]
+- Fix assert when backing up apps in errored state
+- Fix bug where multiple redis installations caused an error
+
+[0.16.3]
+- Timeout in 10mins if app restore fails because of external domain CNAME setup
+
+[0.16.4]
+- Setup email aliases to only alias names for the Cloudron domain
+
+[0.16.5]
+- Allow sending email with alias as the From
+
+[0.16.6]
+- Add plan migration interface
+- Initial EC2 support
+
+[0.17.0]
+- Public beta release of Cloudron Mail Server
+- Add new DNS & Certs UI that enables easy migration to a custom domain
+- Allow sending and receiving email from alias subaddresses
+- Fix installation issue with some apps on the naked domain
+
+[0.17.1]
+- Preliminary user impersonation support
+- Fix crash in mail container when generating bounces
+
+[0.17.2]
+- Add config option to embed apps in other sites
+
+[0.17.3]
+- Incremental infrastructure update logic
+- Keep eventlogs only for a week
+- Throttle OOM mails
+
+[0.17.4]
+- Add warning for users moving to custom domains
+- Out of disk space and certificate renewal mails are now sent to cloudron owner for selfhosters
+- Fix a bug where selfhosted Cloudrons do not start because of a MySQL error
+- Implement new app versioning & update scheme

baseimage/createDOImage

@@ -10,7 +10,6 @@ readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
 export JSON="${SOURCE_DIR}/node_modules/.bin/json"
 
-provider="digitalocean"
 installer_revision=$(git rev-parse HEAD)
 box_name=""
 server_id=""
@@ -23,14 +22,13 @@ deploy_env="dev"
 [[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
 readonly GNU_GETOPT
 
-args=$(${GNU_GETOPT} -o "" -l "provider:,revision:,regions:,size:,name:,no-destroy,env:" -n "$0" -- "$@")
+args=$(${GNU_GETOPT} -o "" -l "revision:,regions:,size:,name:,no-destroy,env:" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
     --env) deploy_env="$2"; shift 2;;
     --revision) installer_revision="$2"; shift 2;;
-    --provider) provider="$2"; shift 2;;
     --name) box_name="$2"; destroy_server="no"; shift 2;;
     --no-destroy) destroy_server="no"; shift 2;;
     --) break;;
@@ -38,28 +36,23 @@ while true; do
     esac
 done
 
-echo "Creating image using ${provider}"
-if [[ "${provider}" == "digitalocean" ]]; then
-    if [[ "${deploy_env}" == "staging" ]]; then
-        assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING
-        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}"
-    elif [[ "${deploy_env}" == "dev" ]]; then
-        assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV
-        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}"
-    elif [[ "${deploy_env}" == "prod" ]]; then
-        assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD
-        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}"
-    else
-        echo "No such env ${deploy_env}."
-        exit 1
-    fi
-
-    vps="/bin/bash ${SCRIPT_DIR}/digitalocean.sh"
+echo "Creating digitalocean image"
+if [[ "${deploy_env}" == "staging" ]]; then
+    assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING
+    export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}"
+elif [[ "${deploy_env}" == "dev" ]]; then
+    assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV
+    export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}"
+elif [[ "${deploy_env}" == "prod" ]]; then
+    assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD
+    export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}"
 else
-    echo "Unknown provider : ${provider}"
+    echo "No such env ${deploy_env}."
     exit 1
 fi
 
+vps="/bin/bash ${SCRIPT_DIR}/digitalocean.sh"
+
 readonly ssh_keys="${HOME}/.ssh/id_rsa_caas_${deploy_env}"
 readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
 readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
@@ -137,8 +130,8 @@ while true; do
     sleep 30
 done
 
-echo "Copying INFRA_VERSION"
-$scp22 "${SCRIPT_DIR}/../src/INFRA_VERSION" root@${server_ip}:.
+echo "Copying infra_version.js"
+$scp22 "${SCRIPT_DIR}/../src/infra_version.js" root@${server_ip}:.
 
 echo "Copying box source"
 cd "${SOURCE_DIR}"

baseimage/createEC2Image

@@ -0,0 +1,185 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+export JSON="${SOURCE_DIR}/node_modules/.bin/json"
+
+installer_revision=$(git rev-parse HEAD)
+instance_id=""
+server_ip=""
+destroy_server="yes"
+
+ami_id="ami-f9e30f96"
+region="eu-central-1"
+aws_credentials="baseimage"
+security_group="sg-b9a473d1"
+instance_type="t2.small"
+subnet_id="subnet-801402e9"
+key_pair_name="id_rsa_yellowtent"
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "revisio0n:,no-destroy" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --revision) installer_revision="$2"; shift 2;;
+    --no-destroy) destroy_server="no"; shift 2;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+readonly ssh_keys="${HOME}/.ssh/id_rsa_yellowtent"
+readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+readonly ssh202="ssh -p 202 -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly ssh22="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+if [[ ! -f "${ssh_keys}" ]]; then
+    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
+    exit 1
+fi
+
+function debug() {
+    echo "$@" >&2
+}
+
+function get_pretty_revision() {
+    local git_rev="$1"
+    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
+
+    echo "${sha1}"
+}
+
+now=$(date "+%Y-%m-%d-%H%M%S")
+pretty_revision=$(get_pretty_revision "${installer_revision}")
+
+echo "Creating EC2 instance"
+instance_id=$(aws ec2 run-instances --image-id ${ami_id} --region ${region} --profile ${aws_credentials} --security-group-ids ${security_group} --instance-type ${instance_type} --key-name ${key_pair_name} --subnet-id ${subnet_id} --associate-public-ip-address | $JSON Instances[0].InstanceId)
+echo "Got InstanceId: ${instance_id}"
+
+# name the instance
+aws ec2 create-tags --profile ${aws_credentials} --resources ${instance_id} --tags "Key=Name,Value=baseimage-${pretty_revision}"
+
+echo "Waiting for instance to be running..."
+while true; do
+    event_status=`aws ec2 describe-instances --instance-id ${instance_id} --region ${region} --profile ${aws_credentials} | $JSON Reservations[0].Instances[0].State.Name`
+    if [[ "${event_status}" == "running" ]]; then
+        break
+    fi
+    debug -n "."
+    sleep 10
+done
+
+server_ip=$(aws ec2 describe-instances --instance-id ${instance_id} --region ${region} --profile ${aws_credentials} | $JSON Reservations[0].Instances[0].PublicIpAddress)
+echo "Server IP is: ${server_ip}"
+
+while true; do
+    echo "Trying to copy init script to server"
+    if $scp22 "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" ubuntu@${server_ip}:.; then
+        break
+    fi
+    echo "Timedout, trying again in 30 seconds"
+    sleep 30
+done
+
+echo "Copying infra_version.js"
+$scp22 "${SCRIPT_DIR}/../src/infra_version.js" ubuntu@${server_ip}:.
+
+echo "Copying box source"
+cd "${SOURCE_DIR}"
+git archive --format=tar HEAD | $ssh22 "ubuntu@${server_ip}" "cat - > /tmp/box.tar.gz"
+
+echo "Enabling root ssh access"
+if ! $ssh22 "ubuntu@${server_ip}" "sudo sed -e 's/.* \(ssh-rsa.*\)/\1/' -i /root/.ssh/authorized_keys"; then
+    echo "Unable to enable root access"
+    echo "Make sure to cleanup the ec2 instance ${instance_id}"
+    exit 1
+fi
+
+echo "Executing init script"
+if ! $ssh22 "root@${server_ip}" "/bin/bash /home/ubuntu/initializeBaseUbuntuImage.sh ${installer_revision}"; then
+    echo "Init script failed"
+    echo "Make sure to cleanup the ec2 instance ${instance_id}"
+    exit 1
+fi
+
+snapshot_name="cloudron-${pretty_revision}-${now}"
+
+echo "Creating ami image ${snapshot_name}"
+image_id=$(aws ec2 create-image --region ${region} --profile ${aws_credentials} --instance-id ${instance_id} --name ${snapshot_name} | $JSON ImageId)
+
+echo "Image creation started for image id: ${image_id}"
+
+echo "Waiting for image creation to finish..."
+while true; do
+    event_status=`aws ec2 describe-images --region ${region} --profile ${aws_credentials} --image-id ${image_id} | $JSON Images[0].State`
+    if [[ "${event_status}" == "available" ]]; then
+        break
+    fi
+    debug -n "."
+    sleep 10
+done
+
+echo "Terminating instance"
+aws ec2 terminate-instances --region ${region} --profile ${aws_credentials} --instance-ids ${instance_id}
+
+echo "Make image public"
+aws ec2 modify-image-attribute --region ${region} --profile ${aws_credentials} --image-id ${image_id} --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
+
+
+# http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
+# Images are currently created in eu-central-1
+echo "Coping image to other regions"
+ec2_regions=( "us-east-1" "us-west-1" "us-west-2" "ap-south-1" "ap-northeast-2" "ap-southeast-1" "ap-southeast-2" "ap-northeast-1" "eu-west-1" "sa-east-1" )
+ec2_amis=( )
+
+for r in ${ec2_regions[@]}; do
+    echo "=> ${r}"
+    ami_id=$(aws ec2 copy-image --region ${r} --profile ${aws_credentials} --source-image-id ${image_id} --source-region ${region} --name ${snapshot_name} | $JSON ImageId)
+
+    # append in the same order as the regions
+    ec2_amis+=( ${ami_id} )
+done
+
+# wait for all images to be available
+echo "Waiting for images to be ready (first will take the longest)..."
+region_string="${region}=${image_id}"
+i=0
+while [ $i -lt ${#ec2_regions[*]} ]; do
+    echo "=> ${ec2_regions[$i]} ${ec2_amis[$i]}"
+    while true; do
+        event_status=`aws ec2 describe-images --region ${ec2_regions[$i]} --profile ${aws_credentials} --image-id ${ec2_amis[$i]} | $JSON Images[0].State`
+        if [[ "${event_status}" == "available" ]]; then
+            echo "done"
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+
+    # now make it public
+    aws ec2 modify-image-attribute --region ${ec2_regions[$i]} --profile ${aws_credentials} --image-id ${ec2_amis[$i]} --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
+
+    # append to output string for release tool
+    region_string+=",${ec2_regions[$i]}=${ec2_amis[$i]}"
+
+    # inc the iteration counter
+    i=$(( $i + 1));
+done
+
+echo ""
+echo "--------------------------------------------------"
+echo "New image id is: ${image_id}"
+echo "Image region string for release:"
+echo "${region_string}"
+echo "--------------------------------------------------"
+echo ""

baseimage/digitalocean.sh

@@ -10,7 +10,7 @@ if [[ -z "${JSON}" ]]; then
     exit 1
 fi
 
-readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:"
+readonly CURL="curl --retry 5 -s -u ${DIGITAL_OCEAN_TOKEN}:"
 
 function debug() {
     echo "$@" >&2
@@ -49,9 +49,9 @@ function get_droplet_ip() {
 
 function get_droplet_id() {
     local droplet_name="$1"
-    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=100" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
+    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=200" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
     [[ -z "$id" ]] && exit 1
-    echo "$id"  
+    echo "$id"
 }
 
 function power_off_droplet() {
@@ -109,13 +109,24 @@ function get_image_id() {
     local snapshot_name="$1"
     local image_id=""
 
-    image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \
-       | $JSON images \
-       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id)
-
-    if [[ -n "${image_id}" ]]; then
-        echo "${image_id}"
+    if ! response=$($CURL "https://api.digitalocean.com/v2/images?per_page=200"); then
+        echo "Failed to get image listing. ${response}"
+        return 1
     fi
+
+    if ! image_id=$(echo "$response" \
+       | $JSON images \
+       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id); then
+        echo "Failed to parse curl response: ${response}"
+        return 1
+    fi
+
+    if [[ -z "${image_id}" ]]; then
+        echo "Failed to get image id of ${snapshot_name}. reponse: ${response}"
+        return 1
+    fi
+
+    echo "${image_id}"
 }
 
 function snapshot_droplet() {
@@ -128,16 +139,26 @@ function snapshot_droplet() {
     debug -n "Waiting for snapshot to complete"
 
     while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if ! response=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}"); then
+            echo "Could not get action status. ${response}"
+            continue
+        fi
+        if ! event_status=$(echo "${response}" | $JSON action.status); then
+            echo "Could not parse action.status from response. ${response}"
+            continue
+        fi
         if [[ "${event_status}" == "completed" ]]; then
             break
         fi
         debug -n "."
         sleep 10
     done
-    debug ""
+    debug "! done"
 
-    get_image_id "${snapshot_name}"
+    if ! image_id=$(get_image_id "${snapshot_name}"); then
+        return 1
+    fi
+    echo "${image_id}"
 }
 
 function destroy_droplet() {
@@ -177,7 +198,7 @@ function transfer_image_to_all_regions() {
     local image_id="$1"
 
     xfer_events=()
-    image_regions=(ams3) ## sfo1 is where the image is created
+    image_regions=(ams2) ## sfo1 is where the image is created
     for image_region in ${image_regions[@]}; do
         xfer_event=$(transfer_image ${image_id} ${image_region})
         echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}"

baseimage/initializeBaseUbuntuImage.sh

@@ -6,7 +6,6 @@ readonly USER=yellowtent
 readonly USER_HOME="/home/${USER}"
 readonly INSTALLER_SOURCE_DIR="${USER_HOME}/installer"
 readonly INSTALLER_REVISION="$1"
-readonly SELFHOSTED=$(( $# > 1 ? 1 : 0 ))
 readonly USER_DATA_FILE="/root/user_data.img"
 readonly USER_DATA_DIR="/home/yellowtent/data"
 
@@ -19,18 +18,6 @@ function die {
 
 [[ "$(systemd --version 2>&1)" == *"systemd 229"* ]] || die "Expecting systemd to be 229"
 
-if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
-    source "${SOURCE_DIR}/INFRA_VERSION"
-else
-    echo "No INFRA_VERSION found, skip pulling docker images"
-fi
-
-if [ ${SELFHOSTED} == 0 ]; then
-    echo "!! Initializing Ubuntu image for CaaS"
-else
-    echo "!! Initializing Ubuntu image for Selfhosting"
-fi
-
 echo "==== Create User ${USER} ===="
 if ! id "${USER}"; then
     useradd "${USER}" -m
@@ -66,15 +53,11 @@ iptables -P OUTPUT ACCEPT
 # NOTE: keep these in sync with src/apps.js validatePortBindings
 # allow ssh, http, https, ping, dns
 iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-if [ ${SELFHOSTED} == 0 ]; then
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,202,443,886 -j ACCEPT
-else
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,22,443,886 -j ACCEPT
-fi
+iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,202,443,587,993,4190 -j ACCEPT
 iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
 iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
 iptables -A INPUT -p udp --sport 53 -j ACCEPT
-iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+iptables -A INPUT -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
 
 # loopback
 iptables -A INPUT -i lo -j ACCEPT
@@ -156,30 +139,22 @@ update-grub
 # now add the user to the docker group
 usermod "${USER}" -a -G docker
 
-if [ -z $(echo "${INFRA_VERSION}") ]; then
-    echo "Skip pulling base docker images"
-else
-    echo "=== Pulling base docker images ==="
-    docker pull "${BASE_IMAGE}"
+echo "==== Install nodejs ===="
+# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
+mkdir -p /usr/local/node-4.1.1
+curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
+ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
+ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
+apt-get install -y python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
-    echo "=== Pulling mysql addon image ==="
-    docker pull "${MYSQL_IMAGE}"
+echo "==== Downloading docker images ===="
+images=$(node -e "var i = require('${SOURCE_DIR}/infra_version.js'); console.log(i.baseImage, Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
-    echo "=== Pulling postgresql addon image ==="
-    docker pull "${POSTGRESQL_IMAGE}"
-
-    echo "=== Pulling redis addon image ==="
-    docker pull "${REDIS_IMAGE}"
-
-    echo "=== Pulling mongodb addon image ==="
-    docker pull "${MONGODB_IMAGE}"
-
-    echo "=== Pulling graphite docker images ==="
-    docker pull "${GRAPHITE_IMAGE}"
-
-    echo "=== Pulling mail relay ==="
-    docker pull "${MAIL_IMAGE}"
-fi
+echo "Pulling images: ${images}"
+for image in ${images}; do
+    docker pull "${image}"
+done
 
 echo "==== Install nginx ===="
 apt-get -y install nginx-full
@@ -210,25 +185,11 @@ echo "==== Install logrotate ==="
 apt-get install -y cron logrotate
 systemctl enable cron
 
-echo "==== Install nodejs ===="
-# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
-mkdir -p /usr/local/node-4.1.1
-curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
-ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
-ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
-apt-get install -y python   # Install python which is required for npm rebuild
-[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
-
 echo "=== Rebuilding npm packages ==="
 cd "${INSTALLER_SOURCE_DIR}" && npm install --production
 chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"
 
 echo "==== Install installer systemd script ===="
-provisionEnv="PROVISION=digitalocean"
-if [ ${SELFHOSTED} == 1 ]; then
-    provisionEnv="PROVISION=local"
-fi
-
 cat > /etc/systemd/system/cloudron-installer.service <<EOF
 [Unit]
 Description=Cloudron Installer
@@ -238,7 +199,7 @@ BindsTo=systemd-journald.service
 [Service]
 Type=idle
 ExecStart="${INSTALLER_SOURCE_DIR}/src/server.js"
-Environment="DEBUG=installer*,connect-lastmile" ${provisionEnv}
+Environment="DEBUG=installer*,connect-lastmile"
 ; kill any child (installer.sh) as well
 KillMode=control-group
 Restart=on-failure
@@ -265,12 +226,13 @@ EOF
 
 # Allocate swap files
 # https://bbs.archlinux.org/viewtopic.php?id=194792 ensures this runs after do-resize.service
+# On ubuntu ec2 we use cloud-init https://wiki.archlinux.org/index.php/Cloud-init
 echo "==== Install box-setup systemd script ===="
 cat > /etc/systemd/system/box-setup.service <<EOF
 [Unit]
 Description=Box Setup
 Before=docker.service collectd.service mysql.service
-After=do-resize.service
+After=do-resize.service cloud-init.service
 
 [Service]
 Type=oneshot
@@ -311,16 +273,14 @@ chown root:systemd-journal /var/log/journal
 systemctl restart systemd-journald
 setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
 
-if [ ${SELFHOSTED} == 0 ]; then
-    echo "==== Install ssh ==="
-    apt-get -y install openssh-server
-    # https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
-    sed -e 's/^#\?Port .*/Port 202/g' \
-        -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
-        -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
-        -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
-        -i /etc/ssh/sshd_config
+echo "==== Install ssh ==="
+apt-get -y install openssh-server
+# https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
+sed -e 's/^#\?Port .*/Port 202/g' \
+    -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
+    -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
+    -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
+    -i /etc/ssh/sshd_config
 
-    # required so we can connect to this machine since port 22 is blocked by iptables by now
-    systemctl reload sshd
-fi
+# required so we can connect to this machine since port 22 is blocked by iptables by now
+systemctl reload sshd

installer.sh.ejs

@@ -1,164 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-echo ""
-echo "======== Cloudron Installer ========"
-echo ""
-
-if [ $# -lt 4 ]; then
-    echo "Usage: ./installer.sh <fqdn> <aws key id> <aws key secret> <bucket> <provider> <revision>"
-    exit 1
-fi
-
-# commandline arguments
-readonly fqdn="${1}"
-readonly aws_access_key_id="${2}"
-readonly aws_access_key_secret="${3}"
-readonly aws_backup_bucket="${4}"
-readonly provider="${5}"
-readonly revision="${6}"
-
-# environment specific urls
-<% if (env === 'prod') { %>
-readonly api_server_origin="https://api.cloudron.io"
-readonly web_server_origin="https://cloudron.io"
-<% } else { %>
-readonly api_server_origin="https://api.<%= env %>.cloudron.io"
-readonly web_server_origin="https://<%= env %>.cloudron.io"
-<% } %>
-readonly release_bucket_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases"
-readonly versions_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases/versions.json"
-readonly installer_code_url="${release_bucket_url}/box-${revision}.tar.gz"
-
-# runtime consts
-readonly installer_code_file="/tmp/box.tar.gz"
-readonly installer_tmp_dir="/tmp/box"
-readonly cert_folder="/tmp/certificates"
-
-# check for fqdn in /ets/hosts
-echo "[INFO] checking for hostname entry"
-readonly hostentry_found=$(grep "${fqdn}" /etc/hosts || true)
-if [[ -z $hostentry_found ]]; then
-    echo "[WARNING] No entry for ${fqdn} found in /etc/hosts"
-    echo "Adding an entry ..."
-
-    cat >> /etc/hosts <<EOF
-
-# The following line was added by the Cloudron installer script
-127.0.1.1 ${fqdn} ${fqdn}
-EOF
-else
-    echo "Valid hostname entry found in /etc/hosts"
-fi
-echo ""
-
-echo "[INFO] ensure minimal dependencies ..."
-apt-get update
-apt-get install -y curl
-echo ""
-
-echo "[INFO] Generating certificates ..."
-rm -rf "${cert_folder}"
-mkdir -p "${cert_folder}"
-
-cat > "${cert_folder}/CONFIG" <<EOF
-[ req ]
-default_bits           = 1024
-default_keyfile        = keyfile.pem
-distinguished_name     = req_distinguished_name
-prompt                 = no
-req_extensions         = v3_req
-
-[ req_distinguished_name ]
-C                      = DE
-ST                     = Berlin
-L                      = Berlin
-O                      = Cloudron UG
-OU                     = Cloudron
-CN                     = ${fqdn}
-emailAddress           = cert@cloudron.io
-
-[ v3_req ]
-# Extensions to add to a certificate request
-basicConstraints = CA:FALSE
-keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-subjectAltName = @alt_names
-
-[alt_names]
-DNS.1 = ${fqdn}
-DNS.2 = *.${fqdn}
-EOF
-
-# generate cert files
-openssl genrsa 2048 > "${cert_folder}/host.key"
-openssl req -new -out "${cert_folder}/host.csr" -key "${cert_folder}/host.key" -config "${cert_folder}/CONFIG"
-openssl x509 -req -days 3650 -in "${cert_folder}/host.csr" -signkey "${cert_folder}/host.key" -out "${cert_folder}/host.cert" -extensions v3_req -extfile "${cert_folder}/CONFIG"
-
-# make them json compatible, by collapsing to one line
-tls_cert=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.cert")
-tls_key=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.key")
-echo ""
-
-echo "[INFO] Fetching installer code ..."
-curl "${installer_code_url}" -o "${installer_code_file}"
-echo ""
-
-echo "[INFO] Extracting installer code to ${installer_tmp_dir} ..."
-rm -rf "${installer_tmp_dir}" && mkdir -p "${installer_tmp_dir}"
-tar xvf "${installer_code_file}" -C "${installer_tmp_dir}"
-echo ""
-
-echo "Creating initial provisioning config ..."
-cat > /root/provision.json <<EOF
-{
-    "sourceTarballUrl": "",
-    "data": {
-        "apiServerOrigin": "${api_server_origin}",
-        "webServerOrigin": "${web_server_origin}",
-        "fqdn": "${fqdn}",
-        "token": "",
-        "isCustomDomain": true,
-        "boxVersionsUrl": "${versions_url}",
-        "version": "",
-        "tlsCert": "${tls_cert}",
-        "tlsKey": "${tls_key}",
-        "provider": "${provider}",
-        "backupConfig": {
-            "provider": "s3",
-            "accessKeyId": "${aws_access_key_id}",
-            "secretAccessKey": "${aws_access_key_secret}",
-            "bucket": "${aws_backup_bucket}",
-            "prefix": "backups"
-        },
-        "dnsConfig": {
-            "provider": "route53",
-            "accessKeyId": "${aws_access_key_id}",
-            "secretAccessKey": "${aws_access_key_secret}"
-        },
-        "tlsConfig": {
-            "provider": "letsencrypt-<%= env %>"
-        }
-    }
-}
-EOF
-
-echo "[INFO] Running Ubuntu initializing script ..."
-/bin/bash "${installer_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "${revision}" selfhosting
-echo ""
-
-echo "[INFO] Reloading systemd daemon ..."
-systemctl daemon-reload
-echo ""
-
-echo "[INFO] Restart docker ..."
-systemctl restart docker
-echo ""
-
-echo "[FINISHED] Now starting Cloudron init jobs ..."
-systemctl start box-setup
-
-# TODO this is only for convenience we should probably just let the user do a restart
-sleep 5 && sync
-systemctl start cloudron-installer
-journalctl -u cloudron-installer.service -f

installer/package.json

@@ -21,6 +21,7 @@
     "json": "^9.0.3",
     "morgan": "^1.5.1",
     "proxy-middleware": "^0.15.0",
+    "request": "^2.72.0",
     "safetydance": "0.0.19",
     "semver": "^5.1.0",
     "superagent": "^0.21.0"

installer/src/server.js

@@ -16,6 +16,7 @@ var assert = require('assert'),
     json = require('body-parser').json,
     lastMile = require('connect-lastmile'),
     morgan = require('morgan'),
+    request = require('request'),
     superagent = require('superagent');
 
 exports = module.exports = {
@@ -29,17 +30,42 @@ var CLOUDRON_CONFIG_FILE = '/home/yellowtent/configs/cloudron.conf';
 var gHttpServer = null; // update server; used for updates
 
 function provisionDigitalOcean(callback) {
-    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
-
     superagent.get('http://169.254.169.254/metadata/v1.json').end(function (error, result) {
         if (error || result.statusCode !== 200) {
             console.error('Error getting metadata', error);
             return callback(new Error('Error getting metadata'));
         }
 
-        var userData = JSON.parse(result.body.user_data);
+        callback(null, JSON.parse(result.body.user_data));
+    });
+}
 
-        installer.provision(userData, callback);
+function provisionEC2(callback) {
+    // need to use request, since octet-stream data
+    request('http://169.254.169.254/latest/user-data', function (error, response, body) {
+        if (error || response.statusCode !== 200) {
+            console.error('Error getting metadata', error);
+            return callback(new Error('Error getting metadata'));
+        }
+
+        callback(null, JSON.parse(body));
+    });
+}
+
+function provision(callback) {
+    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
+
+    // try first digitalocean, then ec2
+    provisionDigitalOcean(function (error, userData) {
+        if (!error) return installer.provision(userData, callback);
+
+        provisionEC2(function (error, userData) {
+            if (!error) return installer.provision(userData, callback);
+
+            console.error('Unable to get meta data', error);
+
+            callback(new Error('Error getting metadata'));
+        });
     });
 }
 
@@ -122,7 +148,7 @@ function start(callback) {
 
         actions = [
             startUpdateServer,
-            provisionDigitalOcean
+            provision
         ];
     }
 

installer/systemd/box-setup.sh

@@ -16,6 +16,9 @@ if [[ -b "/dev/xvda1" ]]; then
     disk_device="/dev/xvda1"
 fi
 
+# allow root access over ssh
+sed -e 's/.* \(ssh-rsa.*\)/\1/' -i /root/.ssh/authorized_keys
+
 # all sizes are in mb
 readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }')
 readonly swap_size="${physical_memory}" # if you change this, fix enoughResourcesAvailable() in client.js

migrations/20160527010012-mailboxes-add-table.js

@@ -0,0 +1,24 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE mailboxes(' +
+				'name VARCHAR(128) NOT NULL,' +
+				'aliasTarget VARCHAR(128),' +
+				'creationTime TIMESTAMP,' +
+				'PRIMARY KEY (name))';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE mailboxes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160528090040-mailboxes-import-existing-users.js

@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+// imports mailbox entries for existing users
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function addUserMailboxes(done) {
+            db.all('SELECT username FROM users', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (r, next) {
+                    if (!r.username) return next();
+
+                    db.runSql('INSERT INTO mailboxes (name) VALUES (?)', [ r.username ], next);
+                }, done);
+            });
+        },
+       db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+  callback();
+};

migrations/20160614021819-apps-drop-lastBackupConfigJson.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupConfigJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160621032914-apps-alter-installationProgress.js

@@ -0,0 +1,16 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY installationProgress TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY installationProgress VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160714130142-apps-add-xFrameOptions.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN xFrameOptions VARCHAR(512)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN xFrameOptions', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS tokens(
     identifier VARCHAR(128) NOT NULL,
     clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
-    expires BIGINT NOT NULL,
+    expires BIGINT NOT NULL, // FIXME: make this a timestamp
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
@@ -54,7 +54,7 @@ CREATE TABLE IF NOT EXISTS apps(
     id VARCHAR(128) NOT NULL UNIQUE,
     appStoreId VARCHAR(128) NOT NULL,
     installationState VARCHAR(512) NOT NULL,
-    installationProgress VARCHAR(512),
+    installationProgress TEXT,
     runState VARCHAR(512),
     health VARCHAR(128),
     containerId VARCHAR(128),
@@ -62,16 +62,16 @@ CREATE TABLE IF NOT EXISTS apps(
     httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
     location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
-    accessRestrictionJson TEXT,
+    accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
     oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
     memoryLimit BIGINT DEFAULT 0,
     altDomain VARCHAR(256),
+    xFrameOptions VARCHAR(512),
 
-    lastBackupId VARCHAR(128),
-    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+    lastBackupId VARCHAR(128), // tracks last valid backup, can be removed
 
-    oldConfigJson TEXT, // used to pass old config for apptask
+    oldConfigJson TEXT, // used to pass old config for apptask, can be removed when we use a queue
 
     PRIMARY KEY(id));
 
@@ -86,7 +86,7 @@ CREATE TABLE IF NOT EXISTS authcodes(
     authCode VARCHAR(128) NOT NULL UNIQUE,
     userId VARCHAR(128) NOT NULL,
     clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL,
+    expiresAt BIGINT NOT NULL, // ## FIXME: make this a timestamp
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
@@ -115,6 +115,17 @@ CREATE TABLE IF NOT EXISTS eventlog(
     action VARCHAR(128) NOT NULL,
     source JSON, /* { userId, username, ip }. userId can be null for cron,sysadmin */
     data JSON, /* free flowing json based on action */
+    creationTime TIMESTAMP, /* FIXME: precision must be TIMESTAMP(2) */
+
+    PRIMARY KEY (id));
+
+/* Future fields:
+   * accessRestriction - to determine who can access it. So this has foreign keys
+   * quota - per mailbox quota
+*/
+CREATE TABLE IF NOT EXISTS mailboxes(
+    name VARCHAR(128) NOT NULL,
+    aliasTarget VARCHAR(128), /* the target name type is an alias */
     creationTime TIMESTAMP,
 
     PRIMARY KEY (id));

package.json

@@ -16,10 +16,9 @@
     "async": "^1.2.1",
     "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "bytes": "^2.3.0",
-    "cloudron-manifestformat": "^2.3.0",
+    "cloudron-manifestformat": "^2.4.0",
     "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.13",
+    "connect-lastmile": "^0.1.0",
     "connect-timeout": "^1.5.0",
     "cookie-parser": "^1.3.5",
     "cookie-session": "^1.1.0",
@@ -33,6 +32,7 @@
     "express": "^4.12.4",
     "express-session": "^1.11.3",
     "hat": "0.0.3",
+    "ini": "^1.3.4",
     "json": "^9.0.3",
     "ldapjs": "^0.7.1",
     "mime": "^1.3.4",
@@ -56,7 +56,6 @@
     "proxy-middleware": "^0.13.0",
     "safetydance": "^0.1.1",
     "semver": "^4.3.6",
-    "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
     "superagent": "^1.8.3",
     "supererror": "^0.7.1",
@@ -89,7 +88,6 @@
     "mocha": "*",
     "nock": "^3.4.0",
     "node-sass": "^3.0.0-alpha.0",
-    "redis": "^2.4.2",
     "request": "^2.65.0",
     "sinon": "^1.12.2",
     "yargs": "^3.15.0"

setup/argparser.sh

@@ -10,7 +10,8 @@ arg_fqdn=""
 arg_is_custom_domain="false"
 arg_restore_key=""
 arg_restore_url=""
-arg_retire="false"
+arg_retire_reason=""
+arg_retire_info=""
 arg_tls_config=""
 arg_tls_cert=""
 arg_tls_key=""
@@ -23,20 +24,30 @@ arg_update_config=""
 arg_provider=""
 arg_app_bundle=""
 
-args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
+args=$(getopt -o "" -l "data:,retire-reason:,retire-info:" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
-    --retire)
-        arg_retire="true"
-        shift
+    --retire-reason)
+        arg_retire_reason="$2"
+        shift 2
+        ;;
+    --retire-info)
+        arg_retire_info="$2"
+        shift 2
         ;;
     --data)
-        # only read mandatory non-empty parameters here
-        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_is_custom_domain arg_box_versions_url arg_version <<EOF
-        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn isCustomDomain boxVersionsUrl version | tr '\n' ' ')
-EOF
+        # these params must be valid in all cases
+        arg_fqdn=$(echo "$2" | $json fqdn)
+        arg_is_custom_domain=$(echo "$2" | $json isCustomDomain)
+
+        # only update/restore have this valid (but not migrate)
+        arg_api_server_origin=$(echo "$2" | $json apiServerOrigin)
+        arg_web_server_origin=$(echo "$2" | $json webServerOrigin)
+        arg_box_versions_url=$(echo "$2" | $json boxVersionsUrl)
+        arg_version=$(echo "$2" | $json version)
+
         # read possibly empty parameters here
         arg_app_bundle=$(echo "$2" | $json appBundle)
         [[ "${arg_app_bundle}" == "" ]] && arg_app_bundle="[]"

setup/container/mysql.cnf

@@ -5,3 +5,6 @@
 [mysqld]
 performance_schema=OFF
 max_connections=50
+# on ec2, without this we get a sporadic connection drop when doing the initial migration
+max_allowed_packet=32M
+

setup/container/sudoers

@@ -31,6 +31,3 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh
 
-Defaults!/home/yellowtent/box/src/scripts/setup_infra.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setup_infra.sh
-

setup/splashpage.sh

@@ -9,8 +9,6 @@ readonly BOX_SRC_DIR="/home/yellowtent/box"
 readonly DATA_DIR="/home/yellowtent/data"
 readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
 
-source "${script_dir}/../src/INFRA_VERSION" # this injects INFRA_VERSION
-
 echo "Setting up nginx update page"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
@@ -24,17 +22,24 @@ rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
 
 # create nginx config
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+readonly current_infra=$(node -e "console.log(require('${script_dir}/../src/infra_version.js').version);")
+existing_infra="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && existing_infra=$(node -e "console.log(JSON.parse(require('fs').readFileSync('${DATA_DIR}/INFRA_VERSION', 'utf8')).version);")
+if [[ "${arg_retire_reason}" != "" || "${existing_infra}" != "${current_infra}" ]]; then
+    echo "Showing progress bar on all subdomains in retired mode or infra update. retire: ${arg_retire_reason} existing: ${existing_infra} current: ${current_infra}"
     rm -f ${DATA_DIR}/nginx/applications/*
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 else
+    echo "Show progress bar only on admin domain for normal update"
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 fi
 
-echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
+if [[ "${arg_retire_reason}" == "migrate" ]]; then
+    echo "{ \"migrate\": { \"percent\": \"10\", \"message\": \"Migrating cloudron. This could take up to 15 minutes.\", \"info\": ${arg_retire_info} }, \"backup\": null, \"apiServerOrigin\": \"${arg_api_server_origin}\" }" > "${SETUP_WEBSITE_DIR}/progress.json"
+else
+    echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
+fi
 
 nginx -s reload

setup/start.sh

@@ -34,6 +34,14 @@ set_progress() {
 set_progress "1" "Create container"
 $script_dir/container.sh
 
+set_progress "5" "Adjust system settings"
+hostnamectl set-hostname "${arg_fqdn}"
+
+# ec2 instances use lots of cpu for swapping, which can be significantly reduced adjusting the swappiness
+if [[ "${arg_provider}" == 'ec2' ]]; then
+    sysctl vm.swappiness=0
+fi
+
 set_progress "10" "Ensuring directories"
 # keep these in sync with paths.js
 [[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
@@ -107,7 +115,7 @@ if [[ -f "${DATA_DIR}/box/certs/${admin_fqdn}.cert" && -f "${DATA_DIR}/box/certs
     admin_key_file="${DATA_DIR}/box/certs/${admin_fqdn}.key"
 fi
 ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 
 mkdir -p "${DATA_DIR}/nginx/cert"
 if [[ -f "${DATA_DIR}/box/certs/host.cert" && -f "${DATA_DIR}/box/certs/host.key" ]]; then
@@ -120,6 +128,7 @@ fi
 
 set_progress "33" "Changing ownership"
 chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+chown "${USER}:${USER}" "${DATA_DIR}/INFRA_VERSION" || true
 chown "${USER}:${USER}" "${DATA_DIR}"
 
 set_progress "65" "Creating cloudron.conf"
@@ -135,7 +144,6 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
     "fqdn": "${arg_fqdn}",
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
-    "adminEmail": "\"Cloudron\" <no-reply@${arg_fqdn}>",
     "provider": "${arg_provider}",
     "database": {
         "hostname": "localhost",
@@ -188,18 +196,20 @@ if [[ ! -z "${arg_tls_config}" ]]; then
         -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
 fi
 
-# Add webadmin oauth client
 # The domain might have changed, therefor we have to update the record
 # !!! This needs to be in sync with the webadmin, specifically login_callback.js
-echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add webadmin api cient"
+readonly ADMIN_SCOPES="cloudron,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"Settings\", \"built-in\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
 
-echo "Add localhost test oauth client"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add SDK api client"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-sdk\", \"SDK\", \"built-in\", \"secret-sdk\", \"${admin_origin}\", \"*,roleSdk\")" box
+
+echo "Add cli api client"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-cli\", \"Cloudron Tool\", \"built-in\", \"secret-cli\", \"${admin_origin}\", \"*,roleSdk\")" box
 
 set_progress "80" "Starting Cloudron"
 systemctl start cloudron.target

setup/start/nginx/appconfig.ejs

@@ -24,6 +24,9 @@ server {
     ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
     add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
 
+    # https://developer.mozilla.org/en-US/docs/Web/HTTP/X-Frame-Options
+    add_header X-Frame-Options "<%= xFrameOptions %>";
+
     proxy_http_version 1.1;
     proxy_intercept_errors on;
     proxy_read_timeout       3500;
@@ -59,6 +62,7 @@ server {
             client_max_body_size 1m;
         }
 
+        # the read timeout is between successive reads and not the whole connection
         location ~ ^/api/v1/apps/.*/exec$ {
             proxy_pass   http://127.0.0.1:3000;
             proxy_read_timeout 30m;

setup/start/nginx/nginx.conf

@@ -24,7 +24,14 @@ http {
 
     sendfile        on;
 
-    keepalive_timeout  65;
+    # timeout for client to finish sending headers
+    client_header_timeout 30s;
+
+    # timeout for reading client request body (successive read timeout and not whole body!)
+    client_body_timeout 60s;
+
+    # keep-alive connections timeout in 65s. this is because many browsers timeout in 60 seconds
+    keepalive_timeout  65s;
 
     # HTTP server
     server {
@@ -50,22 +57,15 @@ http {
         }
     }
 
-    # We have to enable https for nginx to read in the vhost in http request
-    # and send a 404. This is a side-effect of using wildcard DNS
+    # This server handles the naked domain for custom domains.
+    # It can also be used for wildcard subdomain 404. This feature is not used by the Cloudron itself
+    # because box always sets up DNS records for app subdomains.
     server {
         listen 443 default_server;
         ssl on;
         ssl_certificate      cert/host.cert;
         ssl_certificate_key  cert/host.key;
 
-        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
-        proxy_buffer_size       128k;
-        proxy_buffers           4 256k;
-        proxy_busy_buffers_size 256k;
-
-        # Disable check to allow unlimited body sizes
-        client_max_body_size 0;
-
         error_page 404 = @fallback;
         location @fallback {
             internal;
@@ -79,6 +79,7 @@ http {
             rewrite ^/$ /nakeddomain.html break;
         }
 
+        # required for /api/v1/cloudron/avatar
         location /api/ {
             proxy_pass http://127.0.0.1:3000;
             client_max_body_size 1m;

src/INFRA_VERSION

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# If you change the infra version, be sure to put a warning
-# in the change log
-
-INFRA_VERSION=30
-
-# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
-# These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.11.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.9.0
-MONGODB_IMAGE=cloudron/mongodb:0.9.0
-REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.12.0
-GRAPHITE_IMAGE=cloudron/graphite:0.8.0
-
-MYSQL_REPO=cloudron/mysql
-POSTGRESQL_REPO=cloudron/postgresql
-MONGODB_REPO=cloudron/mongodb
-REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
-MAIL_REPO=cloudron/mail
-GRAPHITE_REPO=cloudron/graphite

src/addons.js

@@ -1,15 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    initialize: initialize,
-
     setupAddons: setupAddons,
     teardownAddons: teardownAddons,
     backupAddons: backupAddons,
     restoreAddons: restoreAddons,
 
     getEnvironment: getEnvironment,
-    getLinksSync: getLinksSync,
     getBindsSync: getBindsSync,
     getContainerNamesSync: getContainerNamesSync,
 
@@ -21,29 +18,34 @@ exports = module.exports = {
 var appdb = require('./appdb.js'),
     assert = require('assert'),
     async = require('async'),
-    certificates = require('./certificates.js'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
-    DatabaseError = require('./databaseerror.js'),
+    ClientsError = clients.ClientsError,
     debug = require('debug')('box:addons'),
     docker = require('./docker.js'),
     dockerConnection = docker.connection,
     fs = require('fs'),
     generatePassword = require('password-generator'),
     hat = require('hat'),
+    infra = require('./infra_version.js'),
     once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');
 
 var NOOP = function (app, options, callback) { return callback(); };
 
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
+    email: {
+        setup: setupEmail,
+        teardown: teardownEmail,
+        backup: NOOP,
+        restore: setupEmail
+    },
     ldap: {
         setup: setupLdap,
         teardown: teardownLdap,
@@ -80,6 +82,12 @@ var KNOWN_ADDONS = {
         backup: backupPostgreSql,
         restore: restorePostgreSql
     },
+    recvmail: {
+        setup: setupRecvMail,
+        teardown: teardownRecvMail,
+        backup: NOOP,
+        restore: setupRecvMail
+    },
     redis: {
         setup: setupRedis,
         teardown: teardownRedis,
@@ -112,8 +120,7 @@ var KNOWN_ADDONS = {
     }
 };
 
-var RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh'),
-    SETUP_INFRA_CMD = path.join(__dirname, 'scripts/setup_infra.sh');;
+var RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh');
 
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
@@ -122,17 +129,6 @@ function debugApp(app, args) {
     debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
 }
 
-function initialize(callback) {
-    if (process.env.BOX_ENV === 'test') return callback();
-
-    debug('initializing addon infrastructure');
-    certificates.getAdminCertificatePath(function (error, certFilePath, keyFilePath) {
-        if (error) return callback(error);
-
-        shell.sudo('seutp_infra', [ SETUP_INFRA_CMD, config.fqdn(), config.adminFqdn(), certFilePath, keyFilePath ], callback);
-    });
-}
-
 function setupAddons(app, addons, callback) {
     assert.strictEqual(typeof app, 'object');
     assert(!addons || typeof addons === 'object');
@@ -212,28 +208,6 @@ function getEnvironment(app, callback) {
     appdb.getAddonConfigByAppId(app.id, callback);
 }
 
-function getLinksSync(app, addons) {
-    assert.strictEqual(typeof app, 'object');
-    assert(!addons || typeof addons === 'object');
-
-    var links = [ ];
-
-    if (!addons) return links;
-
-    for (var addon in addons) {
-        switch (addon) {
-        case 'mysql': links.push('mysql:mysql'); break;
-        case 'postgresql': links.push('postgresql:postgresql'); break;
-        case 'sendmail': links.push('mail:mail'); break;
-        case 'redis': links.push('redis-' + app.id + ':redis-' + app.id); break;
-        case 'mongodb': links.push('mongodb:mongodb'); break;
-        default: break;
-        }
-    }
-
-    return links;
-}
-
 function getBindsSync(app, addons) {
     assert.strictEqual(typeof app, 'object');
     assert(!addons || typeof addons === 'object');
@@ -280,22 +254,18 @@ function setupOauth(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile';
 
-    debugApp(app, 'setupOauth: id:%s clientSecret:%s', id, clientSecret);
+    clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);
 
-    clientdb.delByAppIdAndType(appId, clientdb.TYPE_OAUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_OAUTH, clientSecret, redirectURI, scope, function (error) {
+        clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) {
             if (error) return callback(error);
 
             var env = [
-                'OAUTH_CLIENT_ID=' + id,
-                'OAUTH_CLIENT_SECRET=' + clientSecret,
+                'OAUTH_CLIENT_ID=' + result.id,
+                'OAUTH_CLIENT_SECRET=' + result.clientSecret,
                 'OAUTH_ORIGIN=' + config.adminOrigin()
             ];
 
@@ -313,8 +283,8 @@ function teardownOauth(app, options, callback) {
 
     debugApp(app, 'teardownOauth');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_OAUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'oauth', callback);
     });
@@ -326,23 +296,20 @@ function setupSimpleAuth(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-' + uuid.v4();
     var scope = 'profile';
 
-    debugApp(app, 'setupSimpleAuth: id:%s', id);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_SIMPLE_AUTH, '', '', scope, function (error) {
+        clients.add(appId, clients.TYPE_SIMPLE_AUTH, '', scope, function (error, result) {
             if (error) return callback(error);
 
             var env = [
-                'SIMPLE_AUTH_SERVER=172.17.0.1',
+                'SIMPLE_AUTH_SERVER=172.18.0.1',
                 'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
-                'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_CLIENT_ID=' + id
+                'SIMPLE_AUTH_URL=http://172.18.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
+                'SIMPLE_AUTH_ORIGIN=http://172.18.0.1:' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_CLIENT_ID=' + result.id
             ];
 
             debugApp(app, 'Setting simple auth addon config to %j', env);
@@ -359,26 +326,57 @@ function teardownSimpleAuth(app, options, callback) {
 
     debugApp(app, 'teardownSimpleAuth');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'simpleauth', callback);
     });
 }
 
+function setupEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation)
+    var env = [
+        'MAIL_SMTP_SERVER=mail',
+        'MAIL_SMTP_PORT=2525',
+        'MAIL_IMAP_SERVER=mail',
+        'MAIL_IMAP_PORT=9993',
+        'MAIL_SIEVE_SERVER=mail',
+        'MAIL_SIEVE_PORT=4190',
+        'MAIL_DOMAIN=' + config.fqdn()
+    ];
+
+    debugApp(app, 'Setting up Email');
+
+    appdb.setAddonConfig(app.id, 'email', env, callback);
+}
+
+function teardownEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Tearing down Email');
+
+    appdb.unsetAddonConfig(app.id, 'email', callback);
+}
+
 function setupLdap(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var env = [
-        'LDAP_SERVER=172.17.0.1',
+        'LDAP_SERVER=172.18.0.1',
         'LDAP_PORT=' + config.get('ldapPort'),
-        'LDAP_URL=ldap://172.17.0.1:' + config.get('ldapPort'),
+        'LDAP_URL=ldap://172.18.0.1:' + config.get('ldapPort'),
         'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
         'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron',
         'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron',
-        'LDAP_BIND_PASSWORD=' + hat(256) // this is ignored
+        'LDAP_BIND_PASSWORD=' + hat(8 * 128) // this is ignored
     ];
 
     debugApp(app, 'Setting up LDAP');
@@ -401,20 +399,17 @@ function setupSendMail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '-app';
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
 
-    var env = [
-        'MAIL_SMTP_SERVER=mail',
-        'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
-        'MAIL_SMTP_USERNAME=' + from, // change this to app.id after apps have moved
-        'MAIL_SMTP_PASSWORD=' + 'app-' + app.id, // this is ignored
-        'MAIL_FROM=' + from + '@' + config.fqdn(),
-        'MAIL_DOMAIN=' + config.fqdn()
-    ];
+    var cmd = [ '/addons/mail/service.sh', 'add-send', from ];
 
-    debugApp(app, 'Setting up sendmail');
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
 
-    appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting sendmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+    });
 }
 
 function teardownSendMail(app, options, callback) {
@@ -422,9 +417,55 @@ function teardownSendMail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debugApp(app, 'Tearing down sendmail');
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
 
-    appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    var cmd = [ '/addons/mail/service.sh', 'remove-send', from ];
+
+    debugApp(app, 'Tearing down sendmail : %j', cmd);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    });
+}
+
+function setupRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up recvmail');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'add-recv', to ];
+
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
+
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting recvmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'recvmail', env, callback);
+    });
+}
+
+function teardownRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'remove-recv', to ];
+
+    debugApp(app, 'Tearing down recvmail: %j', cmd);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'recvmail', callback);
+    });
 }
 
 function setupMySql(app, options, callback) {
@@ -617,38 +658,6 @@ function restoreMongoDb(app, options, callback) {
     });
 }
 
-function forwardRedisPort(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    dockerConnection.getContainer('redis-' + appId).inspect(function (error, data) {
-        if (error) return callback(new Error('Unable to inspect container:' + error));
-
-        var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
-        if (!Number.isInteger(redisPort)) return callback(new Error('Unable to get container port mapping'));
-
-        return callback(null);
-    });
-}
-
-function stopAndRemoveRedis(container, callback) {
-    function ignoreError(func) {
-        return function (callback) {
-            func(function (error) {
-                if (error) debug('stopAndRemoveRedis: Ignored error:', error);
-                callback();
-            });
-        };
-    }
-
-    // stopping redis with SIGTERM makes it commit the database to disk
-    async.series([
-        ignoreError(container.stop.bind(container, { t: 10 })),
-        ignoreError(container.wait.bind(container)),
-        ignoreError(container.remove.bind(container, { force: true, v: true }))
-    ], callback);
-}
-
 // Ensures that app's addon redis container is running. Can be called when named container already exists/running
 function setupRedis(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
@@ -665,57 +674,32 @@ function setupRedis(app, options, callback) {
 
     if (!safe.fs.mkdirSync(redisDataDir) && safe.error.code !== 'EEXIST') return callback(new Error('Error creating redis data dir:' + safe.error));
 
-    var createOptions = {
-        name: 'redis-' + app.id,
-        Hostname: 'redis-' + app.location,
-        Tty: true,
-        Image: 'cloudron/redis:0.8.0', // if you change this, fix src/INFRA_VERSION as well
-        Cmd: null,
-        Volumes: {
-            '/tmp': {},
-            '/run': {}
-        },
-        VolumesFrom: [],
-        HostConfig: {
-            Binds: [
-                redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
-                redisDataDir + ':/var/lib/redis:rw'
-            ],
-            Memory: 1024 * 1024 * 75, // 100mb
-            MemorySwap: 1024 * 1024 * 75 * 2, // 150mb
-            PortBindings: {
-                '6379/tcp': [{ HostPort: '0', HostIp: '127.0.0.1' }]
-            },
-            ReadonlyRootfs: true,
-            RestartPolicy: {
-                'Name': 'always',
-                'MaximumRetryCount': 0
-            }
-        }
-    };
+    const tag = infra.images.redis.tag, redisName = 'redis-' + app.id;
+    const cmd = `docker run --restart=always -d --name=${redisName} \
+                --net cloudron \
+                --net-alias ${redisName} \
+                -m 100m \
+                --memory-swap 150m \
+                -v ${redisVarsFile}:/etc/redis/redis_vars.sh:ro \
+                -v ${redisDataDir}:/var/lib/redis:rw \
+                --read-only -v /tmp -v /run ${tag}`;
 
     var env = [
         'REDIS_URL=redis://redisuser:' + redisPassword + '@redis-' + app.id,
         'REDIS_PASSWORD=' + redisPassword,
-        'REDIS_HOST=redis-' + app.id,
+        'REDIS_HOST=' + redisName,
         'REDIS_PORT=6379'
     ];
 
-    var redisContainer = dockerConnection.getContainer(createOptions.name);
-    stopAndRemoveRedis(redisContainer, function () {
-        dockerConnection.createContainer(createOptions, function (error) {
-            if (error && error.statusCode !== 409) return callback(error); // if not already created
-
-            redisContainer.start(function (error) {
-                if (error && error.statusCode !== 304) return callback(error); // if not already running
-
-                appdb.setAddonConfig(app.id, 'redis', env, function (error) {
-                    if (error) return callback(error);
-
-                    forwardRedisPort(app.id, callback);
-                });
-            });
-        });
+    async.series([
+        // stop so that redis can flush itself with SIGTERM
+        shell.execSync.bind(null, 'stopRedis', `docker stop --time=10 ${redisName} 2>/dev/null || true`),
+        shell.execSync.bind(null, 'stopRedis', `docker rm --volumes ${redisName} 2>/dev/null || true`),
+        shell.execSync.bind(null, 'startRedis', cmd),
+        appdb.setAddonConfig.bind(null, app.id, 'redis', env)
+    ], function (error) {
+        if (error) debug('Error setting up redis: ', error);
+        callback(error);
     });
 }
 

src/appdb.js

@@ -26,6 +26,7 @@ exports = module.exports = {
 
     // installation codes (keep in sync in UI)
     ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
+    ISTATE_PENDING_CLONE: 'pending_clone', // clone
     ISTATE_PENDING_CONFIGURE: 'pending_configure', // config (location, port) changes and on infra update
     ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
     ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
@@ -58,7 +59,7 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain', 'apps.xFrameOptions' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -69,10 +70,6 @@ function postProcess(result) {
     result.manifest = safe.JSON.parse(result.manifestJson);
     delete result.manifestJson;
 
-    assert(result.lastBackupConfigJson === null || typeof result.lastBackupConfigJson === 'string');
-    result.lastBackupConfig = safe.JSON.parse(result.lastBackupConfigJson);
-    delete result.lastBackupConfigJson;
-
     assert(result.oldConfigJson === null || typeof result.oldConfigJson === 'string');
     result.oldConfig = safe.JSON.parse(result.oldConfigJson);
     delete result.oldConfigJson;
@@ -95,6 +92,9 @@ function postProcess(result) {
     result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
     if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
     delete result.accessRestrictionJson;
+
+    // TODO remove later once all apps have this attribute
+    result.xFrameOptions = result.xFrameOptions || 'SAMEORIGIN';
 }
 
 function get(id, callback) {
@@ -160,27 +160,32 @@ function getAll(callback) {
     });
 }
 
-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, altDomain, callback) {
+function add(id, appStoreId, manifest, location, portBindings, data, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof manifest.version, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert.strictEqual(typeof memoryLimit, 'number');
-    assert(altDomain === null || typeof altDomain === 'string');
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
 
     var manifestJson = JSON.stringify(manifest);
+
+    var accessRestriction = data.accessRestriction || null;
     var accessRestrictionJson = JSON.stringify(accessRestriction);
+    var memoryLimit = data.memoryLimit || 0;
+    var altDomain = data.altDomain || null;
+    var xFrameOptions = data.xFrameOptions || '';
+    var installationState = data.installationState || exports.ISTATE_PENDING_INSTALL;
+    var lastBackupId = data.lastBackupId || null; // used when cloning
 
     var queries = [ ];
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit, altDomain ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId ]
     });
 
     Object.keys(portBindings).forEach(function (env) {
@@ -284,9 +289,6 @@ function updateWithConstraints(id, app, constraints, callback) {
         if (p === 'manifest') {
             fields.push('manifestJson = ?');
             values.push(JSON.stringify(app[p]));
-        } else if (p === 'lastBackupConfig') {
-            fields.push('lastBackupConfigJson = ?');
-            values.push(JSON.stringify(app[p]));
         } else if (p === 'oldConfig') {
             fields.push('oldConfigJson = ?');
             values.push(JSON.stringify(app[p]));
@@ -345,14 +347,17 @@ function setInstallationCommand(appId, installationState, values, callback) {
     // uninstall is allowed in any state
     // force update is allowed in any state including pending_uninstall! (for better or worse)
     // restore is allowed from installed or error state
-    // update and configure are allowed only in installed state
+    // configure is allowed in installed state or currently configuring or in error state
+    // update and backup are allowed only in installed state
 
     if (installationState === exports.ISTATE_PENDING_UNINSTALL || installationState === exports.ISTATE_PENDING_FORCE_UPDATE) {
         updateWithConstraints(appId, values, '', callback);
     } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
         updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
-    } else if (installationState === exports.ISTATE_PENDING_UPDATE || installationState === exports.ISTATE_PENDING_CONFIGURE || installationState === exports.ISTATE_PENDING_BACKUP) {
+    } else if (installationState === exports.ISTATE_PENDING_UPDATE || installationState === exports.ISTATE_PENDING_BACKUP) {
         updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
+    } else if (installationState === exports.ISTATE_PENDING_CONFIGURE) {
+        updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "pending_configure" OR installationState = "error")', callback);
     } else {
         callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));
     }

src/apphealthmonitor.js

@@ -17,7 +17,7 @@ exports = module.exports = {
 };
 
 var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
-var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
+var UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes
 var gHealthInfo = { }; // { time, emailSent }
 var gRunTimeout = null;
 var gDockerEventStream = null;
@@ -138,13 +138,16 @@ function run() {
 
 /*
     OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:0.3.3 /bin/bash
+        docker run -ti -m 100M cloudron/base:0.8.1 /bin/bash
         apt-get update && apt-get install stress
         stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
 function processDockerEvents() {
     // note that for some reason, the callback is called only on the first event
     debug('Listening for docker events');
+    const OOM_MAIL_LIMIT = 60 * 60 * 1000; // 60 minutes
+    var lastOomMailTime = new Date(new Date() - OOM_MAIL_LIMIT);
+
     docker.getEvents({ filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
         if (error) return console.error(error);
 
@@ -154,15 +157,19 @@ function processDockerEvents() {
         stream.on('data', function (data) {
             var ev = JSON.parse(data);
             debug('Container ' + ev.id + ' went OOM');
-            appdb.getByContainerId(ev.id, function (error, app) {
+            appdb.getByContainerId(ev.id, function (error, app) { // this can error for addons
                 var program = error || !app.appStoreId ? ev.id : app.appStoreId;
                 var context = JSON.stringify(ev);
+                var now = new Date();
                 if (app) context = context + '\n\n' + JSON.stringify(app, null, 4) + '\n';
 
                 debug('OOM Context: %s', context);
 
                 // do not send mails for dev apps
-                if (error || app.appStoreId !== '') mailer.unexpectedExit(program, context); // app can be null if it's an addon crash
+                if ((!app || app.appStoreId !== '') && (now - lastOomMailTime > OOM_MAIL_LIMIT)) {
+                    mailer.unexpectedExit(program, context); // app can be null if it's an addon crash
+                    lastOomMailTime = now;
+                }
             });
         });
 
@@ -194,7 +201,7 @@ function stop(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clearTimeout(gRunTimeout);
-    gDockerEventStream.end();
+    if (gDockerEventStream) gDockerEventStream.end();
 
     callback();
 }

src/apps.js

@@ -15,6 +15,7 @@ exports = module.exports = {
     uninstall: uninstall,
 
     restore: restore,
+    clone: clone,
 
     update: update,
 
@@ -30,7 +31,12 @@ exports = module.exports = {
 
     checkManifestConstraints: checkManifestConstraints,
 
-    autoupdateApps: autoupdateApps,
+    updateApps: updateApps,
+
+    restoreInstalledApps: restoreInstalledApps,
+    configureInstalledApps: configureInstalledApps,
+
+    getAppConfig: getAppConfig,
 
     // exported for testing
     _validateHostname: validateHostname,
@@ -43,6 +49,7 @@ var addons = require('./addons.js'),
     assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
+    BackupsError = backups.BackupsError,
     certificates = require('./certificates.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
@@ -61,7 +68,9 @@ var addons = require('./addons.js'),
     split = require('split'),
     superagent = require('superagent'),
     taskmanager = require('./taskmanager.js'),
+    url = require('url'),
     util = require('util'),
+    uuid = require('node-uuid'),
     validator = require('validator');
 
 // http://dustinsenos.com/articles/customErrorsInNode
@@ -103,16 +112,16 @@ AppsError.BAD_CERTIFICATE = 'Invalid certificate';
 // https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
 // We are validating the validity of the location-fqdn as host name
 function validateHostname(location, fqdn) {
-    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION, constants.SMTP_LOCATION, constants.IMAP_LOCATION ];
+    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION, constants.SMTP_LOCATION, constants.IMAP_LOCATION, constants.MAIL_LOCATION, constants.POSTMAN_LOCATION ];
 
-    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new Error(location + ' is reserved');
+    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new AppsError(AppsError.BAD_FIELD, location + ' is reserved');
 
     if (location === '') return null; // bare location
 
-    if ((location.length + 1 /*+ hyphen */ + fqdn.indexOf('.')) > 63) return new Error('Hostname length cannot be greater than 63');
-    if (location.match(/^[A-Za-z0-9-]+$/) === null) return new Error('Hostname can only contain alphanumerics and hyphen');
-    if (location[0] === '-' || location[location.length-1] === '-') return new Error('Hostname cannot start or end with hyphen');
-    if (location.length + 1 /* hyphen */ + fqdn.length > 253) return new Error('FQDN length exceeds 253 characters');
+    if ((location.length + 1 /*+ hyphen */ + fqdn.indexOf('.')) > 63) return new AppsError(AppsError.BAD_FIELD, 'Hostname length cannot be greater than 63');
+    if (location.match(/^[A-Za-z0-9-]+$/) === null) return new AppsError(AppsError.BAD_FIELD, 'Hostname can only contain alphanumerics and hyphen');
+    if (location[0] === '-' || location[location.length-1] === '-') return new AppsError(AppsError.BAD_FIELD, 'Hostname cannot start or end with hyphen');
+    if (location.length + 1 /* hyphen */ + fqdn.length > 253) return new AppsError(AppsError.BAD_FIELD, 'FQDN length exceeds 253 characters');
 
     return null;
 }
@@ -137,10 +146,12 @@ function validatePortBindings(portBindings, tcpPorts) {
         2020, /* install server */
         config.get('port'), /* app server (lo) */
         config.get('sysadminPort'), /* sysadmin app server (lo) */
+        config.get('smtpPort'), /* internal smtp port (lo) */
         config.get('ldapPort'), /* ldap server (lo) */
         config.get('oauthProxyPort'), /* oauth proxy server (lo) */
         config.get('simpleAuthPort'), /* simple auth server (lo) */
         3306, /* mysql (lo) */
+        4190, /* managesieve */
         8000 /* graphite (lo) */
     ];
 
@@ -150,8 +161,8 @@ function validatePortBindings(portBindings, tcpPorts) {
     for (env in portBindings) {
         if (!/^[a-zA-Z0-9_]+$/.test(env)) return new AppsError(AppsError.BAD_FIELD, env + ' is not valid environment variable');
 
-        if (!Number.isInteger(portBindings[env])) return new Error(portBindings[env] + ' is not an integer');
-        if (portBindings[env] <= 0 || portBindings[env] > 65535) return new Error(portBindings[env] + ' is out of range');
+        if (!Number.isInteger(portBindings[env])) return new AppsError(AppsError.BAD_FIELD, portBindings[env] + ' is not an integer');
+        if (portBindings[env] <= 0 || portBindings[env] > 65535) return new AppsError(AppsError.BAD_FIELD, portBindings[env] + ' is out of range');
 
         if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, String(portBindings[env]));
     }
@@ -174,19 +185,20 @@ function validateAccessRestriction(accessRestriction) {
     var noUsers = true, noGroups = true;
 
     if (accessRestriction.users) {
-        if (!Array.isArray(accessRestriction.users)) return new Error('users array property required');
-        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new Error('All users have to be strings');
+        if (!Array.isArray(accessRestriction.users)) return new AppsError(AppsError.BAD_FIELD, 'users array property required');
+        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new AppsError(AppsError.BAD_FIELD, 'All users have to be strings');
         noUsers = accessRestriction.users.length === 0;
     }
 
     if (accessRestriction.groups) {
-        if (!Array.isArray(accessRestriction.groups)) return new Error('groups array property required');
-        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new Error('All groups have to be strings');
+        if (!Array.isArray(accessRestriction.groups)) return new AppsError(AppsError.BAD_FIELD, 'groups array property required');
+        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new AppsError(AppsError.BAD_FIELD, 'All groups have to be strings');
         noGroups = accessRestriction.groups.length === 0;
     }
 
-    if (noUsers && noGroups) return new Error('users and groups array cannot both be empty');
+    if (noUsers && noGroups) return new AppsError(AppsError.BAD_FIELD, 'users and groups array cannot both be empty');
 
+    // TODO: maybe validate if the users and groups actually exist
     return null;
 }
 
@@ -201,12 +213,26 @@ function validateMemoryLimit(manifest, memoryLimit) {
     // this is needed so an app update can change the value in the manifest, and if not set by the user, the new value should be used
     if (memoryLimit === 0) return null;
 
-    if (memoryLimit < min) return new Error('memoryLimit too small');
-    if (memoryLimit > max) return new Error('memoryLimit too large');
+    if (memoryLimit < min) return new AppsError(AppsError.BAD_FIELD, 'memoryLimit too small');
+    if (memoryLimit > max) return new AppsError(AppsError.BAD_FIELD, 'memoryLimit too large');
 
     return null;
 }
 
+// https://tools.ietf.org/html/rfc7034
+function validateXFrameOptions(xFrameOptions) {
+    assert.strictEqual(typeof xFrameOptions, 'string');
+
+    if (xFrameOptions === 'DENY') return null;
+    if (xFrameOptions === 'SAMEORIGIN') return null;
+
+    var parts = xFrameOptions.split(' ');
+    if (parts.length !== 2 || parts[0] !== 'ALLOW-FROM') return new AppsError(AppsError.BAD_FIELD, 'xFrameOptions must be "DENY", "SAMEORIGIN" or "ALLOW-FROM uri"' );
+
+    var uri = url.parse(parts[1]);
+    return (uri.protocol === 'http:' || uri.protocol === 'https:') ? null : new AppsError(AppsError.BAD_FIELD, 'xFrameOptions ALLOW-FROM uri must be a valid http[s] uri' );
+}
+
 function getDuplicateErrorDetails(location, portBindings, error) {
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
@@ -229,6 +255,18 @@ function getDuplicateErrorDetails(location, portBindings, error) {
     return new AppsError(AppsError.ALREADY_EXISTS);
 }
 
+function getAppConfig(app) {
+    return {
+        manifest: app.manifest,
+        location: app.location,
+        accessRestriction: app.accessRestriction,
+        portBindings: app.portBindings,
+        memoryLimit: app.memoryLimit,
+        xFrameOptions: app.xFrameOptions || 'SAMEORIGIN',
+        altDomain: app.altDomain
+    };
+}
+
 function getIconUrlSync(app) {
     var iconPath = paths.APPICONS_DIR + '/' + app.id + '.png';
     return fs.existsSync(iconPath) ? '/api/v1/apps/' + app.id + '/icon' : null;
@@ -342,145 +380,194 @@ function purchase(appStoreId, callback) {
     });
 }
 
-function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, cert, key, memoryLimit, altDomain, auditSource, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof appStoreId, 'string');
-    assert(manifest && typeof manifest === 'object');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert(!icon || typeof icon === 'string');
-    assert(cert === null || typeof cert === 'string');
-    assert(key === null || typeof key === 'string');
-    assert.strictEqual(typeof memoryLimit, 'number');
-    assert(altDomain === null || typeof altDomain === 'string');
+function downloadManifest(appStoreId, manifest, callback) {
+    if (!appStoreId && !manifest) return callback(new AppsError(AppsError.BAD_FIELD, 'Neither manifest nor appStoreId provided'));
+
+    if (!appStoreId) return callback(null, '', manifest);
+
+    var parts = appStoreId.split('@');
+
+    var url = config.apiServerOrigin() + '/api/v1/apps/' + parts[0] + (parts[1] ? '/versions/' + parts[1] : '');
+
+    debug('downloading manifest from %s', url);
+
+    superagent.get(url).end(function (error, result) {
+        if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Network error downloading manifest:' + error.message));
+
+        if (result.statusCode !== 200) return callback(new AppsError(AppsError.BAD_FIELD, util.format('Failed to get app info from store.', result.statusCode, result.text)));
+
+        callback(null, parts[0], result.body.manifest);
+    });
+}
+
+function install(data, auditSource, callback) {
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var error = manifestFormat.parse(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error: ' + error.message));
+    var location = data.location.toLowerCase(),
+        portBindings = data.portBindings || null,
+        accessRestriction = data.accessRestriction || null,
+        icon = data.icon || null,
+        cert = data.cert || null,
+        key = data.key || null,
+        memoryLimit = data.memoryLimit || 0,
+        altDomain = data.altDomain || null,
+        xFrameOptions = data.xFrameOptions || 'SAMEORIGIN';
 
-    error = checkManifestConstraints(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+    assert(data.appStoreId || data.manifest); // atleast one of them is required
 
-    error = validateHostname(location, config.fqdn());
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = validatePortBindings(portBindings, manifest.tcpPorts);
-    if (error) return callback(error);
-
-    error = validateAccessRestriction(accessRestriction);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = validateMemoryLimit(manifest, memoryLimit);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    // memoryLimit might come in as 0 if not specified
-    memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
-
-    if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
-
-    // singleUser mode requires accessRestriction to contain exactly one user
-    if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
-    if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
-
-    if (icon) {
-        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
-
-        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
-            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
-        }
-    }
-
-    error = certificates.validateCertificate(cert, key, config.appFqdn(location));
-    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
-
-    debug('Will install app with id : ' + appId);
-
-    purchase(appStoreId, function (error) {
+    downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
         if (error) return callback(error);
 
-        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, memoryLimit, altDomain, function (error) {
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        error = manifestFormat.parse(manifest);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error: ' + error.message));
 
-            // save cert to data/box/certs
-            if (cert && key) {
-                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
-                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+        error = checkManifestConstraints(manifest);
+        if (error) return callback(error);
+
+        error = validateHostname(location, config.fqdn());
+        if (error) return callback(error);
+
+        error = validatePortBindings(portBindings, manifest.tcpPorts);
+        if (error) return callback(error);
+
+        error = validateAccessRestriction(accessRestriction);
+        if (error) return callback(error);
+
+        error = validateMemoryLimit(manifest, memoryLimit);
+        if (error) return callback(error);
+
+        error = validateXFrameOptions(xFrameOptions);
+        if (error) return callback(error);
+
+        // memoryLimit might come in as 0 if not specified
+        memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+
+        if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
+
+        // singleUser mode requires accessRestriction to contain exactly one user
+        if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
+        if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
+
+        var appId = uuid.v4();
+
+        if (icon) {
+            if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
+
+            if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
+                return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
             }
+        }
 
-            taskmanager.restartAppTask(appId);
+        error = certificates.validateCertificate(cert, key, config.appFqdn(location));
+        if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
 
-            eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId: appId, location: location, manifest: manifest });
+        debug('Will install app with id : ' + appId);
 
-            callback(null);
+        purchase(appStoreId, function (error) {
+            if (error) return callback(error);
+
+            var data = {
+                accessRestriction: accessRestriction,
+                memoryLimit: memoryLimit,
+                altDomain: altDomain,
+                xFrameOptions: xFrameOptions
+            };
+
+            appdb.add(appId, appStoreId, manifest, location, portBindings, data, function (error) {
+                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                // save cert to data/box/certs
+                if (cert && key) {
+                    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+                }
+
+                taskmanager.restartAppTask(appId);
+
+                eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId: appId, location: location, manifest: manifest });
+
+                callback(null, { id : appId });
+            });
         });
     });
 }
 
-function configure(appId, location, portBindings, accessRestriction, cert, key, memoryLimit, altDomain, auditSource, callback) {
+function configure(appId, data, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert(cert === null || typeof cert === 'string');
-    assert(key === null || typeof key === 'string');
-    assert.strictEqual(typeof memoryLimit, 'number');
-    assert(altDomain === null || typeof altDomain === 'string');
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var error = validateHostname(location, config.fqdn());
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = validateAccessRestriction(accessRestriction);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = certificates.validateCertificate(cert, key, config.appFqdn(location));
-    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
-
-    if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
-
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        error = validatePortBindings(portBindings, app.manifest.tcpPorts);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-        error = validateMemoryLimit(app.manifest, memoryLimit);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-        // memoryLimit might come in as 0 if not specified
-        memoryLimit = memoryLimit || app.memoryLimit || app.manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
-
-        // save cert to data/box/certs
-        if (cert && key) {
-            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
-            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+        var location, portBindings, values = { };
+        if ('location' in data) {
+            location = values.location = data.location.toLowerCase();
+            error = validateHostname(values.location, config.fqdn());
+            if (error) return callback(error);
+        } else {
+            location = app.location;
         }
 
-        var values = {
-            location: location.toLowerCase(),
-            accessRestriction: accessRestriction,
-            portBindings: portBindings,
-            memoryLimit: memoryLimit,
-            altDomain: altDomain,
+        if ('accessRestriction' in data) {
+            values.accessRestriction = data.accessRestriction;
+            error = validateAccessRestriction(values.accessRestriction);
+            if (error) return callback(error);
+        }
 
-            oldConfig: {
-                location: app.location,
-                accessRestriction: app.accessRestriction,
-                portBindings: app.portBindings,
-                memoryLimit: app.memoryLimit,
-                altDomain: altDomain
+        if ('altDomain' in data) {
+            values.altDomain = data.altDomain;
+            if (values.altDomain !== null && !validator.isFQDN(values.altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
+        }
+
+        if ('portBindings' in data) {
+            portBindings = values.portBindings = data.portBindings;
+            error = validatePortBindings(values.portBindings, app.manifest.tcpPorts);
+            if (error) return callback(error);
+        } else {
+            portBindings = app.portBindings;
+        }
+
+        if ('memoryLimit' in data) {
+            values.memoryLimit = data.memoryLimit;
+            error = validateMemoryLimit(app.manifest, values.memoryLimit);
+            if (error) return callback(error);
+
+            // memoryLimit might come in as 0 if not specified
+            values.memoryLimit = values.memoryLimit || app.memoryLimit || app.manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+        }
+
+        if ('xFrameOptions' in data) {
+            values.xFrameOptions = data.xFrameOptions;
+            error = validateXFrameOptions(values.xFrameOptions);
+            if (error) return callback(error);
+        }
+
+        // save cert to data/box/certs. TODO: move this to apptask when we have a real task queue
+        if ('cert' in data && 'key' in data) {
+            if (data.cert && data.key) {
+                error = certificates.validateCertificate(data.cert, data.key, config.appFqdn(location));
+                if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
+
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), data.cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), data.key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+            } else { // remove existing cert/key
+                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'))) debug('Error removing cert: ' + safe.error.message);
+                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'))) debug('Error removing key: ' + safe.error.message);
             }
-        };
+        }
+
+        values.oldConfig = getAppConfig(app);
 
         debug('Will configure app with id:%s values:%j', appId, values);
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_CONFIGURE, values, function (error) {
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -493,76 +580,75 @@ function configure(appId, location, portBindings, accessRestriction, cert, key,
     });
 }
 
-function update(appId, force, manifest, portBindings, icon, auditSource, callback) {
+function update(appId, data, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof force, 'boolean');
-    assert(manifest && typeof manifest === 'object');
-    assert(typeof portBindings === 'object'); // can be null
-    assert(!icon || typeof icon === 'string');
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debug('Will update app with id:%s', appId);
 
-    var error = manifestFormat.parse(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error:' + error.message));
+    downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
+        if (error) return callback(error);
 
-    error = checkManifestConstraints(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed:' + error.message));
+        var values = { };
 
-    error = validatePortBindings(portBindings, manifest.tcpPorts);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+        error = manifestFormat.parse(manifest);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error:' + error.message));
 
-    if (icon) {
-        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
+        error = checkManifestConstraints(manifest);
+        if (error) return callback(error);
 
-        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
-            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
-        }
-    }
+        values.manifest = manifest;
 
-    appdb.get(appId, function (error, app) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        var appStoreId = app.appStoreId;
-
-        // prevent user from installing a app with different manifest id over an existing app
-        // this allows cloudron install -f --app <appid> for an app installed from the appStore
-        if (app.manifest.id !== manifest.id) {
-            if (!force) return callback(new AppsError(AppsError.BAD_FIELD, 'manifest id does not match. force to override'));
-            // clear appStoreId so that this app does not get updates anymore. this will mark is a dev app
-            appStoreId = '';
+        if ('portBindings' in data) {
+            values.portBindings = data.portBindings;
+            error = validatePortBindings(data.portBindings, values.manifest.tcpPorts);
+            if (error) return callback(error);
         }
 
-        // Ensure we update the memory limit in case the new app requires more memory as a minimum
-        var memoryLimit = manifest.memoryLimit ? (app.memoryLimit < manifest.memoryLimit ? manifest.memoryLimit : app.memoryLimit) : app.memoryLimit;
+        if ('icon' in data) {
+            if (data.icon) {
+                if (!validator.isBase64(data.icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
 
-        var values = {
-            appStoreId: appStoreId,
-            manifest: manifest,
-            portBindings: portBindings,
-            memoryLimit: memoryLimit,
-
-            oldConfig: {
-                manifest: app.manifest,
-                portBindings: app.portBindings,
-                accessRestriction: app.accessRestriction,
-                memoryLimit: app.memoryLimit,
-                altDomain: app.altDomain
+                if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(data.icon, 'base64'))) {
+                    return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
+                }
+            } else {
+                safe.fs.unlinkSync(path.join(paths.APPICONS_DIR, appId + '.png'));
             }
-        };
+        }
 
-        appdb.setInstallationCommand(appId, force ? appdb.ISTATE_PENDING_FORCE_UPDATE : appdb.ISTATE_PENDING_UPDATE, values, function (error) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails('' /* location cannot conflict */, portBindings, error));
+        appdb.get(appId, function (error, app) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-            taskmanager.restartAppTask(appId);
+            // prevent user from installing a app with different manifest id over an existing app
+            // this allows cloudron install -f --app <appid> for an app installed from the appStore
+            if (app.manifest.id !== values.manifest.id) {
+                if (!data.force) return callback(new AppsError(AppsError.BAD_FIELD, 'manifest id does not match. force to override'));
+                // clear appStoreId so that this app does not get updates anymore. this will mark it as a dev app
+                values.appStoreId = '';
+            }
 
-            eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId: appId, toManifest: manifest, fromManifest: app.manifest });
+            // Ensure we update the memory limit in case the new app requires more memory as a minimum
+            if (values.manifest.memoryLimit && app.memoryLimit < values.manifest.memoryLimit) {
+                values.memoryLimit = values.manifest.memoryLimit;
+            }
 
-            callback(null);
+            values.oldConfig = getAppConfig(app);
+
+            appdb.setInstallationCommand(appId, data.force ? appdb.ISTATE_PENDING_FORCE_UPDATE : appdb.ISTATE_PENDING_UPDATE, values, function (error) {
+                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails('' /* location cannot conflict */, values.portBindings, error));
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                taskmanager.restartAppTask(appId);
+
+                eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId: appId, toManifest: manifest, fromManifest: app.manifest, force: data.force });
+
+                callback(null);
+            });
         });
     });
 }
@@ -612,8 +698,9 @@ function getLogs(appId, lines, follow, callback) {
     });
 }
 
-function restore(appId, auditSource, callback) {
+function restore(appId, data, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -623,43 +710,100 @@ function restore(appId, auditSource, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        // restore without a backup is the same as re-install
-        var restoreConfig = app.lastBackupConfig, values = { };
-        if (restoreConfig) {
-            // re-validate because this new box version may not accept old configs.
-            // if we restore location, it should be validated here as well
-            error = checkManifestConstraints(restoreConfig.manifest);
-            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+        // for empty or null backupId, use existing manifest to mimic a reinstall
+        var func = data.backupId ? backups.getRestoreConfig.bind(null, data.backupId) : function (next) { return next(null, { manifest: app.manifest }); };
 
-            error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
-            if (error) return callback(error);
-
-            // ## should probably query new location, access restriction from user
-            values = {
-                manifest: restoreConfig.manifest,
-                portBindings: restoreConfig.portBindings,
-                memoryLimit: restoreConfig.memoryLimit,
-
-                oldConfig: {
-                    location: app.location,
-                    accessRestriction: app.accessRestriction,
-                    portBindings: app.portBindings,
-                    memoryLimit: app.memoryLimit,
-                    manifest: app.manifest,
-                    altDomain: app.altDomain
-                }
-            };
-        }
-
-        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+        func(function (error, restoreConfig) {
+            if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-            taskmanager.restartAppTask(appId);
+            if (!restoreConfig) callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Could not get restore config'));
 
-            eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { appId: appId });
+            // re-validate because this new box version may not accept old configs
+            error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(error);
 
-            callback(null);
+            var values = {
+                lastBackupId: data.backupId || null, // when null, apptask simply reinstalls
+                manifest: restoreConfig.manifest,
+
+                oldConfig: getAppConfig(app)
+            };
+
+            appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
+                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                taskmanager.restartAppTask(appId);
+
+                eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { appId: appId });
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function clone(appId, data, auditSource, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will clone app with id:%s', appId);
+
+    var location = data.location.toLowerCase(),
+        portBindings = data.portBindings || null,
+        backupId = data.backupId;
+
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        backups.getRestoreConfig(backupId, function (error, restoreConfig) {
+            if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            if (!restoreConfig) callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Could not get restore config'));
+
+            // re-validate because this new box version may not accept old configs
+            error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(error);
+
+            error = validateHostname(location, config.fqdn());
+            if (error) return callback(error);
+
+            error = validatePortBindings(portBindings, restoreConfig.manifest.tcpPorts);
+            if (error) return callback(error);
+
+            var newAppId = uuid.v4(), appStoreId = app.appStoreId, manifest = restoreConfig.manifest;
+
+            purchase(appStoreId, function (error) {
+                if (error) return callback(error);
+
+                var data = {
+                    installationState: appdb.ISTATE_PENDING_CLONE,
+                    memoryLimit: app.memoryLimit,
+                    accessRestriction: app.accessRestriction,
+                    xFrameOptions: app.xFrameOptions,
+                    lastBackupId: backupId
+                };
+
+                appdb.add(newAppId, appStoreId, manifest, location, portBindings, data, function (error) {
+                    if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
+                    if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                    taskmanager.restartAppTask(newAppId);
+
+                    eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, location: location, manifest: manifest });
+
+                    callback(null, { id : newAppId });
+                });
+            });
         });
     });
 }
@@ -716,14 +860,16 @@ function stop(appId, callback) {
 }
 
 function checkManifestConstraints(manifest) {
-    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+    assert(manifest && typeof manifest === 'object');
+
+    if (!manifest.dockerImage) return new AppsError(AppsError.BAD_FIELD, 'Missing dockerImage'); // dockerImage is optional in manifest
 
     if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
-        return new Error('Box version exceeds Apps maxBoxVersion');
+        return new AppsError(AppsError.BAD_FIELD, 'Box version exceeds Apps maxBoxVersion');
     }
 
     if (semver.valid(manifest.minBoxVersion) && semver.gt(manifest.minBoxVersion, config.version())) {
-        return new Error('minBoxVersion exceeds Box version');
+        return new AppsError(AppsError.BAD_FIELD, 'minBoxVersion exceeds Box version');
     }
 
     return null;
@@ -747,10 +893,14 @@ function exec(appId, options, callback) {
 
         var container = docker.connection.getContainer(app.containerId);
 
-       var execOptions = {
+        var execOptions = {
             AttachStdin: true,
             AttachStdout: true,
             AttachStderr: true,
+            // A pseudo tty is a terminal which processes can detect (for example, disable colored output)
+            // Creating a pseudo terminal also assigns a terminal driver which detects control sequences
+            // When passing binary data, tty must be disabled. In addition, the stdout/stderr becomes a single
+            // unified stream because of the nature of a tty (see https://github.com/docker/docker/issues/19696)
             Tty: options.tty,
             Cmd: cmd
         };
@@ -760,9 +910,18 @@ function exec(appId, options, callback) {
             var startOptions = {
                 Detach: false,
                 Tty: options.tty,
-                stdin: true // this is a dockerode option that enabled openStdin in the modem
+                // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
+                // we can work with half-close connections (not defined in http). this way, the client
+                // can properly signal that stdin is EOF by closing it's side of the socket. In http,
+                // the whole connection will be dropped when stdin get EOF.
+                // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
+                hijack: true,
+                stream: true,
+                stdin: true,
+                stdout: true,
+                stderr: true
             };
-            exec.start(startOptions, function(error, stream) {
+            exec.start(startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
                 if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
                 if (options.rows && options.columns) {
@@ -775,23 +934,25 @@ function exec(appId, options, callback) {
     });
 }
 
-function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { manifest } }
+function updateApps(updateInfo, auditSource, callback) { // updateInfo is { appId -> { manifest } }
     assert.strictEqual(typeof updateInfo, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     function canAutoupdateApp(app, newManifest) {
-        var tcpPorts = newManifest.tcpPorts || { };
+        var newTcpPorts = newManifest.tcpPorts || { };
+        var oldTcpPorts = app.manifest.tcpPorts || { };
         var portBindings = app.portBindings; // this is never null
 
-        if (Object.keys(tcpPorts).length === 0 && Object.keys(portBindings).length === 0) return null;
-        if (Object.keys(tcpPorts).length === 0) return new Error('tcpPorts is now empty but portBindings is not');
-        if (Object.keys(portBindings).length === 0) return new Error('portBindings is now empty but tcpPorts is not');
-
-        for (var env in tcpPorts) {
-            if (!(env in portBindings)) return new Error(env + ' is required from user');
+        for (var env in newTcpPorts) {
+            if (!(env in oldTcpPorts)) return new Error(env + ' is required from user');
         }
 
-        // it's fine if one or more keys got removed
+        for (env in portBindings) {
+            if (!(env in newTcpPorts)) return new Error(env + ' was in use but new update removes it');
+        }
+
+        // it's fine if one or more (unused) keys got removed
         return null;
     }
 
@@ -810,8 +971,12 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
                 return iteratorDone();
             }
 
-           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings,
-                  null /* icon */, { userId: null, username: 'autoupdater' }, function (error) {
+            var data = {
+                manifest: updateInfo[appId].manifest,
+                force: false
+            };
+
+            update(appId, data, auditSource, function (error) {
                 if (error) debug('Error initiating autoupdate of %s. %s', appId, error.message);
 
                 iteratorDone(null);
@@ -858,3 +1023,39 @@ function listBackups(page, perPage, appId, callback) {
         });
     });
 }
+
+function restoreInstalledApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        async.map(apps, function (app, iteratorDone) {
+            debug('marking %s for restore', app.location || app.id);
+
+            appdb.setInstallationCommand(app.id, appdb.ISTATE_PENDING_RESTORE, { oldConfig: null }, function (error) {
+                if (error) debug('did not mark %s for restore', app.location || app.id, error);
+
+                iteratorDone(); // always succeed
+            });
+        }, callback);
+    });
+}
+
+function configureInstalledApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        async.map(apps, function (app, iteratorDone) {
+            debug('marking %s for reconfigure', app.location || app.id);
+
+            appdb.setInstallationCommand(app.id, appdb.ISTATE_PENDING_CONFIGURE, { oldConfig: null }, function (error) {
+                if (error) debug('did not mark %s for reconfigure', app.location || app.id, error);
+
+                iteratorDone(); // always succeed
+            });
+        }, callback);
+    });
+}

src/apptask.js

@@ -36,15 +36,14 @@ var addons = require('./addons.js'),
     async = require('async'),
     backups = require('./backups.js'),
     certificates = require('./certificates.js'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
+    ClientsError = clients.ClientsError,
     database = require('./database.js'),
-    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apptask'),
     docker = require('./docker.js'),
     ejs = require('ejs'),
     fs = require('fs'),
-    hat = require('hat'),
     manifestFormat = require('cloudron-manifestformat'),
     net = require('net'),
     nginx = require('./nginx.js'),
@@ -57,7 +56,6 @@ var addons = require('./addons.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
-    uuid = require('node-uuid'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
@@ -163,20 +161,18 @@ function allocateOAuthProxyCredentials(app, callback) {
 
     if (!nginx.requiresOAuthProxy(app)) return callback(null);
 
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile';
 
-    clientdb.add(id, app.id, clientdb.TYPE_PROXY, clientSecret, redirectURI, scope, callback);
+    clients.add(app.id, clients.TYPE_PROXY, redirectURI, scope, callback);
 }
 
 function removeOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_PROXY, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) {
+    clients.delByAppIdAndType(app.id, clients.TYPE_PROXY, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) {
             debugApp(app, 'Error removing OAuth client id', error);
             return callback(error);
         }
@@ -335,7 +331,9 @@ function waitForDnsPropagation(app, callback) {
 function waitForAltDomainDnsPropagation(app, callback) {
     if (!app.altDomain) return callback(null);
 
-    waitForDns(app.altDomain, config.appFqdn(app.location), 'CNAME', callback); // waits forever
+    // try for 10 minutes before giving up. this allows the user to "reconfigure" the app in the case where
+    // an app has an external domain and cloudron is migrated to custom domain.
+    waitForDns(app.altDomain, config.appFqdn(app.location), 'CNAME', { interval: 10000, times: 60 }, callback);
 }
 
 // updates the app object and the database
@@ -417,7 +415,7 @@ function install(app, callback) {
         updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for External Domain CNAME setup' }),
         exports._waitForAltDomainDnsPropagation.bind(null, app), // required when restoring and !lastBackupId
 
         updateApp.bind(null, app, { installationProgress: '95, Configure nginx' }),
@@ -443,7 +441,7 @@ function backup(app, callback) {
 
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
-        backups.backupApp.bind(null, app, app.manifest.addons),
+        backups.backupApp.bind(null, app, app.manifest),
 
         // done!
         function (callback) {
@@ -521,7 +519,7 @@ function restore(app, callback) {
         updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for External Domain CNAME setup' }),
         exports._waitForAltDomainDnsPropagation.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '95, Configuring Nginx' }),
@@ -583,7 +581,7 @@ function configure(app, callback) {
         updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '85, Waiting for Alt Domain DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for External Domain CNAME setup' }),
         exports._waitForAltDomainDnsPropagation.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '90, Configuring Nginx' }),
@@ -629,7 +627,6 @@ function update(app, callback) {
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
         deleteContainers.bind(null, app),
-        addons.teardownAddons.bind(null, app, unusedAddons),
         function deleteImageIfChanged(done) {
              if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
 
@@ -641,11 +638,14 @@ function update(app, callback) {
             if (app.installationState === appdb.ISTATE_PENDING_FORCE_UPDATE) return next(null);
 
             async.series([
-                updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
-                backups.backupApp.bind(null, app, app.oldConfig.manifest.addons)
+                updateApp.bind(null, app, { installationProgress: '30, Backing up app' }),
+                backups.backupApp.bind(null, app, app.oldConfig.manifest)
             ], next);
         },
 
+        // only delete unused addons after backup
+        addons.teardownAddons.bind(null, app, unusedAddons),
+
         updateApp.bind(null, app, { installationProgress: '45, Downloading icon' }),
         downloadIcon.bind(null, app),
 
@@ -780,6 +780,7 @@ function startTask(appId, callback) {
         case appdb.ISTATE_PENDING_BACKUP: return backup(app, callback);
         case appdb.ISTATE_INSTALLED: return handleRunCommand(app, callback);
         case appdb.ISTATE_PENDING_INSTALL: return install(app, callback);
+        case appdb.ISTATE_PENDING_CLONE: return restore(app, callback);
         case appdb.ISTATE_PENDING_FORCE_UPDATE: return update(app, callback);
         case appdb.ISTATE_ERROR:
             debugApp(app, 'Internal error. apptask launched with error status.');
@@ -796,6 +797,11 @@ if (require.main === module) {
 
     debug('Apptask for %s', process.argv[2]);
 
+    process.on('SIGTERM', function () {
+        debug('taskmanager sent SIGTERM since it got a new task for this app');
+        process.exit(0);
+    });
+
     initialize(function (error) {
         if (error) throw error;
 

src/auth.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -10,17 +8,16 @@ exports = module.exports = {
 var assert = require('assert'),
     BasicStrategy = require('passport-http').BasicStrategy,
     BearerStrategy = require('passport-http-bearer').Strategy,
-    clientdb = require('./clientdb'),
+    clients = require('./clients'),
     ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    ClientsError = clients.ClientsError,
     DatabaseError = require('./databaseerror'),
     debug = require('debug')('box:auth'),
     LocalStrategy = require('passport-local').Strategy,
     crypto = require('crypto'),
-    groups = require('./groups'),
     passport = require('passport'),
     tokendb = require('./tokendb'),
     user = require('./user'),
-    userdb = require('./userdb'),
     UserError = user.UserError,
     _ = require('underscore');
 
@@ -32,7 +29,7 @@ function initialize(callback) {
     });
 
     passport.deserializeUser(function(userId, callback) {
-        userdb.get(userId, function (error, result) {
+        user.get(userId, function (error, result) {
             if (error) return callback(error);
 
             var md5 = crypto.createHash('md5').update(result.email.toLowerCase()).digest('hex');
@@ -67,8 +64,8 @@ function initialize(callback) {
             debug('BasicStrategy: detected client id %s instead of username:password', username);
             // username is actually client id here
             // password is client secret
-            clientdb.get(username, function (error, client) {
-                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+            clients.get(username, function (error, client) {
+                if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
                 if (client.clientSecret != password) return callback(null, false);
                 return callback(null, client);
@@ -85,8 +82,8 @@ function initialize(callback) {
     }));
 
     passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clientdb.get(clientId, function(error, client) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+        clients.get(clientId, function(error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
             if (error) { return callback(error); }
             if (client.clientSecret != clientSecret) { return callback(null, false); }
             return callback(null, client);
@@ -101,37 +98,12 @@ function initialize(callback) {
             // scopes here can define what capabilities that token carries
             // passport put the 'info' object into req.authInfo, where we can further validate the scopes
             var info = { scope: token.scope };
-            var tokenType;
 
-            if (token.identifier.indexOf(tokendb.PREFIX_DEV) === 0) {
-                token.identifier = token.identifier.slice(tokendb.PREFIX_DEV.length);
-                tokenType = tokendb.TYPE_DEV;
-            } else if (token.identifier.indexOf(tokendb.PREFIX_APP) === 0) {
-                tokenType = tokendb.TYPE_APP;
-                return callback(null, { id: token.identifier.slice(tokendb.PREFIX_APP.length), tokenType: tokenType }, info);
-            } else if (token.identifier.indexOf(tokendb.PREFIX_USER) === 0) {
-                tokenType = tokendb.TYPE_USER;
-                token.identifier = token.identifier.slice(tokendb.PREFIX_USER.length);
-            } else {
-                // legacy tokens assuming a user access token
-                tokenType = tokendb.TYPE_USER;
-            }
-
-            userdb.get(token.identifier, function (error, user) {
+            user.get(token.identifier, function (error, user) {
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
 
-                // amend the tokenType of the token owner
-                user.tokenType = tokenType;
-
-                // amend the admin flag
-                groups.isMember(groups.ADMIN_GROUP_ID, user.id, function (error, isAdmin) {
-                    if (error) return callback(error);
-
-                    user.admin = isAdmin;
-
-                    callback(null, user, info);
-                });
+                callback(null, user, info);
             });
         });
     }));
@@ -144,4 +116,3 @@ function uninitialize(callback) {
 
     callback(null);
 }
-

src/backups.js

@@ -7,6 +7,7 @@ exports = module.exports = {
     getByAppIdPaged: getByAppIdPaged,
 
     getRestoreUrl: getRestoreUrl,
+    getRestoreConfig: getRestoreConfig,
 
     ensureBackup: ensureBackup,
 
@@ -36,6 +37,7 @@ var addons = require('./addons.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
     settings = require('./settings.js'),
+    superagent = require('superagent'),
     util = require('util'),
     webhooks = require('./webhooks.js');
 
@@ -135,12 +137,13 @@ function getBoxBackupCredentials(appBackupIds, callback) {
     });
 }
 
-function getAppBackupCredentials(app, callback) {
+function getAppBackupCredentials(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var now = new Date();
-    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), app.manifest.version);
+    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), manifest.version);
     var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';
 
     settings.getBackupConfig(function (error, backupConfig) {
@@ -161,6 +164,32 @@ function getAppBackupCredentials(app, callback) {
     });
 }
 
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
+function getRestoreConfig(backupId, callback) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var configFile = backupId.replace(/\.tar\.gz$/, '.json');
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).getRestoreUrl(backupConfig, configFile, function (error, result) {
+            if (error) return callback(error);
+
+            superagent.get(result.url).buffer(true).end(function (error, response) {
+                if (error && !error.response) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+                if (response.statusCode !== 200) return callback(new Error('Invalid response code when getting config.json : ' + response.statusCode));
+
+                var config = safe.JSON.parse(response.text);
+                if (!config) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error in config:' + safe.error.message));
+
+                return callback(null, config);
+            });
+        });
+    });
+}
+
 // backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
@@ -185,14 +214,15 @@ function getRestoreUrl(backupId, callback) {
     });
 }
 
-function copyLastBackup(app, callback) {
+function copyLastBackup(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var now = new Date();
-    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, now.toISOString(), app.manifest.version);
-    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, now.toISOString(), app.manifest.version);
+    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, now.toISOString(), manifest.version);
+    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, now.toISOString(), manifest.version);
 
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
@@ -225,7 +255,8 @@ function backupBoxWithAppBackupIds(appBackupIds, callback) {
 
         debug('backupBoxWithAppBackupIds:  %j', result);
 
-        var args = [ result.s3Url, result.accessKeyId, result.secretAccessKey, result.sessionToken, result.region, result.backupKey ];
+        var args = [ result.s3Url, result.accessKeyId, result.secretAccessKey, result.region, result.backupKey ];
+        if (result.sessionToken) args.push(result.sessionToken);
 
         shell.sudo('backupBox', [ BACKUP_BOX_CMD ].concat(args), function (error) {
             if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
@@ -267,11 +298,12 @@ function canBackupApp(app) {
 
 // set the 'creation' date of lastBackup so that the backup persists across time based archival rules
 // s3 does not allow changing creation time, so copying the last backup is easy way out for now
-function reuseOldAppBackup(app, callback) {
+function reuseOldAppBackup(app, manifest, callback) {
     assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    copyLastBackup(app, function (error, newBackupId) {
+    copyLastBackup(app, manifest, function (error, newBackupId) {
         if (error) return callback(error);
 
         debugApp(app, 'reuseOldAppBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
@@ -280,28 +312,28 @@ function reuseOldAppBackup(app, callback) {
     });
 }
 
-function createNewAppBackup(app, addonsToBackup, callback) {
+function createNewAppBackup(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    getAppBackupCredentials(app, function (error, result) {
+    getAppBackupCredentials(app, manifest, function (error, result) {
         if (error) return callback(error);
 
         debugApp(app, 'createNewAppBackup: backup url:%s backup config url:%s', result.s3DataUrl, result.s3ConfigUrl);
 
-        var args = [ app.id, result.s3ConfigUrl, result.s3DataUrl, result.accessKeyId, result.secretAccessKey,
-                     result.sessionToken, result.region, result.backupKey ];
+        var args = [ app.id, result.s3ConfigUrl, result.s3DataUrl, result.accessKeyId, result.secretAccessKey, result.region, result.backupKey ];
+        if (result.sessionToken) args.push(result.sessionToken);
 
         async.series([
-            addons.backupAddons.bind(null, app, addonsToBackup),
+            addons.backupAddons.bind(null, app, manifest.addons),
             shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD ].concat(args))
         ], function (error) {
             if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
             debugApp(app, 'createNewAppBackup: %s done', result.id);
 
-            backupdb.add({ id: result.id, version: app.manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+            backupdb.add({ id: result.id, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
                 if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
                 callback(null, result.id);
@@ -310,13 +342,12 @@ function createNewAppBackup(app, addonsToBackup, callback) {
     });
 }
 
-function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
+function setRestorePoint(appId, lastBackupId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof lastBackupId, 'string');
-    assert.strictEqual(typeof lastBackupConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    appdb.update(appId, { lastBackupId: lastBackupId, lastBackupConfig: lastBackupConfig }, function (error) {
+    appdb.update(appId, { lastBackupId: lastBackupId }, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
@@ -324,12 +355,12 @@ function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
     });
 }
 
-function backupApp(app, addonsToBackup, callback) {
+function backupApp(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var appConfig = null, backupFunction;
+    var backupFunction;
 
     if (!canBackupApp(app)) {
         if (!app.lastBackupId) {
@@ -337,17 +368,11 @@ function backupApp(app, addonsToBackup, callback) {
             return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy and never backed up previously'));
         }
 
-        appConfig = app.lastBackupConfig;
-        backupFunction = reuseOldAppBackup.bind(null, app);
+        backupFunction = reuseOldAppBackup.bind(null, app, manifest);
     } else {
-        appConfig = {
-            manifest: app.manifest,
-            location: app.location,
-            portBindings: app.portBindings,
-            accessRestriction: app.accessRestriction,
-            memoryLimit: app.memoryLimit
-        };
-        backupFunction = createNewAppBackup.bind(null, app, addonsToBackup);
+        var appConfig = apps.getAppConfig(app);
+        appConfig.manifest = manifest;
+        backupFunction = createNewAppBackup.bind(null, app, manifest);
 
         if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
             return callback(safe.error);
@@ -359,7 +384,7 @@ function backupApp(app, addonsToBackup, callback) {
 
         debugApp(app, 'backupApp: successful id:%s', backupId);
 
-        setRestorePoint(app.id, backupId, appConfig, function (error) {
+        setRestorePoint(app.id, backupId, function (error) {
             if (error) return callback(error);
 
             return callback(null, backupId);
@@ -386,7 +411,7 @@ function backupBoxAndApps(auditSource, callback) {
         async.mapSeries(allApps, function iterator(app, iteratorCallback) {
             ++processed;
 
-            backupApp(app, app.manifest.addons, function (error, backupId) {
+            backupApp(app, app.manifest, function (error, backupId) {
                 if (error && error.reason !== BackupsError.BAD_STATE) {
                     debugApp(app, 'Unable to backup', error);
                     return iteratorCallback(error);
@@ -425,7 +450,7 @@ function backup(auditSource, callback) {
     progress.set(progress.BACKUP, 0, 'Starting'); // ensure tools can 'wait' on progress
 
     backupBoxAndApps(auditSource, function (error) { // start the backup operation in the background
-        if (error) console.error('backup failed.', error);
+        if (error) debug('backup failed.', error);
 
         locker.unlock(locker.OP_FULL_BACKUP);
     });
@@ -433,8 +458,8 @@ function backup(auditSource, callback) {
     callback(null);
 }
 
-function ensureBackup(callback) {
-    callback = callback || NOOP_CALLBACK;
+function ensureBackup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
 
     getPaged(1, 1, function (error, backups) {
         if (error) {
@@ -447,8 +472,7 @@ function ensureBackup(callback) {
             return callback(null);
         }
 
-        var eventSource = { userId: null, username: 'cron' };
-        backup(eventSource, callback);
+        backup(auditSource, callback);
     });
 }
 

src/cert/acme.js

@@ -19,7 +19,10 @@ var CA_PROD = 'https://acme-v01.api.letsencrypt.org',
     LE_AGREEMENT = 'https://letsencrypt.org/documents/LE-SA-v1.0.1-July-27-2015.pdf';
 
 exports = module.exports = {
-    getCertificate: getCertificate
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'acme'
 };
 
 function AcmeError(reason, errorOrMessage) {

src/cert/caas.js

@@ -1,7 +1,10 @@
 'use strict';
 
 exports = module.exports = {
-    getCertificate: getCertificate
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'caas'
 };
 
 var assert = require('assert'),

src/certificates.js

@@ -2,13 +2,16 @@
 
 exports = module.exports = {
     installAdminCertificate: installAdminCertificate,
-    autoRenew: autoRenew,
+    renewAll: renewAll,
     setFallbackCertificate: setFallbackCertificate,
     setAdminCertificate: setAdminCertificate,
     CertificatesError: CertificatesError,
     validateCertificate: validateCertificate,
     ensureCertificate: ensureCertificate,
-    getAdminCertificatePath: getAdminCertificatePath
+    getAdminCertificatePath: getAdminCertificatePath,
+
+    // exported for testing
+    _getApi: getApi
 };
 
 var acme = require('./cert/acme.js'),
@@ -16,7 +19,6 @@ var acme = require('./cert/acme.js'),
     assert = require('assert'),
     async = require('async'),
     caas = require('./cert/caas.js'),
-    cloudron = require('./cloudron.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:src/certificates'),
@@ -34,8 +36,6 @@ var acme = require('./cert/acme.js'),
     waitForDns = require('./waitfordns.js'),
     x509 = require('x509');
 
-var NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 function CertificatesError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -66,17 +66,22 @@ function getApi(app, callback) {
     settings.getTlsConfig(function (error, tlsConfig) {
         if (error) return callback(error);
 
-        var api = !app.altDomain && tlsConfig.provider === 'caas' ? caas : acme;
+        // use acme if we have altDomain or the tlsConfig is not caas
+        var api = (app.altDomain || tlsConfig.provider) !== 'caas' ? acme : caas;
 
         var options = { };
-        options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
+        if (tlsConfig.provider === 'caas') {
+            options.prod = !config.isDev(); // with altDomain, we will choose acme setting based on this
+        } else { // acme
+            options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
+        }
 
         // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
         // we cannot use admin@fqdn because the user might not have set it up.
         // we simply update the account with the latest email we have each time when getting letsencrypt certs
         // https://github.com/ietf-wg-acme/acme/issues/30
         user.getOwner(function (error, owner) {
-            options.email = error ? 'admin@cloudron.io' : owner.email; // can error if not activated yet
+            options.email = error ? 'support@cloudron.io' : owner.email; // can error if not activated yet
 
             callback(null, api, options);
         });
@@ -84,8 +89,6 @@ function getApi(app, callback) {
 }
 
 function installAdminCertificate(callback) {
-    if (cloudron.isConfiguredSync()) return callback();
-
     settings.getTlsConfig(function (error, tlsConfig) {
         if (error) return callback(error);
 
@@ -94,8 +97,8 @@ function installAdminCertificate(callback) {
         sysinfo.getIp(function (error, ip) {
             if (error) return callback(error);
 
-            waitForDns(config.adminFqdn(), ip, 'A', function (error) {
-                if (error) return callback(error); // this cannot happen because we retry forever
+            waitForDns(config.adminFqdn(), ip, 'A', { interval: 30000, times: 50000 }, function (error) {
+                if (error) return callback(error);
 
                 ensureCertificate({ location: constants.ADMIN_LOCATION }, function (error, certFilePath, keyFilePath) {
                     if (error) { // currently, this can never happen
@@ -123,9 +126,11 @@ function isExpiringSync(certFilePath, hours) {
     return result.status === 1; // 1 - expired 0 - not expired
 }
 
-function autoRenew(callback) {
-    debug('autoRenew: Checking certificates for renewal');
-    callback = callback || NOOP_CALLBACK;
+function renewAll(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('renewAll: Checking certificates for renewal');
 
     apps.getAll(function (error, allApps) {
         if (error) return callback(error);
@@ -139,7 +144,7 @@ function autoRenew(callback) {
             var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
 
             if (!safe.fs.existsSync(keyFilePath)) {
-                debug('autoRenew: no existing key file for %s. skipping', appDomain);
+                debug('renewAll: no existing key file for %s. skipping', appDomain);
                 continue;
             }
 
@@ -148,7 +153,7 @@ function autoRenew(callback) {
             }
         }
 
-        debug('autoRenew: %j needs to be renewed', expiringApps.map(function (a) { return a.altDomain || config.appFqdn(a.location); }));
+        debug('renewAll: %j needs to be renewed', expiringApps.map(function (a) { return a.altDomain || config.appFqdn(a.location); }));
 
         async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
             var domain = app.altDomain || config.appFqdn(app.location);
@@ -156,28 +161,29 @@ function autoRenew(callback) {
             getApi(app, function (error, api, apiOptions) {
                 if (error) return callback(error);
 
-                debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
+                debug('renewAll: renewing cert for %s with options %j', domain, apiOptions);
 
                 api.getCertificate(domain, apiOptions, function (error) {
                     var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
                     var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
 
                     var errorMessage = error ? error.message : '';
-                    eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, { userId: null, username: 'cron' }, { domain: domain, errorMessage: errorMessage });
-                    mailer.certificateRenewed(domain, errorMessage);
+                    eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, auditSource, { domain: domain, errorMessage: errorMessage });
 
                     if (error) {
-                        debug('autoRenew: could not renew cert for %s because %s', domain, error);
+                        debug('renewAll: could not renew cert for %s because %s', domain, error);
+
+                        mailer.certificateRenewalError(domain, errorMessage);
 
                         // check if we should fallback if we expire in the coming day
                         if (!isExpiringSync(certFilePath, 24 * 1)) return iteratorCallback();
 
-                        debug('autoRenew: using fallback certs for %s since it expires soon', domain, error);
+                        debug('renewAll: using fallback certs for %s since it expires soon', domain, error);
 
                         certFilePath = 'cert/host.cert';
                         keyFilePath = 'cert/host.key';
                     } else {
-                        debug('autoRenew: certificate for %s renewed', domain);
+                        debug('renewAll: certificate for %s renewed', domain);
                     }
 
                     // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback

src/clientdb.js

@@ -5,6 +5,7 @@
 exports = module.exports = {
     get: get,
     getAll: getAll,
+    getAllWithTokenCount: getAllWithTokenCount,
     getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
     add: add,
     del: del,
@@ -14,13 +15,7 @@ exports = module.exports = {
     delByAppId: delByAppId,
     delByAppIdAndType: delByAppIdAndType,
 
-    _clear: clear,
-
-    TYPE_EXTERNAL: 'external',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
-    TYPE_PROXY: 'addon-proxy',
-    TYPE_ADMIN: 'admin'
+    _clear: clear
 };
 
 var assert = require('assert'),
@@ -52,14 +47,24 @@ function getAll(callback) {
     });
 }
 
+function getAllWithTokenCount(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
 function getAllWithTokenCountByIdentifier(identifier, callback) {
     assert.strictEqual(typeof identifier, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        callback(null, results);
 
+        callback(null, results);
     });
 }
 
@@ -71,7 +76,7 @@ function getByAppId(appId, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result[0]);
+        callback(null, result[0]);
     });
 }
 
@@ -84,7 +89,7 @@ function getByAppIdAndType(appId, type, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result[0]);
+        callback(null, result[0]);
     });
 }
 
@@ -127,7 +132,7 @@ function delByAppId(appId, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null);
+        callback(null);
     });
 }
 
@@ -140,17 +145,17 @@ function delByAppIdAndType(appId, type, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null);
+        callback(null);
     });
 }
 
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
+    database.query('DELETE FROM clients WHERE id!="cid-webadmin" AND id!="cid-sdk" AND id!="cid-cli"', function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        return callback(null);
+        callback(null);
     });
 }
 

src/clients.js

@@ -6,16 +6,32 @@ exports = module.exports = {
     add: add,
     get: get,
     del: del,
-    getAllWithDetailsByUserId: getAllWithDetailsByUserId,
+    getAll: getAll,
+    getByAppIdAndType: getByAppIdAndType,
     getClientTokensByUserId: getClientTokensByUserId,
     delClientTokensByUserId: delClientTokensByUserId,
+    delByAppIdAndType: delByAppIdAndType,
+    addClientTokenByUserId: addClientTokenByUserId,
+    delToken: delToken,
 
+    // keep this in sync with start.sh ADMIN_SCOPES that generates the cid-webadmin
     SCOPE_APPS: 'apps',
     SCOPE_DEVELOPER: 'developer',
     SCOPE_PROFILE: 'profile',
-    SCOPE_ROOT: 'root',
+    SCOPE_CLOUDRON: 'cloudron',
     SCOPE_SETTINGS: 'settings',
-    SCOPE_USERS: 'users'
+    SCOPE_USERS: 'users',
+
+    // roles are handled just like the above scopes, they are parallel to scopes
+    // scopes enclose API groups, roles specify the usage role
+    SCOPE_ROLE_SDK: 'roleSdk',
+
+    // client type enums
+    TYPE_EXTERNAL: 'external',
+    TYPE_BUILT_IN: 'built-in',
+    TYPE_OAUTH: 'addon-oauth',
+    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
+    TYPE_PROXY: 'addon-proxy'
 };
 
 var assert = require('assert'),
@@ -23,7 +39,6 @@ var assert = require('assert'),
     hat = require('hat'),
     appdb = require('./appdb.js'),
     tokendb = require('./tokendb.js'),
-    constants = require('./constants.js'),
     async = require('async'),
     clientdb = require('./clientdb.js'),
     DatabaseError = require('./databaseerror.js'),
@@ -50,6 +65,22 @@ function ClientsError(reason, errorOrMessage) {
 util.inherits(ClientsError, Error);
 ClientsError.INVALID_SCOPE = 'Invalid scope';
 ClientsError.INVALID_CLIENT = 'Invalid client';
+ClientsError.INVALID_TOKEN = 'Invalid token';
+ClientsError.BAD_FIELD = 'Bad field';
+ClientsError.NOT_FOUND = 'Not found';
+ClientsError.INTERNAL_ERROR = 'Internal Error';
+ClientsError.NOT_ALLOWED = 'Not allowed to remove this client';
+
+function validateName(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (name.length < 1) return new ClientsError(ClientsError.BAD_FIELD, 'Name must be atleast 1 character');
+    if (name.length > 128) return new ClientsError(ClientsError.BAD_FIELD, 'Name too long');
+
+    if (/[^a-zA-Z0-9\-]/.test(name)) return new ClientsError(ClientsError.BAD_FIELD, 'Username can only contain alphanumerals and dash');
+
+    return null;
+}
 
 function validateScope(scope) {
     assert.strictEqual(typeof scope, 'string');
@@ -58,16 +89,17 @@ function validateScope(scope) {
         exports.SCOPE_APPS,
         exports.SCOPE_DEVELOPER,
         exports.SCOPE_PROFILE,
-        exports.SCOPE_ROOT,
+        exports.SCOPE_CLOUDRON,
         exports.SCOPE_SETTINGS,
-        exports.SCOPE_USERS
+        exports.SCOPE_USERS,
+        '*',    // includes all scopes, but not roles
+        exports.SCOPE_ROLE_SDK
     ];
 
-    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
-    if (scope === '*') return null;
+    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE, 'Empty scope not allowed');
 
     var allValid = scope.split(',').every(function (s) { return VALID_SCOPES.indexOf(s) !== -1; });
-    if (!allValid) return new ClientsError(ClientsError.INVALID_SCOPE);
+    if (!allValid) return new ClientsError(ClientsError.INVALID_SCOPE, 'Invalid scope. Available scopes are ' + VALID_SCOPES.join(', '));
 
     return null;
 }
@@ -79,11 +111,18 @@ function add(appId, type, redirectURI, scope, callback) {
     assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    // allow whitespace
+    scope = scope.split(',').map(function (s) { return s.trim(); }).join(',');
+
     var error = validateScope(scope);
     if (error) return callback(error);
 
+    // appId is also client name
+    error = validateName(appId);
+    if (error) return callback(error);
+
     var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
+    var clientSecret = hat(8 * 128);
 
     clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
         if (error) return callback(error);
@@ -106,6 +145,7 @@ function get(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clientdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
         if (error) return callback(error);
         callback(null, result);
     });
@@ -116,24 +156,24 @@ function del(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clientdb.del(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
         if (error) return callback(error);
         callback(null, result);
     });
 }
 
-function getAllWithDetailsByUserId(userId, callback) {
-    assert.strictEqual(typeof userId, 'string');
+function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.getAllWithTokenCountByIdentifier(tokendb.PREFIX_USER + userId, function (error, results) {
+    clientdb.getAll(function (error, results) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
         if (error) return callback(error);
 
         var tmp = [];
         async.each(results, function (record, callback) {
-            if (record.type === clientdb.TYPE_ADMIN) {
-                record.name = constants.ADMIN_NAME;
-                record.location = constants.ADMIN_LOCATION;
+            if (record.type === exports.TYPE_EXTERNAL || record.type === exports.TYPE_BUILT_IN) {
+                // the appId in this case holds the name
+                record.name = record.appId;
 
                 tmp.push(record);
 
@@ -142,14 +182,13 @@ function getAllWithDetailsByUserId(userId, callback) {
 
             appdb.get(record.appId, function (error, result) {
                 if (error) {
-                    console.error('Failed to get app details for oauth client', result, error);
+                    console.error('Failed to get app details for oauth client', record.appId, error);
                     return callback(null);  // ignore error so we continue listing clients
                 }
 
-                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
-                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
+                if (record.type === exports.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
+                if (record.type === exports.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
+                if (record.type === exports.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
 
                 record.location = result.location;
 
@@ -164,15 +203,27 @@ function getAllWithDetailsByUserId(userId, callback) {
     });
 }
 
+function getByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.getByAppIdAndType(appId, type, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
 function getClientTokensByUserId(clientId, userId, callback) {
     assert.strictEqual(typeof clientId, 'string');
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    tokendb.getByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error, result) {
+    tokendb.getByIdentifierAndClientId(userId, clientId, function (error, result) {
         if (error && error.reason === DatabaseError.NOT_FOUND) {
             // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                 if (error) return callback(error);
                 callback(null, []);
             });
@@ -188,10 +239,10 @@ function delClientTokensByUserId(clientId, userId, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    tokendb.delByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error) {
+    tokendb.delByIdentifierAndClientId(userId, clientId, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) {
             // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                 if (error) return callback(error);
                 callback(null);
             });
@@ -201,3 +252,57 @@ function delClientTokensByUserId(clientId, userId, callback) {
         callback(null);
     });
 }
+
+function delByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.delByAppIdAndType(appId, type, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null);
+    });
+}
+
+function addClientTokenByUserId(clientId, userId, expiresAt, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof expiresAt, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        var token = tokendb.generateToken();
+
+        tokendb.add(token, userId, result.id, expiresAt, result.scope, function (error) {
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null, {
+                accessToken: token,
+                identifier: userId,
+                clientId: result.id,
+                scope: result.id,
+                expires: expiresAt
+            });
+        });
+    });
+}
+
+function delToken(clientId, tokenId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof tokenId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        tokendb.del(tokenId, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.INVALID_TOKEN, 'Invalid token'));
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null);
+        });
+    });
+}

src/cloudron.js

@@ -12,9 +12,9 @@ exports = module.exports = {
     sendHeartbeat: sendHeartbeat,
 
     updateToLatest: updateToLatest,
-    update: update,
     reboot: reboot,
     retire: retire,
+    migrate: migrate,
 
     isConfiguredSync: isConfiguredSync,
 
@@ -31,8 +31,9 @@ var apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:cloudron'),
     df = require('node-df'),
     eventlog = require('./eventlog.js'),
@@ -45,6 +46,7 @@ var apps = require('./apps.js'),
     progress = require('./progress.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
+    SettingsError = settings.SettingsError,
     shell = require('./shell.js'),
     subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
@@ -53,9 +55,9 @@ var apps = require('./apps.js'),
     updateChecker = require('./updatechecker.js'),
     user = require('./user.js'),
     UserError = user.UserError,
-    userdb = require('./userdb.js'),
+    user = require('./user.js'),
     util = require('util'),
-    uuid = require('node-uuid');
+    _ = require('underscore');
 
 var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
     INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
@@ -63,9 +65,21 @@ var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
 
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
+// result to not depend on the appstore
+const BOX_AND_USER_TEMPLATE = {
+    box: {
+        region: null,
+        size: null,
+        plan: 'Custom Plan'
+    },
+    user: {
+        billing: false,
+        currency: ''
+    }
+};
+
 var gUpdatingDns = false,                // flag for dns update reentrancy
-    gCloudronDetails = null,             // cached cloudron details like region,size...
-    gAppstoreUserDetails = {},
+    gBoxAndUserDetails = null,         // cached cloudron details like region,size...
     gIsConfigured = null;                // cached configured state so that return value is synchronous. null means we are not initialized yet
 
 function CloudronError(reason, errorOrMessage) {
@@ -91,13 +105,10 @@ CloudronError.BAD_FIELD = 'Field error';
 CloudronError.INTERNAL_ERROR = 'Internal Error';
 CloudronError.EXTERNAL_ERROR = 'External Error';
 CloudronError.ALREADY_PROVISIONED = 'Already Provisioned';
-CloudronError.BAD_USERNAME = 'Bad username';
-CloudronError.BAD_EMAIL = 'Bad email';
-CloudronError.BAD_PASSWORD = 'Bad password';
-CloudronError.BAD_NAME = 'Bad name';
 CloudronError.BAD_STATE = 'Bad state';
 CloudronError.ALREADY_UPTODATE = 'No Update Available';
 CloudronError.NOT_FOUND = 'Not found';
+CloudronError.SELF_UPGRADE_NOT_SUPPORTED = 'Self upgrade not supported';
 
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
@@ -231,19 +242,17 @@ function activate(username, password, email, displayName, ip, auditSource, callb
 
     user.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
         if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
-        if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
-        if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
-        if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
+        if (error && error.reason === UserError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_FIELD, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
+        clients.get('cid-webadmin', function (error, result) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             // Also generate a token so the admin creation can also act as a login
             var token = tokendb.generateToken();
             var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-            tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
+            tokendb.add(token, userObject.id, result.id, expires, '*', function (error) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
                 // EE API is sync. do not keep the REST API reponse waiting
@@ -260,7 +269,7 @@ function activate(username, password, email, displayName, ip, auditSource, callb
 function getStatus(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    userdb.count(function (error, count) {
+    user.count(function (error, count) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
         settings.getCloudronName(function (error, cloudronName) {
@@ -278,47 +287,34 @@ function getStatus(callback) {
     });
 }
 
-function getCloudronDetails(callback) {
+function getBoxAndUserDetails(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (gCloudronDetails) return callback(null, gCloudronDetails);
+    if (gBoxAndUserDetails) return callback(null, gBoxAndUserDetails);
 
-    if (!config.token()) {
-        gCloudronDetails = {
-            region: null,
-            size: null
-        };
-
-        return callback(null, gCloudronDetails);
-    }
+    // only supported for caas
+    if (config.provider() !== 'caas') return callback(null, {});
 
     superagent
         .get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn())
         .query({ token: config.token() })
         .end(function (error, result) {
-            if (error && !error.response) return callback(error);
-            if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            if (error && !error.response) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, 'Cannot reach appstore'));
+            if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
 
-            gCloudronDetails = result.body.box;
-            gAppstoreUserDetails = result.body.user;
+            gBoxAndUserDetails = result.body;
 
-            return callback(null, gCloudronDetails);
+            return callback(null, gBoxAndUserDetails);
         });
 }
 
 function getConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    getCloudronDetails(function (error, result) {
-        if (error) {
-            debug('Failed to fetch cloudron details.', error);
+    getBoxAndUserDetails(function (error, result) {
+        if (error) debug('Failed to fetch cloudron details.', error.reason, error.message);
 
-            // set fallback values to avoid dependency on appstore
-            result = {
-                region: result ? result.region : null,
-                size: result ? result.size : null
-            };
-        }
+        result = _.extend(BOX_AND_USER_TEMPLATE, result || {});
 
         settings.getCloudronName(function (error, cloudronName) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
@@ -340,9 +336,11 @@ function getConfig(callback) {
                         progress: progress.get(),
                         isCustomDomain: config.isCustomDomain(),
                         developerMode: developerMode,
-                        region: result.region,
-                        size: result.size,
-                        billing: !!gAppstoreUserDetails.billing,
+                        region: result.box.region,
+                        size: result.box.size,
+                        billing: !!result.user.billing,
+                        plan: result.box.plan,
+                        currency: result.user.currency,
                         memory: os.totalmem(),
                         provider: config.provider(),
                         cloudronName: cloudronName
@@ -444,24 +442,28 @@ function addDnsRecords() {
     sysinfo.getIp(function (error, ip) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        var webadminRecord = { subdomain: 'my', type: 'A', values: [ ip ] };
+        var webadminRecord = { subdomain: constants.ADMIN_LOCATION, type: 'A', values: [ ip ] };
         // t=s limits the domainkey to this domain and not it's subdomains
         var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
         // DMARC requires special setup if report email id is in different domain
         var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };
 
+        var mxRecord = { subdomain: '', type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] };
+
         var records = [ ];
         if (config.isCustomDomain()) {
             records.push(webadminRecord);
             records.push(dkimRecord);
+            records.push(mxRecord);
         } else {
-            // for custom domains, we show a nakeddomain.html page
+            // for non-custom domains, we show a nakeddomain.html page
             var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
 
             records.push(nakedDomainRecord);
             records.push(webadminRecord);
             records.push(dkimRecord);
             records.push(dmarcRecord);
+            records.push(mxRecord);
         }
 
         debug('addDnsRecords: %j', records);
@@ -497,8 +499,9 @@ function reboot(callback) {
     shell.sudo('reboot', [ REBOOT_CMD ], callback);
 }
 
-function update(boxUpdateInfo, callback) {
+function update(boxUpdateInfo, auditSource, callback) {
     assert.strictEqual(typeof boxUpdateInfo, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     if (!boxUpdateInfo) return callback(null);
@@ -506,6 +509,8 @@ function update(boxUpdateInfo, callback) {
     var error = locker.lock(locker.OP_BOX_UPDATE);
     if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
 
+    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
+
     // ensure tools can 'wait' on progress
     progress.set(progress.UPDATE, 0, 'Starting');
 
@@ -544,9 +549,9 @@ function updateToLatest(auditSource, callback) {
     var boxUpdateInfo = updateChecker.getUpdateInfo().box;
     if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
 
-    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
+    if (boxUpdateInfo.upgrade && config.provider() !== 'caas') return callback(new CloudronError(CloudronError.SELF_UPGRADE_NOT_SUPPORTED));
 
-    update(boxUpdateInfo, callback);
+    update(boxUpdateInfo, auditSource, callback);
 }
 
 function doShortCircuitUpdate(boxUpdateInfo, callback) {
@@ -583,7 +588,7 @@ function doUpgrade(boxUpdateInfo, callback) {
 
             // no need to unlock since this is the last thing we ever do on this box
             callback();
-            retire();
+            retire('upgrade');
         });
     });
 }
@@ -607,6 +612,7 @@ function doUpdate(boxUpdateInfo, callback) {
 
             // this data is opaque to the installer
             data: {
+                provider: config.provider(),
                 token: config.token(),
                 apiServerOrigin: config.apiServerOrigin(),
                 webServerOrigin: config.webServerOrigin(),
@@ -656,23 +662,16 @@ function installAppBundle(callback) {
     }
 
     async.eachSeries(bundle, function (appInfo, iteratorCallback) {
-        var appstoreId = appInfo.appstoreId;
-        var parts = appstoreId.split('@');
+        debug('autoInstall: installing %s at %s', appInfo.appstoreId, appInfo.location);
 
-        var url = config.apiServerOrigin() + '/api/v1/apps/' + parts[0] + (parts[1] ? '/versions/' + parts[1] : '');
+        var data = {
+            appStoreId: appInfo.appstoreId,
+            location: appInfo.location,
+            portBindings: appInfo.portBindings || null,
+            accessRestriction: appInfo.accessRestriction || null,
+        };
 
-        superagent.get(url).end(function (error, result) {
-            if (error && !error.response) return iteratorCallback(new Error('Network error: ' + error.message));
-
-            if (result.statusCode !== 200) return iteratorCallback(util.format('Failed to get app info from store.', result.statusCode, result.text));
-
-            debug('autoInstall: installing %s at %s', appstoreId, appInfo.location);
-
-            apps.install(uuid.v4(), appstoreId, result.body.manifest, appInfo.location,
-                appInfo.portBindings || null, appInfo.accessRestriction || null,
-                null /* icon */, null /* cert */, null /* key */, 0 /* default mem limit */,
-                null /* altDomain */, { userId: null, username: 'autoinstaller' }, iteratorCallback);
-        });
+        apps.install(data, { userId: null, username: 'autoinstaller' }, iteratorCallback);
     }, function (error) {
         if (error) debug('autoInstallApps: ', error);
 
@@ -694,7 +693,7 @@ function checkDiskSpace(callback) {
 
         var oos = entries.some(function (entry) {
             return (entry.mount === paths.DATA_DIR && entry.capacity >= 0.90) ||
-                   (entry.mount === '/' && entry.used <= (1.25 * 1024 * 1024)); // 1.5G
+                   (entry.mount === '/' && entry.available <= (1.25 * 1024 * 1024)); // 1.5G
         });
 
         debug('Disk space checked. ok: %s', !oos);
@@ -705,13 +704,73 @@ function checkDiskSpace(callback) {
     });
 }
 
-function retire(callback) {
+function retire(reason, info, callback) {
+    assert(reason === 'migrate' || reason === 'upgrade');
+    info = info || { };
     callback = callback || NOOP_CALLBACK;
 
     var data = {
+        apiServerOrigin: config.apiServerOrigin(),
         isCustomDomain: config.isCustomDomain(),
         fqdn: config.fqdn()
     };
-    shell.sudo('retire', [ RETIRE_CMD, JSON.stringify(data) ], callback);
+    shell.sudo('retire', [ RETIRE_CMD, reason, JSON.stringify(info), JSON.stringify(data) ], callback);
 }
 
+function doMigrate(options, callback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_MIGRATE);
+    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
+
+    function unlock(error) {
+        debug('Failed to migrate', error);
+        locker.unlock(locker.OP_MIGRATE);
+        progress.set(progress.MIGRATE, -1, 'Backup failed: ' + error.message);
+    }
+
+    progress.set(progress.MIGRATE, 10, 'Backing up for migration');
+
+    // initiate the migration in the background
+    backups.backupBoxAndApps({ userId: null, username: 'migrator' }, function (error, backupId) {
+        if (error) return unlock(error);
+
+        debug('migrate: domain: %s size %s region %s', options.domain, options.size, options.region);
+
+        options.restoreKey = backupId;
+
+        superagent
+          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
+          .query({ token: config.token() })
+          .send(options)
+          .end(function (error, result) {
+            if (error && !error.response) return unlock(error); // network error
+            if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
+            if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
+            if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            progress.set(progress.MIGRATE, 10, 'Migrating');
+
+            retire('migrate', _.pick(options, 'domain', 'size', 'region'));
+        });
+    });
+
+    callback(null);
+}
+
+function migrate(options, callback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!options.domain) return doMigrate(options, callback);
+
+    var dnsConfig = _.pick(options, 'domain', 'provider', 'accessKeyId', 'secretAccessKey', 'region', 'endpoint');
+
+    settings.setDnsConfig(dnsConfig, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_FIELD, error.message));
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        doMigrate(options, callback);
+    });
+}

src/config.js

@@ -28,9 +28,9 @@ exports = module.exports = {
     internalAdminOrigin: internalAdminOrigin,
     sysadminOrigin: sysadminOrigin, // caas routes
     adminFqdn: adminFqdn,
+    mailFqdn: mailFqdn,
     appFqdn: appFqdn,
     zoneName: zoneName,
-    adminEmail: adminEmail,
 
     isDev: isDev,
 
@@ -73,11 +73,11 @@ function initConfig() {
     data.fqdn = 'localhost';
 
     data.token = null;
-    data.adminEmail = null;
     data.boxVersionsUrl = null;
     data.version = null;
     data.isCustomDomain = false;
     data.webServerOrigin = null;
+    data.smtpPort = 2525; // // this value comes from mail container
     data.sysadminPort = 3001;
     data.ldapPort = 3002;
     data.oauthProxyPort = 3003;
@@ -100,7 +100,6 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
-        data.adminEmail = 'test@cloudron.foo';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -139,10 +138,6 @@ function get(key) {
     return safe.query(data, key);
 }
 
-function adminEmail() {
-    return get('adminEmail');
-}
-
 function apiServerOrigin() {
     return get('apiServerOrigin');
 }
@@ -167,6 +162,10 @@ function adminFqdn() {
     return appFqdn(constants.ADMIN_LOCATION);
 }
 
+function mailFqdn() {
+    return appFqdn(constants.MAIL_LOCATION);
+}
+
 function adminOrigin() {
     return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }

src/constants.js

@@ -6,12 +6,16 @@ exports = module.exports = {
     API_LOCATION: 'api', // this is unused but reserved for future use (#403)
     SMTP_LOCATION: 'smtp',
     IMAP_LOCATION: 'imap',
+    MAIL_LOCATION: 'my', // not a typo! should be same as admin location until we figure out certificates
+    POSTMAN_LOCATION: 'postman', // used in dovecot bounces
 
     ADMIN_NAME: 'Settings',
 
     ADMIN_CLIENT_ID: 'webadmin', // oauth client id
     ADMIN_APPID: 'admin', // admin appid (settingsdb)
 
+    GHOST_USER_FILE: '/tmp/cloudron_ghost.json',
+
     DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024) // see also client.js
 };
 

src/cron.js

@@ -13,6 +13,7 @@ var apps = require('./apps.js'),
     config = require('./config.js'),
     CronJob = require('cron').CronJob,
     debug = require('debug')('box:cron'),
+    eventlog = require('./eventlog.js'),
     janitor = require('./janitor.js'),
     scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
@@ -27,9 +28,11 @@ var gAutoupdaterJob = null,
     gDockerVolumeCleanerJob = null,
     gSchedulerSyncJob = null,
     gCertificateRenewJob = null,
-    gCheckDiskSpaceJob = null;
+    gCheckDiskSpaceJob = null,
+    gCleanupEventlogJob = null;
 
 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
+var AUDIT_SOURCE = { userId: null, username: 'cron' };
 
 // cron format
 // Seconds: 0-59
@@ -65,8 +68,8 @@ function recreateJobs(unusedTimeZone, callback) {
 
         if (gBackupJob) gBackupJob.stop();
         gBackupJob = new CronJob({
-            cronTime: '00 00 */4 * * *', // every 4 hours
-            onTick: backups.ensureBackup,
+            cronTime: '00 00 */4 * * *', // every 4 hours. backups.ensureBackup() will only trigger a backup once per day
+            onTick: backups.ensureBackup.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
             start: true,
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
@@ -103,6 +106,14 @@ function recreateJobs(unusedTimeZone, callback) {
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
 
+        if (gCleanupEventlogJob) gCleanupEventlogJob.stop();
+        gCleanupEventlogJob = new CronJob({
+            cronTime: '00 */30 * * * *', // every 30 minutes
+            onTick: eventlog.cleanup,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
         if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
         gDockerVolumeCleanerJob = new CronJob({
             cronTime: '00 00 */12 * * *', // every 12 hours
@@ -122,7 +133,7 @@ function recreateJobs(unusedTimeZone, callback) {
         if (gCertificateRenewJob) gCertificateRenewJob.stop();
         gCertificateRenewJob = new CronJob({
             cronTime: '00 00 */12 * * *', // every 12 hours
-            onTick: certificates.autoRenew,
+            onTick: certificates.renewAll.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
             start: true,
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
@@ -154,10 +165,10 @@ function autoupdatePatternChanged(pattern) {
             var updateInfo = updateChecker.getUpdateInfo();
             if (updateInfo.box) {
                 debug('Starting autoupdate to %j', updateInfo.box);
-                cloudron.update(updateInfo.box, NOOP_CALLBACK);
+                cloudron.updateToLatest(AUDIT_SOURCE, NOOP_CALLBACK);
             } else if (updateInfo.apps) {
                 debug('Starting app update to %j', updateInfo.apps);
-                apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
+                apps.updateApps(updateInfo.apps, AUDIT_SOURCE, NOOP_CALLBACK);
             } else {
                 debug('No auto updates available');
             }
@@ -193,6 +204,9 @@ function uninitialize(callback) {
     if (gCleanupTokensJob) gCleanupTokensJob.stop();
     gCleanupTokensJob = null;
 
+    if (gCleanupEventlogJob) gCleanupEventlogJob.stop();
+    gCleanupEventlogJob = null;
+
     if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
     gDockerVolumeCleanerJob = null;
 

src/database.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -122,7 +120,8 @@ function clear(callback) {
         require('./groupdb.js')._clear,
         require('./userdb.js')._clear,
         require('./settingsdb.js')._clear,
-        require('./eventlogdb.js')._clear
+        require('./eventlogdb.js')._clear,
+        require('./mailboxdb.js')._clear
     ], callback);
 }
 

src/developer.js

@@ -5,7 +5,7 @@
 exports = module.exports = {
     DeveloperError: DeveloperError,
 
-    enabled: enabled,
+    isEnabled: isEnabled,
     setEnabled: setEnabled,
     issueDeveloperToken: issueDeveloperToken,
     getNonApprovedApps: getNonApprovedApps
@@ -13,6 +13,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     config = require('./config.js'),
+    clients = require('./clients.js'),
     debug = require('debug')('box:developer'),
     eventlog = require('./eventlog.js'),
     tokendb = require('./tokendb.js'),
@@ -42,7 +43,7 @@ util.inherits(DeveloperError, Error);
 DeveloperError.INTERNAL_ERROR = 'Internal Error';
 DeveloperError.EXTERNAL_ERROR = 'External Error';
 
-function enabled(callback) {
+function isEnabled(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     settings.getDeveloperMode(function (error, enabled) {
@@ -72,13 +73,14 @@ function issueDeveloperToken(user, auditSource, callback) {
 
     var token = tokendb.generateToken();
     var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+    var scopes = '*,' + clients.SCOPE_ROLE_SDK;
 
-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users,profile', function (error) {
+    tokendb.add(token, user.id, 'cid-cli', expiresAt, scopes, function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
 
         eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { authType: 'cli', userId: user.id, username: user.username });
 
-        callback(null, { token: token, expiresAt: expiresAt });
+        callback(null, { token: token, expiresAt: new Date(expiresAt).toISOString() });
     });
 }
 
@@ -88,7 +90,7 @@ function getNonApprovedApps(callback) {
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
     superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
         if (error && !error.response) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, error));
-        if (result.statusCode === 401) {
+        if (result.statusCode === 401 || result.statusCode === 403) {
             debug('Failed to list apps in development. Appstore token invalid or missing. Returning empty list.', result.body);
             return callback(null, []);
         }

src/dns/route53.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -7,12 +5,14 @@ exports = module.exports = {
     get: get,
     del: del,
     update: update,
-    getChangeStatus: getChangeStatus
+    getChangeStatus: getChangeStatus,
+
+    // not part of "dns" interface
+    getHostedZone: getHostedZone
 };
 
 var assert = require('assert'),
     AWS = require('aws-sdk'),
-    config = require('../config.js'),
     debug = require('debug')('box:dns/route53'),
     SubdomainError = require('../subdomains.js').SubdomainError,
     util = require('util'),
@@ -52,6 +52,24 @@ function getZoneByName(dnsConfig, zoneName, callback) {
     });
 }
 
+function getHostedZone(dnsConfig, zoneName, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+ 
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.getHostedZone({ Id: zone.Id }, function (error, result) {
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+
+            callback(null, result);
+        });
+    });
+}
+
 function add(dnsConfig, zoneName, subdomain, type, values, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
@@ -65,7 +83,7 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
     getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
 
-        var fqdn = config.appFqdn(subdomain);
+        var fqdn = subdomain === '' ? zoneName : subdomain + '.' + zoneName;
         var records = values.map(function (v) { return { Value: v }; });
 
         var params = {
@@ -153,7 +171,7 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
     getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
 
-        var fqdn = config.appFqdn(subdomain);
+        var fqdn = subdomain === '' ? zoneName : subdomain + '.' + zoneName;
         var records = values.map(function (v) { return { Value: v }; });
 
         var resourceRecordSet = {

src/docker.js

@@ -43,7 +43,6 @@ var addons = require('./addons.js'),
     debug = require('debug')('box:src/docker.js'),
     once = require('once'),
     safe = require('safetydance'),
-    semver = require('semver'),
     spawn = child_process.spawn,
     util = require('util'),
     _ = require('underscore');
@@ -133,7 +132,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var docker = exports.connection,
-        isAppContainer = !cmd;
+        isAppContainer = !cmd; // non app-containers are like scheduler containers
 
     var manifest = app.manifest;
     var developmentMode = !!manifest.developmentMode;
@@ -172,18 +171,16 @@ function createSubcontainer(app, name, cmd, options, callback) {
     // developerMode does not restrict memory usage
     memoryLimit = developmentMode ? 0 : memoryLimit;
 
-    // for subcontainers, this should ideally be false. but docker does not allow network sharing if the app container is not running
-    // this means cloudron exec does not work
-    var isolatedNetworkNs = true;
-
     addons.getEnvironment(app, function (error, addonEnv) {
         if (error) return callback(new Error('Error getting addon environment : ' + error));
 
+        // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail
+        // location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns
+        // name to look up the internal docker ip. this makes curl from within container fail
+        // Note that Hostname has no effect on DNS. We have to use the --net-alias for dns.
+        // Hostname cannot be set with container NetworkMode
         var containerOptions = {
             name: name, // used for filtering logs
-            // do _not_ set hostname to app fqdn. doing so sets up the dns name to look up the internal docker ip. this makes curl from within container fail
-            // for subcontainers, this should not be set because we already share the network namespace with app container
-            Hostname: isolatedNetworkNs ? (semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location)) : null,
             Tty: isAppContainer,
             Image: app.manifest.dockerImage,
             Cmd: (isAppContainer && developmentMode) ? [ '/bin/bash', '-c', 'echo "Development mode. Use cloudron exec to debug. Sleeping" && sleep infinity' ] : cmd,
@@ -211,8 +208,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                 },
                 CpuShares: 512, // relative to 1024 for system processes
                 VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
-                NetworkMode: isolatedNetworkNs ? 'default' : ('container:' + app.containerId), // share network namespace with parent
-                Links: isolatedNetworkNs ? addons.getLinksSync(app, app.manifest.addons) : null, // links is redundant with --net=container
+                NetworkMode: isAppContainer ? 'cloudron' : ('container:' + app.containerId), // share network namespace with parent
                 SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
             }
         };
@@ -375,10 +371,10 @@ function getContainerIdByIp(ip, callback) {
 
         var bridge;
         result.forEach(function (n) {
-            if (n.Name === 'bridge') bridge = n;
+            if (n.Name === 'cloudron') bridge = n;
         });
 
-        if (!bridge) return callback(new Error('Unable to find the bridge network'));
+        if (!bridge) return callback(new Error('Unable to find the cloudron network'));
 
         var containerId;
         for (var id in bridge.Containers) {

src/eventlog.js

@@ -6,9 +6,11 @@ exports = module.exports = {
     add: add,
     get: get,
     getAllPaged: getAllPaged,
+    cleanup: cleanup,
 
     // keep in sync with webadmin index.js filter
     ACTION_ACTIVATE: 'cloudron.activate',
+    ACTION_APP_CLONE: 'app.clone',
     ACTION_APP_CONFIGURE: 'app.configure',
     ACTION_APP_INSTALL: 'app.install',
     ACTION_APP_RESTORE: 'app.restore',
@@ -99,3 +101,16 @@ function getAllPaged(action, search, page, perPage, callback) {
         callback(null, boxes);
     });
 }
+
+function cleanup(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+     var d = new Date();
+     d.setDate(d.getDate() - 7); // 7 days ago
+
+    eventlogdb.delByCreationTime(d, function (error) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}

src/eventlogdb.js

@@ -5,6 +5,7 @@ exports = module.exports = {
     getAllPaged: getAllPaged,
     add: add,
     count: count,
+    delByCreationTime: delByCreationTime,
 
     _clear: clear
 };
@@ -13,7 +14,8 @@ var assert = require('assert'),
     database = require('./database.js'),
     DatabaseError = require('./databaseerror'),
     mysql = require('mysql'),
-    safe = require('safetydance');
+    safe = require('safetydance'),
+    util = require('util');
 
 var EVENTLOGS_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
 
@@ -102,3 +104,13 @@ function clear(callback) {
     });
 }
 
+function delByCreationTime(creationTime, callback) {
+    assert(util.isDate(creationTime));
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM eventlog WHERE creationTime < ?', [ creationTime ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(error);
+    });
+}

src/groupdb.js

@@ -4,6 +4,7 @@ exports = module.exports = {
     get: get,
     getWithMembers: getWithMembers,
     getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
     add: add,
     del: del,
     count: count,
@@ -65,6 +66,19 @@ function getAll(callback) {
     });
 }
 
+function getAllWithMembers(callback) {
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' GROUP BY groups.id', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
+
+        callback(null, results);
+    });
+}
+
 function add(id, name, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof name, 'string');

src/groups.js

@@ -10,6 +10,7 @@ exports = module.exports = {
     get: get,
     getWithMembers: getWithMembers,
     getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
 
     getMembers: getMembers,
     addMember: addMember,
@@ -51,7 +52,7 @@ util.inherits(GroupError, Error);
 GroupError.INTERNAL_ERROR = 'Internal Error';
 GroupError.ALREADY_EXISTS = 'Already Exists';
 GroupError.NOT_FOUND = 'Not Found';
-GroupError.BAD_NAME = 'Bad name';
+GroupError.BAD_FIELD = 'Field error';
 GroupError.NOT_EMPTY = 'Not Empty';
 GroupError.NOT_ALLOWED = 'Not Allowed';
 
@@ -59,12 +60,12 @@ function validateGroupname(name) {
     assert.strictEqual(typeof name, 'string');
     var RESERVED = [ 'admins', 'users' ]; // ldap code uses 'users' pseudo group
 
-    if (name.length <= 2) return new GroupError(GroupError.BAD_NAME, 'name must be atleast 2 chars');
-    if (name.length >= 200) return new GroupError(GroupError.BAD_NAME, 'name too long');
+    if (name.length <= 2) return new GroupError(GroupError.BAD_FIELD, 'name must be atleast 2 chars');
+    if (name.length >= 200) return new GroupError(GroupError.BAD_FIELD, 'name too long');
 
-    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_NAME, 'name can only have A-Za-z0-9_-');
+    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_FIELD, 'name can only have A-Za-z0-9_-');
 
-    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_NAME, 'name is reserved');
+    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_FIELD, 'name is reserved');
 
     return null;
 }
@@ -133,6 +134,16 @@ function getAll(callback) {
     });
 }
 
+function getAllWithMembers(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAllWithMembers(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
 function getMembers(groupId, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof callback, 'function');

src/infra_version.js

@@ -0,0 +1,22 @@
+'use strict';
+
+// WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+// These constants are used in the installer script as well
+// Do not require anything here!
+
+exports = module.exports = {
+    // a version bump means that all containers (apps and addons) are recreated
+    'version': 40,
+
+    'baseImage': 'cloudron/base:0.8.1',
+
+    'images': {
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:0.12.0' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.11.0' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.10.0' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.9.0' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.18.0' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.9.0' }
+    }
+};
+

src/ldap.js

@@ -12,7 +12,9 @@ var assert = require('assert'),
     eventlog = require('./eventlog.js'),
     user = require('./user.js'),
     UserError = user.UserError,
-    ldap = require('ldapjs');
+    ldap = require('ldapjs'),
+    mailboxes = require('./mailboxes.js'),
+    MailboxError = mailboxes.MailboxError;
 
 var gServer = null;
 
@@ -35,8 +37,204 @@ function getAppByRequest(req, callback) {
     if (sourceIp.split('.').length !== 4) return callback(new ldap.InsufficientAccessRightsError('Missing source identifier'));
 
     apps.getByIpAddress(sourceIp, function (error, app) {
-        // we currently allow access in case we can't find the source app
-        callback(null, app || null);
+        if (error) return callback(new ldap.OperationsError(error.message));
+
+        if (!app) return callback(new ldap.OperationsError('Could not detect app source'));
+
+        callback(null, app);
+    });
+}
+
+function userSearch(req, res, next) {
+    debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        // send user objects
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+
+            var groups = [ GROUP_USERS_DN ];
+            if (entry.admin) groups.push(GROUP_ADMINS_DN);
+
+            var displayName = entry.displayName || entry.username;
+            var nameParts = displayName.split(' ');
+            var firstName = nameParts[0];
+            var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['user'],
+                    objectcategory: 'person',
+                    cn: entry.id,
+                    uid: entry.id,
+                    mail: entry.email,
+                    // TODO: check mailboxes before we send this
+                    mailAlternateAddress: entry.username + '@' + config.fqdn(),
+                    displayname: displayName,
+                    givenName: firstName,
+                    username: entry.username,
+                    samaccountname: entry.username,      // to support ActiveDirectory clients
+                    memberof: groups
+                }
+            };
+
+            // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
+            // which is required to have atleast one character if present
+            if (lastName.length !== 0) obj.attributes.sn = lastName;
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function groupSearch(req, res, next) {
+    debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result){
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        var groups = [{
+            name: 'users',
+            admin: false
+        }, {
+            name: 'admins',
+            admin: true
+        }];
+
+        groups.forEach(function (group) {
+            var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+            var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['group'],
+                    cn: group.name,
+                    memberuid: members.map(function(entry) { return entry.id; })
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function mailboxSearch(req, res, next) {
+    debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.name + ',ou=mailboxes,dc=cloudron');
+
+            // TODO: send aliases
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['mailbox'],
+                    objectcategory: 'mailbox',
+                    cn: entry.name,
+                    uid: entry.name,
+                    mail: entry.name + '@' + config.fqdn()
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function authenticateUser(req, res, next) {
+    debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
+
+    // extract the common name which might have different attribute names
+    var attributeName = Object.keys(req.dn.rdns[0])[0];
+    var commonName = req.dn.rdns[0][attributeName];
+    if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+    var api;
+    if (attributeName === 'mail') {
+        api = user.verifyWithEmail;
+    } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
+        var parts = commonName.split('@');
+        if (parts[1] === config.fqdn()) { // internal email, verify with username
+            commonName = parts[0];
+            api = user.verifyWithUsername;
+        } else { // external email
+            api = user.verifyWithEmail;
+        }
+    } else if (commonName.indexOf('uid-') === 0) {
+        api = user.verify;
+    } else {
+        api = user.verifyWithUsername;
+    }
+
+    api(commonName, req.credentials || '', function (error, user) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.message));
+
+        req.user = user;
+
+        next();
+    });
+}
+
+function authorizeUserForApp(req, res, next) {
+    assert(req.user);
+
+    getAppByRequest(req, function (error, app) {
+        if (error) return next(error);
+
+        apps.hasAccessTo(app, req.user, function (error, result) {
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            // we return no such object, to avoid leakage of a users existence
+            if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+            eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: app.id }, { userId: req.user.id });
+
+            res.end();
+        });
+    });
+}
+
+function authorizeUserForMailbox(req, res, next) {
+    assert(req.user);
+
+    // We simply authorize the user to access a mailbox by his own name
+    mailboxes.get(req.user.username, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.message));
+
+        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: req.user.username }, { userId: req.user.username });
+
+        res.end();
     });
 }
 
@@ -45,155 +243,27 @@ function start(callback) {
 
     gServer = ldap.createServer({ log: gLogger });
 
-    gServer.search('ou=users,dc=cloudron', function (req, res, next) {
-        debug('user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+    gServer.search('ou=users,dc=cloudron', userSearch);
+    gServer.search('ou=groups,dc=cloudron', groupSearch);
+    gServer.bind('ou=users,dc=cloudron', authenticateUser, authorizeUserForApp);
 
-        user.list(function (error, result) {
-            if (error) return next(new ldap.OperationsError(error.toString()));
+    gServer.search('ou=mailboxes,dc=cloudron', mailboxSearch);
+    gServer.bind('ou=mailboxes,dc=cloudron', authenticateUser, authorizeUserForMailbox);
 
-            // send user objects
-            result.forEach(function (entry) {
-                var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
-
-                var groups = [ GROUP_USERS_DN ];
-                if (entry.admin) groups.push(GROUP_ADMINS_DN);
-
-                var displayName = entry.displayName || entry.username;
-                var nameParts = displayName.split(' ');
-                var firstName = nameParts[0];
-                var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
-
-                var obj = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['user'],
-                        objectcategory: 'person',
-                        cn: entry.id,
-                        uid: entry.id,
-                        mail: entry.email,
-                        displayname: displayName,
-                        givenName: firstName,
-                        username: entry.username,
-                        samaccountname: entry.username,      // to support ActiveDirectory clients
-                        memberof: groups
-                    }
-                };
-
-                // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
-                // which is required to have atleast one character if present
-                if (lastName.length !== 0) obj.attributes.sn = lastName;
-
-                // ensure all filter values are also lowercase
-                var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                    res.send(obj);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
-        debug('group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
-
-        user.list(function (error, result){
-            if (error) return next(new ldap.OperationsError(error.toString()));
-
-            var groups = [{
-                name: 'users',
-                admin: false
-            }, {
-                name: 'admins',
-                admin: true
-            }];
-
-            groups.forEach(function (group) {
-                var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-                var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
-
-                var obj = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['group'],
-                        cn: group.name,
-                        memberuid: members.map(function(entry) { return entry.id; })
-                    }
-                };
-
-                // ensure all filter values are also lowercase
-                var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                    res.send(obj);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    // this is the bind for the mail addon to authorize apps
-    gServer.bind('ou=sendmail,dc=cloudron', function(req, res, next) {
-        // TODO: validate password
-        debug('sendmail bind: %s', req.dn.toString()); // note: cn can be email or id
+    // this is the bind for addons (after bind, they might search and authenticate)
+    gServer.bind('ou=addons,dc=cloudron', function(req, res, next) {
+        debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id
         res.end();
     });
 
+    // this is the bind for apps (after bind, they might search and authenticate user)
     gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
         // TODO: validate password
         debug('application bind: %s', req.dn.toString());
         res.end();
     });
 
-    gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
-        debug('user bind: %s', req.dn.toString());
-
-        // extract the common name which might have different attribute names
-        var attributeName = Object.keys(req.dn.rdns[0])[0];
-        var commonName = req.dn.rdns[0][attributeName];
-        if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-        var api;
-        // if mail is specified, enforce mail check
-        if (commonName.indexOf('@') !== -1 || attributeName === 'mail') {
-            api = user.verifyWithEmail;
-        } else if (commonName.indexOf('uid-') === 0) {
-            api = user.verify;
-        } else {
-            api = user.verifyWithUsername;
-        }
-
-        // TODO this should be done after we verified the app has access to avoid leakage of user existence
-        api(commonName, req.credentials || '', function (error, userObject) {
-            if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
-            if (error) return next(new ldap.OperationsError(error));
-
-            getAppByRequest(req, function (error, app) {
-                if (error) return next(error);
-
-                if (!app) {
-                    debug('no app found for this container, allow access');
-                    return res.end();
-                }
-
-                apps.hasAccessTo(app, userObject, function (error, result) {
-                    if (error) return next(new ldap.OperationsError(error.toString()));
-
-                    // we return no such object, to avoid leakage of a users existence
-                    if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-                    eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: app.id }, { userId: userObject.id });
-
-                    res.end();
-                });
-            });
-        });
-    });
-
-    gServer.listen(config.get('ldapPort'), callback);
+    gServer.listen(config.get('ldapPort'), '0.0.0.0', callback);
 }
 
 function stop(callback) {

src/mail_templates/certificate_renewal_error.ejs

@@ -1,13 +1,10 @@
 <%if (format === 'text') { %>
 
 Dear Cloudron Team,
-<% if (message) { %>
 <%= domain %> was not renewed.
 
 <%- message %>
-<% } else { %>
-<%= domain %> was renewed.
-<% } %>
+
 Thank you,
 Your Cloudron
 <% } else { %>

src/mail_templates/out_of_disk_space.ejs

@@ -4,7 +4,7 @@ Dear Cloudron Team,
 
 <%= fqdn %> is running out of disk space.
 
-Please see some excerpts of the logs below.
+Disk space logs are attached.
 
 Thank you,
 Your Cloudron

src/mail_templates/user_added.ejs

@@ -2,7 +2,7 @@
 
 Dear Admin,
 
-User with name <%= user.email %> was added in the Cloudron at <%= fqdn %>.
+User with email <%= user.email %> was added in the Cloudron at <%= fqdn %>.
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 

src/mailboxdb.js

@@ -0,0 +1,127 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    getAliases: getAliases,
+    setAliases: setAliases,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var MAILBOX_FIELDS = [ 'name', 'aliasTarget', 'creationTime' ].join(',');
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO mailboxes (name) VALUES (?)', [ name ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE mailboxes', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // deletes aliases as well
+    database.query('DELETE FROM mailboxes WHERE name=? OR aliasTarget = ?', [ name, name ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function postProcess(result) {
+    result.aliases = result.aliases ? result.aliases.split(',') : [ ];
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.name=? AND m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(results[0]);
+
+        callback(null, results[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM mailboxes WHERE aliasTarget = ?', args: [ name ] });
+    aliases.forEach(function (alias) {
+        queries.push({ query: 'INSERT INTO mailboxes (name, aliasTarget) VALUES (?, ?)', args: [ alias, name ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name FROM mailboxes WHERE aliasTarget=? ORDER BY name', [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results = results.map(function (r) { return r.name; });
+        callback(null, results);
+    });
+}

src/mailboxes.js

@@ -0,0 +1,199 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    setAliases: setAliases,
+    getAliases: getAliases,
+
+    setupAliases: setupAliases,
+
+    MailboxError: MailboxError
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:mailboxes'),
+    docker = require('./docker.js'),
+    mailboxdb = require('./mailboxdb.js'),
+    util = require('util');
+
+function MailboxError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(MailboxError, Error);
+MailboxError.ALREADY_EXISTS = 'already exists';
+MailboxError.BAD_FIELD = 'Field error';
+MailboxError.NOT_FOUND = 'not found';
+MailboxError.INTERNAL_ERROR = 'internal error';
+MailboxError.EXTERNAL_ERROR = 'external error';
+
+function validateName(name) {
+    var RESERVED_NAMES = [ 'no-reply', 'postmaster', 'mailer-daemon' ];
+
+    if (!name.length) return new MailboxError(MailboxError.BAD_FIELD, "name cannot be empty");
+
+    if (name.length < 2) return new MailboxError(MailboxError.BAD_FIELD, 'name too small');
+    if (name.length > 127) return new MailboxError(MailboxError.BAD_FIELD, 'name too long');
+    if (RESERVED_NAMES.indexOf(name) !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'name is reserved');
+
+    if (/[^a-zA-Z0-9.]/.test(name)) return new MailboxError(MailboxError.BAD_FIELD, 'name can only contain alphanumerals and dot');
+
+    if (name.indexOf('.app') !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'alias pattern is reserved for apps');
+
+    return null;
+}
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    name = name.toLowerCase();
+
+    var error = validateName(name);
+    if (error) return callback(error);
+
+    mailboxdb.add(name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        debug('Added mailbox %s', name);
+
+        var mailbox = {
+            name: name
+        };
+
+        callback(null, mailbox);
+    });
+}
+
+function pushAlias(name, aliases, callback) {
+    if (process.env.BOX_ENV === 'test') return callback();
+
+    var cmd = [ '/addons/mail/service.sh', 'set-alias', name ].concat(aliases);
+
+    debug('pushing alias for %s : %j', name, aliases);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(new MailboxError(MailboxError.EXTERNAL_ERROR, error));
+
+        callback();
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    pushAlias(name, [ ], function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.del(name, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+            debug('deleted mailbox %s', name);
+
+            callback();
+        });
+    });
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.get(name, function (error, mailbox) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, mailbox);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAll(function (error, results) {
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    for (var i = 0; i < aliases.length; i++) {
+        aliases[i] = aliases[i].toLowerCase();
+
+        var error = validateName(aliases[i]);
+        if (error) return callback(error);
+    }
+
+    pushAlias(name, aliases, function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.setAliases(name, aliases, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS, error.message))
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+
+            callback(null);
+        });
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAliases(name, function (error, aliases) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, aliases);
+    });
+}
+
+// push aliases to the mail container on startup
+function setupAliases(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    getAll(function (error, mailboxes) {
+        if (error) return callback(error);
+
+        async.each(mailboxes, function (mailbox, iteratorDone) {
+            getAliases(mailbox.name, function (error, aliases) {
+                if (error) return iteratorDone(error);
+
+                if (aliases.length === 0) return iteratorDone();
+
+                pushAlias(mailbox.name, aliases, iteratorDone);
+            });
+        }, callback);
+    });
+}
+

src/mailer.js

@@ -18,7 +18,7 @@ exports = module.exports = {
 
     outOfDiskSpace: outOfDiskSpace,
 
-    certificateRenewed: certificateRenewed,
+    certificateRenewalError: certificateRenewalError,
 
     FEEDBACK_TYPE_FEEDBACK: 'feedback',
     FEEDBACK_TYPE_TICKET: 'ticket',
@@ -41,6 +41,7 @@ var assert = require('assert'),
     ejs = require('ejs'),
     nodemailer = require('nodemailer'),
     path = require('path'),
+    platform = require('./platform.js'),
     safe = require('safetydance'),
     smtpTransport = require('nodemailer-smtp-transport'),
     users = require('./user.js'),
@@ -113,7 +114,7 @@ function getTxtRecords(callback) {
 function checkDns() {
     getTxtRecords(function (error, records) {
         if (error || !records) {
-            debug('checkDns: DNS error or no records looking up TXT records for %s %s', config.adminFqdn(), error, records);
+            debug('checkDns: DNS error or no records looking up TXT records for %s %s', config.fqdn(), error, records);
             gCheckDnsTimerId = setTimeout(checkDns, 60000);
             return;
         }
@@ -154,19 +155,19 @@ function sendMails(queue) {
     docker.getContainer('mail').inspect(function (error, data) {
         if (error) return console.error(error);
 
-        var mailServerIp = safe.query(data, 'NetworkSettings.IPAddress');
+        var mailServerIp = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
         if (!mailServerIp) return debug('Error querying mail server IP');
 
         var transport = nodemailer.createTransport(smtpTransport({
             host: mailServerIp,
-            port: 2500, // this value comes from mail container
+            port: config.get('smtpPort'),
             auth: {
-                user: 'no-reply', // derive from adminEmail
-                pass: 'supersecret'
+                user: platform.mailConfig().username,
+                pass: platform.mailConfig().password
             }
         }));
 
-        debug('Processing mail queue of size %d (through %s:2500)', queue.length, mailServerIp);
+        debug('Processing mail queue of size %d (through %s:2525)', queue.length, mailServerIp);
 
         async.mapSeries(queue, function iterator(mailOptions, callback) {
             transport.sendMail(mailOptions, function (error) {
@@ -222,7 +223,7 @@ function mailUserEventToAdmins(user, event) {
         adminEmails = _.difference(adminEmails, [ user.email ]);
 
         var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s %s in Cloudron %s', user.username || user.email, event, config.fqdn()),
             text: render('user_event.ejs', { fqdn: config.fqdn(), user: user, event: event, format: 'text' }),
@@ -248,7 +249,7 @@ function sendInvite(user, invitor) {
     };
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: user.email,
         subject: util.format('Welcome to Cloudron %s', config.fqdn()),
         text: render('welcome_user.ejs', templateData)
@@ -271,7 +272,7 @@ function userAdded(user, inviteSent) {
         var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken;
 
         var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s added in Cloudron %s', user.email, config.fqdn()),
             text: render('user_added.ejs', { fqdn: config.fqdn(), user: user, inviteLink: inviteLink, format: 'text' }),
@@ -306,7 +307,7 @@ function passwordReset(user) {
     var resetLink = config.adminOrigin() + '/api/v1/session/password/reset.html?reset_token=' + user.resetToken;
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: user.email,
         subject: 'Password Reset Request',
         text: render('password_reset.ejs', { fqdn: config.fqdn(), user: user, resetLink: resetLink, format: 'text' })
@@ -324,7 +325,7 @@ function appDied(app) {
         if (error) return console.log('Error getting admins', error);
 
         var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.concat('support@cloudron.io').join(', '),
             subject: util.format('App %s is down', app.location),
             text: render('app_down.ejs', { fqdn: config.fqdn(), title: app.manifest.title, appFqdn: config.appFqdn(app.location), format: 'text' })
@@ -342,7 +343,7 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', config.fqdn()),
             text: render('box_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), newBoxVersion: newBoxVersion, changelog: changelog, format: 'text' })
@@ -360,7 +361,7 @@ function appUpdateAvailable(app, updateInfo) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', app.fqdn),
             text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, updateInfo: updateInfo, format: 'text' })
@@ -373,28 +374,36 @@ function appUpdateAvailable(app, updateInfo) {
 function outOfDiskSpace(message) {
     assert.strictEqual(typeof message, 'string');
 
-    var mailOptions = {
-        from: config.adminEmail(),
-        to: 'admin@cloudron.io',
-        subject: util.format('[%s] Out of disk space alert', config.fqdn()),
-        text: render('out_of_disk_space.ejs', { fqdn: config.fqdn(), message: message, format: 'text' })
-    };
+    getAdminEmails(function (error, adminEmails) {
+        if (error) return console.log('Error getting admins', error);
 
-    sendMails([ mailOptions ]);
+        var mailOptions = {
+            from: platform.mailConfig().from,
+            to: config.provider() === 'caas' ? 'support@cloudron.io' : adminEmails.join(', '),
+            subject: util.format('[%s] Out of disk space alert', config.fqdn()),
+            text: render('out_of_disk_space.ejs', { fqdn: config.fqdn(), message: message, format: 'text' })
+        };
+
+        sendMails([ mailOptions ]);
+    });
 }
 
-function certificateRenewed(domain, message) {
+function certificateRenewalError(domain, message) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof message, 'string');
 
-    var mailOptions = {
-        from: config.adminEmail(),
-        to: 'admin@cloudron.io',
-        subject: util.format('[%s] Certificate was %s renewed', domain, message ? 'not' : ''),
-        text: render('certificate_renewed.ejs', { domain: domain, message: message, format: 'text' })
-    };
+    getAdminEmails(function (error, adminEmails) {
+        if (error) return console.log('Error getting admins', error);
 
-    sendMails([ mailOptions ]);
+        var mailOptions = {
+            from: platform.mailConfig().from,
+            to: config.provider() === 'caas' ? 'support@cloudron.io' : adminEmails.join(', '),
+            subject: util.format('[%s] Certificate renewal error', domain),
+            text: render('certificate_renewal_error.ejs', { domain: domain, message: message, format: 'text' })
+        };
+
+        sendMails([ mailOptions ]);
+    });
 }
 
 // this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized
@@ -404,8 +413,8 @@ function unexpectedExit(program, context) {
     assert.strictEqual(typeof context, 'string');
 
     var mailOptions = {
-        from: config.adminEmail(),
-        to: 'admin@cloudron.io',
+        from: platform.mailConfig().from,
+        to: 'support@cloudron.io',
         subject: util.format('[%s] %s exited unexpectedly', config.fqdn(), program),
         text: render('unexpected_exit.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
     };
@@ -426,7 +435,7 @@ function sendFeedback(user, type, subject, description) {
         type === exports.FEEDBACK_TYPE_APP_ERROR);
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: 'support@cloudron.io',
         subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
         text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})

src/middleware/cors.js

@@ -20,6 +20,7 @@ module.exports = function cors(options) {
         if (!requestOrigin) return next();
 
         requestOrigin = url.parse(requestOrigin);
+        if (!requestOrigin.host) return res.status(405).send('CORS not allowed from this domain');
 
         var hostname = requestOrigin.host.split(':')[0]; // remove any port
         var originAllowed = origins.some(function (o) { return o === '*' || o === hostname; });

src/middleware/index.js

@@ -4,7 +4,6 @@ exports = module.exports = {
     cookieParser: require('cookie-parser'),
     cors: require('./cors'),
     csrf: require('csurf'),
-    favicon: require('serve-favicon'),
     json: require('body-parser').json,
     morgan: require('morgan'),
     proxy: require('proxy-middleware'),

src/nginx.js

@@ -45,7 +45,8 @@ function configureAdmin(certFilePath, keyFilePath, callback) {
         vhost: config.adminFqdn(),
         endpoint: 'admin',
         certFilePath: certFilePath,
-        keyFilePath: keyFilePath
+        keyFilePath: keyFilePath,
+        xFrameOptions: 'SAMEORIGIN'
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, 'admin.conf');
@@ -73,7 +74,8 @@ function configureApp(app, certFilePath, keyFilePath, callback) {
         port: app.httpPort,
         endpoint: endpoint,
         certFilePath: certFilePath,
-        keyFilePath: keyFilePath
+        keyFilePath: keyFilePath,
+        xFrameOptions: app.xFrameOptions || 'SAMEORIGIN'    // once all apps have been updated/
     };
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 

src/oauth2views/account_setup.ejs

@@ -8,7 +8,10 @@
 
 // very basic angular app
 var app = angular.module('Application', []);
-app.controller('Controller', [function () {}]);
+app.controller('Controller', ['$scope', function ($scope) {
+    $scope.username = '<%= (user && user.username) ? user.username : '' %>';
+    $scope.displayName = '<%= (user && user.displayName) ? user.displayName : '' %>';
+}]);
 
 </script>
 
@@ -28,6 +31,12 @@ app.controller('Controller', [function () {}]);
 
                 <center><p class="has-error"><%= error %></p></center>
 
+<% if (user && user.username) { %>
+                <div class="form-group"">
+                    <label class="control-label">Username</label>
+                    <input type="text" class="form-control" ng-model="username" name="username" readonly required>
+                </div>
+<% } else { %>
                 <div class="form-group" ng-class="{ 'has-error': (setupForm.username.$dirty && setupForm.username.$invalid) }">
                     <label class="control-label">Username</label>
                     <div class="control-label" ng-show="setupForm.username.$dirty && setupForm.username.$invalid">
@@ -37,6 +46,7 @@ app.controller('Controller', [function () {}]);
                     </div>
                     <input type="text" class="form-control" ng-model="username" name="username" ng-maxlength="512" ng-minlength="3" required autofocus>
                 </div>
+<% } %>
 
                 <div class="form-group">
                     <label class="control-label">Display Name</label>

src/oauthproxy.js

@@ -7,7 +7,7 @@ exports = module.exports = {
 
 var appdb = require('./appdb.js'),
     assert = require('assert'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:proxy'),
@@ -124,7 +124,7 @@ function authenticate(req, res, next) {
                 return res.send(500, 'Unknown app.');
             }
 
-            clientdb.getByAppIdAndType(result.id, clientdb.TYPE_PROXY, function (error, result) {
+            clients.getByAppIdAndType(result.id, clients.TYPE_PROXY, function (error, result) {
                 if (error) {
                     console.error('Unknown OAuth client.', error);
                     return res.send(500, 'Unknown OAuth client.');

src/paths.js

@@ -27,5 +27,7 @@ exports = module.exports = {
     UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json'),
 
     ACME_CHALLENGES_DIR: path.join(config.baseDir(), 'data/acme'),
-    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key')
+    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key'),
+
+    INFRA_VERSION_FILE: path.join(config.baseDir(), 'data/INFRA_VERSION')
 };

src/platform.js

@@ -0,0 +1,312 @@
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize,
+
+    events: new (require('events').EventEmitter)(),
+    EVENT_READY: 'ready',
+
+    isReadySync: isReadySync,
+
+    mailConfig: mailConfig
+};
+
+var apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    certificates = require('./certificates.js'),
+    debug = require('debug')('box:platform'),
+    fs = require('fs'),
+    hat = require('hat'),
+    infra = require('./infra_version.js'),
+    ini = require('ini'),
+    mailboxes = require('./mailboxes.js'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
+    util = require('util'),
+    _ = require('underscore');
+
+var gAddonVars = null,
+    gPlatformReadyTimer = null;
+
+function initialize(callback) {
+    if (process.env.BOX_ENV === 'test' && !process.env.CREATE_INFRA) return callback();
+
+    debug('initializing addon infrastructure');
+
+    var existingInfra = { version: 'none' };
+    if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
+        existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8'));
+        if (!existingInfra) existingInfra = { version: 'corrupt' };
+    }
+
+    // short-circuit for the restart case
+    if (_.isEqual(infra, existingInfra)) {
+        debug('platform is uptodate at version %s', infra.version);
+        process.nextTick(function () { exports.events.emit(exports.EVENT_READY); });
+        return loadAddonVars(callback);
+    }
+
+    debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version);
+
+    async.series([
+        stopContainers.bind(null, existingInfra),
+        createDockerNetwork,
+        startAddons.bind(null, existingInfra),
+        removeOldImages,
+        startApps.bind(null, existingInfra),
+        loadAddonVars,
+        fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra))
+    ], function (error) {
+        if (error) return callback(error);
+
+        // give 30 seconds for the platform to "settle". For example, mysql might still be initing the
+        // database dir and we cannot call service scripts until that's done.
+        // TODO: make this smarter to not wait for 30secs for the crash-restart case
+        gPlatformReadyTimer = setTimeout(function () {
+            debug('emitting platform ready');
+            gPlatformReadyTimer = null;
+            exports.events.emit(exports.EVENT_READY);
+        }, 30000);
+
+        callback();
+    });
+}
+
+function uninitialize(callback) {
+    clearTimeout(gPlatformReadyTimer);
+    gPlatformReadyTimer = null;
+    callback();
+}
+
+function isReadySync() {
+    return gPlatformReadyTimer === null;
+}
+
+function removeOldImages(callback) {
+    debug('removing old addon images');
+
+    for (var imageName in infra.images) {
+        var image = infra.images[imageName];
+        debug('cleaning up images of %j', image);
+        var cmd = 'docker images "%s" | tail -n +2 | awk \'{ print $1 ":" $2 }\' | grep -v "%s" | xargs --no-run-if-empty docker rmi';
+        shell.execSync('removeOldImagesSync', util.format(cmd, image.repo, image.tag));
+    }
+
+    callback();
+}
+
+function stopContainers(existingInfra, callback) {
+    // TODO: be nice and stop addons cleanly (example, shutdown commands)
+
+    if (existingInfra.version !== infra.version) { // infra upgrade
+        debug('stopping all containers for infra upgrade');
+        shell.execSync('stopContainers', 'docker ps -qa | xargs --no-run-if-empty docker rm -f');
+    } else {
+        assert(typeof infra.images, 'object');
+        var changedAddons = [ ];
+        for (var imageName in infra.images) {
+            if (infra.images[imageName].tag !== existingInfra.images[imageName].tag) changedAddons.push(imageName);
+        }
+
+        debug('stopping addons for incremental infra update: %j', changedAddons);
+        shell.execSync('stopContainers', 'docker rm -f ' + changedAddons.join(' '));
+    }
+
+    callback();
+}
+
+function createDockerNetwork(callback) {
+    shell.execSync('createDockerNetwork', 'docker network create --subnet=172.18.0.0/16 cloudron || true', callback);
+}
+
+function startGraphite(callback) {
+    const tag = infra.images.graphite.tag;
+    const dataDir = paths.DATA_DIR;
+
+    const cmd = `docker run --restart=always -d --name="graphite" \
+                --net cloudron \
+                --net-alias graphite \
+                -m 75m \
+                --memory-swap 150m \
+                -p 127.0.0.1:2003:2003 \
+                -p 127.0.0.1:2004:2004 \
+                -p 127.0.0.1:8000:8000 \
+                -v "${dataDir}/graphite:/app/data" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startGraphite', cmd);
+
+    callback();
+}
+
+function startMysql(callback) {
+    const tag = infra.images.mysql.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/mysql_vars.sh', 
+            'MYSQL_ROOT_PASSWORD=' + rootPassword +'\nMYSQL_ROOT_HOST=172.18.0.1', 'utf8')) {
+        return callback(new Error('Could not create mysql var file:' + safe.error.message));
+    }
+
+    const cmd = `docker run --restart=always -d --name="mysql" \
+                --net cloudron \
+                --net-alias mysql \
+                -m 256m \
+                --memory-swap 512m \
+                -v "${dataDir}/mysql:/var/lib/mysql" \
+                -v "${dataDir}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startMysql', cmd);
+
+    callback();
+}
+
+function startPostgresql(callback) {
+    const tag = infra.images.postgresql.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/postgresql_vars.sh', 'POSTGRESQL_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
+        return callback(new Error('Could not create postgresql var file:' + safe.error.message));
+    }
+
+    const cmd = `docker run --restart=always -d --name="postgresql" \
+                --net cloudron \
+                --net-alias postgresql \
+                -m 100m \
+                --memory-swap 200m \
+                -v "${dataDir}/postgresql:/var/lib/postgresql" \
+                -v "${dataDir}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startPostgresql', cmd);
+
+    callback();
+}
+
+function startMongodb(callback) {
+    const tag = infra.images.mongodb.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/mongodb_vars.sh', 'MONGODB_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
+        return callback(new Error('Could not create mongodb var file:' + safe.error.message));
+    }
+
+    const cmd = `docker run --restart=always -d --name="mongodb" \
+                --net cloudron \
+                --net-alias mongodb \
+                -m 100m \
+                --memory-swap 200m \
+                -v "${dataDir}/mongodb:/var/lib/mongodb" \
+                -v "${dataDir}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startMongodb', cmd);
+
+    callback();
+}
+
+function startMail(callback) {
+    // mail (note: 2525 is hardcoded in mail container and app use this port)
+    // MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
+    // MAIL_DOMAIN is the domain for which this server is relaying mails
+    // mail container uses /app/data for backed up data and /run for restart-able data
+
+    const tag = infra.images.mail.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+    const fqdn = config.fqdn();
+    const mailFqdn = config.adminFqdn();
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/mail_vars.sh',
+            'MAIL_ROOT_USERNAME=no-reply\nMAIL_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
+        return callback(new Error('Could not create mail var file:' + safe.error.message));
+    }
+
+    certificates.getAdminCertificatePath(function (error, certFilePath, keyFilePath) {
+        if (error) return callback(error);
+
+        const cmd = `docker run --restart=always -d --name="mail" \
+                    --net cloudron \
+                    --net-alias mail \
+                    -m 75m \
+                    --memory-swap 150m \
+                    -e "MAIL_DOMAIN=${fqdn}" \
+                    -e "MAIL_SERVER_NAME=${mailFqdn}" \
+                    -v "${dataDir}/box/mail:/app/data" \
+                    -v "${dataDir}/mail:/run" \
+                    -v "${dataDir}/addons/mail_vars.sh:/etc/mail/mail_vars.sh:ro" \
+                    -v "${certFilePath}:/etc/tls_cert.pem:ro" \
+                    -v "${keyFilePath}:/etc/tls_key.pem:ro" \
+                    -p 587:2525 \
+                    -p 993:9993 \
+                    -p 4190:4190 \
+                    -p 25:2525 \
+                    --read-only -v /tmp ${tag}`;
+
+        shell.execSync('startMail', cmd);
+
+        mailboxes.setupAliases(callback);
+    });
+}
+
+function startAddons(existingInfra, callback) {
+    var startFuncs = [ ];
+
+    if (existingInfra.version !== infra.version) {
+        debug('startAddons: no existing infra or infra upgrade. starting all addons');
+        startFuncs.push(startGraphite, startMysql, startPostgresql, startMongodb, startMail);
+    } else {
+        assert.strictEqual(typeof existingInfra.images, 'object');
+
+        if (infra.images.graphite.tag !== existingInfra.images.graphite.tag) startFuncs.push(startGraphite);
+        if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) startFuncs.push(startMysql);
+        if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) startFuncs.push(startPostgresql);
+        if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) startFuncs.push(startMongodb);
+        if (infra.images.mail.tag !== existingInfra.images.mail.tag) startFuncs.push(startMail);
+
+        debug('startAddons: existing infra. incremental addon create %j', startFuncs.map(function (f) { return f.name; }));
+    }
+
+    async.series(startFuncs, callback);
+}
+
+function startApps(existingInfra, callback) {
+    if (existingInfra.version === infra.version) {
+        debug('startApp: apps are already uptodate');
+        callback();
+    } else if (existingInfra.version === 'none') {
+        debug('startApps: restoring installed apps');
+        apps.restoreInstalledApps(callback);
+    } else {
+        debug('startApps: reconfiguring installed apps');
+        apps.configureInstalledApps(callback);
+    }
+}
+
+function loadAddonVars(callback) {
+    gAddonVars = {
+        mail: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mail_vars.sh', 'utf8')),
+        postgresql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/postgresql_vars.sh', 'utf8')),
+        mysql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mysql_vars.sh', 'utf8')),
+        mongodb: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mongodb_vars.sh', 'utf8'))
+    };
+    callback();
+}
+
+function mailConfig() {
+    if (!gAddonVars) return { username: 'no-reply', from: 'no-reply@' + config.fqdn(), password: 'doesnotwork' }; // for tests which don't run infra
+
+    return {
+        username: gAddonVars.mail.MAIL_ROOT_USERNAME,
+        from: '"Cloudron" <' + gAddonVars.mail.MAIL_ROOT_USERNAME + '@' + config.fqdn() + '>',
+        password: gAddonVars.mail.MAIL_ROOT_PASSWORD
+    };
+}

src/progress.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -8,7 +6,8 @@ exports = module.exports = {
     get: get,
 
     UPDATE: 'update',
-    BACKUP: 'backup'
+    BACKUP: 'backup',
+    MIGRATE: 'migrate'
 };
 
 var assert = require('assert'),
@@ -18,12 +17,13 @@ var assert = require('assert'),
 // otherwise no such operation is currently ongoing
 var progress = {
     update: null,
-    backup: null
+    backup: null,
+    migrate: null
 };
 
 // We use -1 for percentage to indicate errors
 function set(tag, percent, message) {
-    assert(tag === exports.UPDATE || tag === exports.BACKUP);
+    assert.strictEqual(typeof tag, 'string');
     assert.strictEqual(typeof percent, 'number');
     assert.strictEqual(typeof message, 'string');
 
@@ -36,7 +36,7 @@ function set(tag, percent, message) {
 }
 
 function clear(tag) {
-    assert(tag === exports.UPDATE || tag === exports.BACKUP);
+    assert.strictEqual(typeof tag, 'string');
 
     progress[tag] = null;
 

src/routes/apps.js

@@ -16,7 +16,9 @@ exports = module.exports = {
 
     stopApp: stopApp,
     startApp: startApp,
-    exec: exec
+    exec: exec,
+
+    cloneApp: cloneApp
 };
 
 var apps = require('../apps.js'),
@@ -28,8 +30,7 @@ var apps = require('../apps.js'),
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     paths = require('../paths.js'),
     safe = require('safetydance'),
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');
 
 function auditSource(req) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -52,7 +53,8 @@ function removeInternalAppFields(app) {
         iconUrl: app.iconUrl,
         fqdn: app.fqdn,
         memoryLimit: app.memoryLimit,
-        altDomain: app.altDomain
+        altDomain: app.altDomain,
+        xFrameOptions: app.xFrameOptions
     };
 }
 
@@ -90,82 +92,77 @@ function getAppIcon(req, res, next) {
     });
 }
 
-/*
- * Installs an app
- * @bodyparam {string} appStoreId The id of the app to be installed
- * @bodyparam {manifest} manifest The app manifest
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from environment variable name to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- * @bodyparam {icon} icon Base64 encoded image
- */
 function installApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId is required'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    // required
     if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
+
+    // optional
+    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+
     if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
+
+    // falsy value in altDomain unsets it
     if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
 
-    // allow tests to provide an appId for testing
-    var appId = (process.env.BOX_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
+    if (data.xFrameOptions && typeof data.xFrameOptions !== 'string') return next(new HttpError(400, 'xFrameOptions must be a string'));
 
-    debug('Installing app id:%s storeid:%s loc:%s port:%j accessRestriction:%j memoryLimit:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.memoryLimit, data.manifest);
+    debug('Installing app :%j', data);
 
-    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.icon || null, data.cert || null, data.key || null, data.memoryLimit || 0, data.altDomain || null, auditSource(req), function (error) {
+    apps.install(data, auditSource(req), function (error, app) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BILLING_REQUIRED) return next(new HttpError(402, 'Billing required'));
-        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_CERTIFICATE) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.USER_REQUIRED) return next(new HttpError(400, 'accessRestriction must specify one user'));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(202, { id: appId } ));
+        next(new HttpSuccess(202, app));
     });
 }
 
-/*
- * Configure an app
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from env to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- */
 function configureApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
-    if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+    if ('location' in data && typeof data.location !== 'string') return next(new HttpError(400, 'location must be string'));
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    if ('accessRestriction' in data && typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction must be an object'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+
     if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
     if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
+    if (data.xFrameOptions && typeof data.xFrameOptions !== 'string') return next(new HttpError(400, 'xFrameOptions must be a string'));
 
-    debug('Configuring app id:%s location:%s bindings:%j accessRestriction:%j memoryLimit:%s', req.params.id, data.location, data.portBindings, data.accessRestriction, data.memoryLimit);
+    debug('Configuring app id:%s data:%j', req.params.id, data);
 
-    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.cert || null, data.key || null, data.memoryLimit || 0, data.altDomain || null, auditSource(req), function (error) {
+    apps.configure(req.params.id, data, auditSource(req), function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
@@ -180,20 +177,50 @@ function configureApp(req, res, next) {
 }
 
 function restoreApp(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
 
+    var data = req.body;
+
     debug('Restore app id:%s', req.params.id);
 
-    apps.restore(req.params.id, auditSource(req), function (error) {
+    if (!('backupId' in req.body)) return next(new HttpError(400, 'backupId is required'));
+    if (data.backupId !== null && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
+
+    apps.restore(req.params.id, data, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(424, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, { }));
     });
 }
 
+function cloneApp(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    var data = req.body;
+
+    debug('Clone app id:%s', req.params.id);
+
+    if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be a string'));
+    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
+    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+
+    apps.clone(req.params.id, data, auditSource(req), function (error, result) {
+        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
+        if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(424, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(201, { id: result.id }));
+    });
+}
+
 function backupApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
@@ -209,10 +236,6 @@ function backupApp(req, res, next) {
     });
 }
 
-/*
- * Uninstalls an app
- * @bodyparam {string} id The id of the app to be uninstalled
- */
 function uninstallApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
@@ -260,15 +283,18 @@ function updateApp(req, res, next) {
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
     debug('Update app id:%s to manifest:%j with portBindings:%j', req.params.id, data.manifest, data.portBindings);
 
-    apps.update(req.params.id, data.force || false, data.manifest, data.portBindings || null, data.icon, auditSource(req), function (error) {
+    apps.update(req.params.id, req.body, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
@@ -338,15 +364,37 @@ function getLogs(req, res, next) {
     });
 }
 
+function demuxStream(stream, stdin) {
+    var header = null;
+
+    stream.on('readable', function() {
+        header = header || stream.read(4);
+
+        while (header !== null) {
+            var length = header.readUInt32BE(0);
+            if (length === 0) {
+                header = null;
+                return stdin.end(); // EOF
+            }
+
+            var payload = stream.read(length);
+
+            if (payload === null) break;
+            stdin.write(payload);
+            header = stream.read(4);
+        }
+    });
+}
+
 function exec(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
-    debug('Execing into app id:%s', req.params.id);
+    debug('Execing into app id:%s and cmd:%s', req.params.id, req.query.cmd);
 
     var cmd = null;
     if (req.query.cmd) {
         cmd = safe.JSON.parse(req.query.cmd);
-        if (!util.isArray(cmd) && cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
+        if (!util.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
     }
 
     var columns = req.query.columns ? parseInt(req.query.columns, 10) : null;
@@ -367,8 +415,16 @@ function exec(req, res, next) {
         req.clearTimeout();
         res.sendUpgradeHandshake();
 
+        // When tty is disabled, the duplexStream has 2 separate streams. When enabled, it has stdout/stderr merged.
         duplexStream.pipe(res.socket);
-        res.socket.pipe(duplexStream);
+
+        if (tty) {
+            res.socket.pipe(duplexStream); // in tty mode, the client always waits for server to exit
+        } else {
+            demuxStream(res.socket, duplexStream);
+            res.socket.on('error', function () { duplexStream.end(); });
+            res.socket.on('end', function () { duplexStream.end(); });
+        }
     });
 }
 

src/routes/backups.js

@@ -3,7 +3,7 @@
 exports = module.exports = {
     get: get,
     create: create,
-    download: download
+    createDownloadUrl: createDownloadUrl
 };
 
 var assert = require('assert'),
@@ -43,7 +43,7 @@ function create(req, res, next) {
     });
 }
 
-function download(req, res, next) {
+function createDownloadUrl(req, res, next) {
     assert.strictEqual(typeof req.params.backupId, 'string');
 
     backups.getRestoreUrl(req.params.backupId, function (error, result) {

src/routes/clients.js

@@ -4,16 +4,16 @@ exports = module.exports = {
     add: add,
     get: get,
     del: del,
-    getAllByUserId: getAllByUserId,
+    getAll: getAll,
+    addClientToken: addClientToken,
     getClientTokens: getClientTokens,
-    delClientTokens: delClientTokens
+    delClientTokens: delClientTokens,
+    delToken: delToken
 };
 
 var assert = require('assert'),
-    clientdb = require('../clientdb.js'),
     clients = require('../clients.js'),
     ClientsError = clients.ClientsError,
-    DatabaseError = require('../databaseerror.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     validUrl = require('valid-url');
@@ -27,8 +27,9 @@ function add(req, res, next) {
     if (typeof data.scope !== 'string' || !data.scope) return next(new HttpError(400, 'scope is required'));
     if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));
 
-    clients.add(data.appId, clientdb.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
-        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, 'Invalid scope'));
+    clients.add(data.appId, clients.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
+        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, error.message));
+        if (error && error.reason === ClientsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(201, result));
     });
@@ -38,7 +39,7 @@ function get(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
 
     clients.get(req.params.clientId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'No such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, result));
     });
@@ -47,26 +48,49 @@ function get(req, res, next) {
 function del(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
 
-    clients.del(req.params.clientId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+    clients.get(req.params.clientId, function (error, result) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
-        next(new HttpSuccess(204, result));
+
+        // we do not allow to use the REST API to delete addon clients
+        if (result.type !== clients.TYPE_EXTERNAL) return next(new HttpError(405, 'Deleting app addon clients is not allowed.'));
+
+        clients.del(req.params.clientId, function (error, result) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+            if (error && error.reason === ClientsError.NOT_ALLOWED) return next(new HttpError(405, error.message));
+            if (error) return next(new HttpError(500, error));
+            next(new HttpSuccess(204, result));
+        });
     });
 }
 
-function getAllByUserId(req, res, next) {
-    clients.getAllWithDetailsByUserId(req.user.id, function (error, result) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return next(new HttpError(500, error));
+function getAll(req, res, next) {
+    clients.getAll(function (error, result) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { clients: result }));
     });
 }
 
+function addClientToken(req, res, next) {
+    assert.strictEqual(typeof req.params.clientId, 'string');
+    assert.strictEqual(typeof req.user, 'object');
+
+    var expiresAt = req.query.expiresAt ? parseInt(req.query.expiresAt, 10) : Date.now() + 24 * 60 * 60 * 1000; // default 1 day;
+    if (isNaN(expiresAt) || expiresAt <= Date.now()) return next(new HttpError(400, 'expiresAt must be a timestamp in the future'));
+
+    clients.addClientTokenByUserId(req.params.clientId, req.user.id, expiresAt, function (error, result) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+        next(new HttpSuccess(201, { token: result }));
+    });
+}
+
 function getClientTokens(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
     assert.strictEqual(typeof req.user, 'object');
 
     clients.getClientTokensByUserId(req.params.clientId, req.user.id, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { tokens: result }));
     });
@@ -77,8 +101,22 @@ function delClientTokens(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
     clients.delClientTokensByUserId(req.params.clientId, req.user.id, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(204));
     });
 }
+
+function delToken(req, res, next) {
+    assert.strictEqual(typeof req.params.clientId, 'string');
+    assert.strictEqual(typeof req.params.tokenId, 'string');
+    assert.strictEqual(typeof req.user, 'object');
+
+    clients.delToken(req.params.clientId, req.params.tokenId, function (error) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error && error.reason === ClientsError.INVALID_TOKEN) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}

src/routes/cloudron.js

@@ -5,13 +5,16 @@ exports = module.exports = {
     setupTokenAuth: setupTokenAuth,
     getStatus: getStatus,
     reboot: reboot,
+    migrate: migrate,
     getProgress: getProgress,
     getConfig: getConfig,
     update: update,
-    feedback: feedback
+    feedback: feedback,
+    checkForUpdates: checkForUpdates
 };
 
 var assert = require('assert'),
+    async = require('async'),
     cloudron = require('../cloudron.js'),
     CloudronError = cloudron.CloudronError,
     config = require('../config.js'),
@@ -20,7 +23,9 @@ var assert = require('assert'),
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     progress = require('../progress.js'),
     mailer = require('../mailer.js'),
-    superagent = require('superagent');
+    superagent = require('superagent'),
+    updateChecker = require('../updatechecker.js'),
+    _ = require('underscore');
 
 function auditSource(req) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -55,9 +60,7 @@ function activate(req, res, next) {
 
     cloudron.activate(username, password, email, displayName, ip, auditSource(req), function (error, info) {
         if (error && error.reason === CloudronError.ALREADY_PROVISIONED) return next(new HttpError(409, 'Already setup'));
-        if (error && error.reason === CloudronError.BAD_USERNAME) return next(new HttpError(400, 'Bad username'));
-        if (error && error.reason === CloudronError.BAD_PASSWORD) return next(new HttpError(400, 'Bad password'));
-        if (error && error.reason === CloudronError.BAD_EMAIL) return next(new HttpError(400, 'Bad email'));
+        if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         // only in caas case do we have to notify the api server about activation
@@ -112,6 +115,31 @@ function reboot(req, res, next) {
     cloudron.reboot();
 }
 
+function migrate(req, res, next) {
+    if (config.provider() !== 'caas') return next(new HttpError(422, 'Cannot use migrate API with this provider'));
+
+    if ('size' in req.body && typeof req.body.size !== 'string') return next(new HttpError(400, 'size must be string'));
+    if ('region' in req.body && typeof req.body.region !== 'string') return next(new HttpError(400, 'region must be string'));
+
+    if ('domain' in req.body) {
+        if (typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be string'));
+        if (typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider must be string'));
+    }
+
+    debug('Migration requested domain:%s size:%s region:%s', req.body.domain, req.body.size, req.body.region);
+
+    var options = _.pick(req.body, 'domain', 'size', 'region');
+    if (Object.keys(options).length === 0) return next(new HttpError(400, 'no migrate option provided'));
+
+    cloudron.migrate(req.body, function (error) { // pass req.body because 'domain' can have arbitrary options
+        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
 function getConfig(req, res, next) {
     cloudron.getConfig(function (error, cloudronConfig) {
         if (error) return next(new HttpError(500, error));
@@ -125,12 +153,22 @@ function update(req, res, next) {
     cloudron.updateToLatest(auditSource(req), function (error) {
         if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
         if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === CloudronError.SELF_UPGRADE_NOT_SUPPORTED) return next(new HttpError(412, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
     });
 }
 
+function checkForUpdates(req, res, next) {
+    async.series([
+        updateChecker.checkAppUpdates,
+        updateChecker.checkBoxUpdates
+    ], function () {
+        next(new HttpSuccess(200, { update: updateChecker.getUpdateInfo() }));
+    });
+}
+
 function feedback(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 

src/routes/developer.js

@@ -19,7 +19,7 @@ function auditSource(req) {
 }
 
 function enabled(req, res, next) {
-    developer.enabled(function (error, enabled) {
+    developer.isEnabled(function (error, enabled) {
         if (enabled) return next();
         next(new HttpError(412, 'Developer mode not enabled'));
     });

src/routes/groups.js

@@ -20,7 +20,7 @@ function create(req, res, next) {
     if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
 
     groups.create(req.body.name, function (error, group) {
-        if (error && error.reason === GroupError.BAD_NAME) return next(new HttpError(400, error.message));
+        if (error && error.reason === GroupError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === GroupError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
         if (error) return next(new HttpError(500, error));
 
@@ -45,7 +45,7 @@ function get(req, res, next) {
 }
 
 function list(req, res, next) {
-    groups.getAll(function (error, result) {
+    groups.getAllWithMembers(function (error, result) {
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(200, { groups: result }));

src/routes/index.js

@@ -9,9 +9,10 @@ exports = module.exports = {
     eventlog: require('./eventlog.js'),
     graphs: require('./graphs.js'),
     groups: require('./groups.js'),
+    mailboxes: require('./mailboxes.js'),
     oauth2: require('./oauth2.js'),
     profile: require('./profile.js'),
-    settings: require('./settings.js'),
     sysadmin: require('./sysadmin.js'),
+    settings: require('./settings.js'),
     user: require('./user.js')
 };

src/routes/mailboxes.js

@@ -0,0 +1,91 @@
+'use strict';
+
+exports = module.exports = {
+    list: list,
+    get: get,
+    remove: remove,
+    create: create,
+    setAliases: setAliases,
+    getAliases: getAliases
+};
+
+var assert = require('assert'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    mailboxes = require('../mailboxes.js'),
+    MailboxError = mailboxes.MailboxError,
+    util = require('util');
+
+function create(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+
+    mailboxes.add(req.body.name, function (error, mailbox) {
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(201, mailbox));
+    });
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.get(req.params.mailboxId, function (error, result) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function list(req, res, next) {
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { mailboxes: result }));
+    });
+}
+
+function remove(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.del(req.params.mailboxId, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'Mailbox not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function setAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    if (!util.isArray(req.body.aliases)) return next(new HttpError(400, 'aliases must be an array'));
+
+    for (var i = 0; i < req.body.aliases.length; i++) {
+        if (typeof req.body.aliases[i] !== 'string') return next(new HttpError(400, 'alias must be a string'));
+    }
+
+    mailboxes.setAliases(req.params.mailboxId, req.body.aliases, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'One or more alias already exist'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
+function getAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.getAliases(req.params.mailboxId, function (error, aliases) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { aliases: aliases }));
+    });
+}

src/routes/oauth2.js

@@ -4,9 +4,9 @@ var appdb = require('../appdb'),
     apps = require('../apps'),
     assert = require('assert'),
     authcodedb = require('../authcodedb'),
-    clientdb = require('../clientdb'),
+    clients = require('../clients'),
+    ClientsError = clients.ClientsError,
     config = require('../config.js'),
-    constants = require('../constants.js'),
     DatabaseError = require('../databaseerror'),
     debug = require('debug')('box:routes/oauth2'),
     eventlog = require('../eventlog.js'),
@@ -21,7 +21,8 @@ var appdb = require('../appdb'),
     url = require('url'),
     user = require('../user.js'),
     UserError = user.UserError,
-    util = require('util');
+    util = require('util'),
+    _ = require('underscore');
 
 function auditSource(req, appId) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -41,7 +42,7 @@ gServer.serializeClient(function (client, callback) {
 });
 
 gServer.deserializeClient(function (id, callback) {
-    clientdb.get(id, callback);
+    clients.get(id, callback);
 });
 
 
@@ -76,7 +77,7 @@ gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client,
     var token = tokendb.generateToken();
     var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-    tokendb.add(token, tokendb.PREFIX_USER + user.id, client.id, expires, client.scope, function (error) {
+    tokendb.add(token, user.id, client.id, expires, client.scope, function (error) {
         if (error) return callback(error);
 
         debug('grant token: new access token for client %s token %s', client.id, token);
@@ -106,7 +107,7 @@ gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI,
             var token = tokendb.generateToken();
             var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-            tokendb.add(token, tokendb.PREFIX_USER + authCode.userId, authCode.clientId, expires, client.scope, function (error) {
+            tokendb.add(token, authCode.userId, authCode.clientId, expires, client.scope, function (error) {
                 if (error) return callback(error);
 
                 debug('exchange: new access token for client %s token %s', client.id, token);
@@ -201,13 +202,13 @@ function loginForm(req, res) {
         });
     }
 
-    clientdb.get(u.query.client_id, function (error, result) {
+    clients.get(u.query.client_id, function (error, result) {
         if (error) return sendError(req, res, 'Unknown OAuth client');
 
         switch (result.type) {
-            case clientdb.TYPE_ADMIN: return render(constants.ADMIN_NAME, '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_EXTERNAL: return render('External Application', '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
+            case clients.TYPE_BUILT_IN: return render(result.appId, '/api/v1/cloudron/avatar');
+            case clients.TYPE_EXTERNAL: return render(result.appId, '/api/v1/cloudron/avatar');
+            case clients.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
             default: break;
         }
 
@@ -306,16 +307,20 @@ function accountSetup(req, res, next) {
     user.getByResetToken(req.body.resetToken, function (error, userObject) {
         if (error) return sendError(req, res, 'Invalid Reset Token');
 
-        userObject.username = req.body.username;
-        userObject.displayName = req.body.displayName;
-
-        user.update(userObject.id, userObject.username, userObject.email, userObject.displayName, auditSource(req), function (error) {
+        var data = _.pick(req.body, 'username', 'displayName');
+        user.update(userObject.id, data, auditSource(req), function (error) {
             if (error && error.reason === UserError.ALREADY_EXISTS) return renderAccountSetupSite(res, req, userObject, 'Username already exists');
+            if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+            if (error && error.reason === UserError.NOT_FOUND) return renderAccountSetupSite(res, req, userObject, 'No such user');
             if (error) return next(new HttpError(500, error));
 
+            userObject.username = req.body.username;
+            userObject.displayName = req.body.displayName;
+
             // setPassword clears the resetToken
             user.setPassword(userObject.id, req.body.password, function (error, result) {
-                if (error && error.reason === UserError.BAD_PASSWORD) return renderAccountSetupSite(res, req, userObject, 'Password invalid');
+                if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+
                 if (error) return next(new HttpError(500, error));
 
                 res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
@@ -357,7 +362,7 @@ function passwordReset(req, res, next) {
 
         // setPassword clears the resetToken
         user.setPassword(userObject.id, req.body.password, function (error, result) {
-            if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(406, 'Password does not meet the requirements'));
+            if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(406, error.message));
             if (error) return next(new HttpError(500, error));
 
             res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
@@ -399,8 +404,8 @@ var authorization = [
     gServer.authorization({}, function (clientId, redirectURI, callback) {
         debug('authorization: client %s with callback to %s.', clientId, redirectURI);
 
-        clientdb.get(clientId, function (error, client) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+        clients.get(clientId, function (error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
             if (error) return callback(error);
 
             // ignore the origin passed into form the client, but use the one from the clientdb
@@ -414,12 +419,12 @@ var authorization = [
         // Handle our different types of oauth clients
         var type = req.oauth2.client.type;
 
-        if (type === clientdb.TYPE_ADMIN) {
-            eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, 'admin'), { userId: req.oauth2.user.id });
+        if (type === clients.TYPE_EXTERNAL || type === clients.TYPE_BUILT_IN) {
+            eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, req.oauth2.client.appId), { userId: req.oauth2.user.id });
             return next();
+        } else if (type === clients.TYPE_SIMPLE_AUTH) {
+            return sendError(req, res, 'Unknown OAuth client.');
         }
-        if (type === clientdb.TYPE_EXTERNAL) return next();
-        if (type === clientdb.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unknown OAuth client.');
 
         appdb.get(req.oauth2.client.appId, function (error, appObject) {
             if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
@@ -451,6 +456,31 @@ var token = [
     gServer.errorHandler()
 ];
 
+// tests if all requestedScopes are attached to the request
+function validateRequestedScopes(req, requestedScopes) {
+    assert.strictEqual(typeof req, 'object');
+    assert(Array.isArray(requestedScopes));
+
+    if (!req.authInfo || !req.authInfo.scope) return new Error('No scope found');
+
+    var scopes = req.authInfo.scope.split(',');
+
+    // check for roles separately
+    if (requestedScopes.indexOf(clients.SCOPE_ROLE_SDK) !== -1 && scopes.indexOf(clients.SCOPE_ROLE_SDK) === -1) {
+        return new Error('Missing required scope role "' + clients.SCOPE_ROLE_SDK + '"');
+    }
+
+    if (scopes.indexOf('*') !== -1) return null;
+
+    for (var i = 0; i < requestedScopes.length; ++i) {
+        if (scopes.indexOf(requestedScopes[i]) === -1) {
+            debug('scope: missing scope "%s".', requestedScopes[i]);
+            return new Error('Missing required scope "' + requestedScopes[i] + '"');
+        }
+    }
+
+    return null;
+}
 
 //  The scope middleware provides an auth middleware for routes.
 //
@@ -470,17 +500,8 @@ function scope(requestedScope) {
     return [
         passport.authenticate(['bearer'], { session: false }),
         function (req, res, next) {
-            if (!req.authInfo || !req.authInfo.scope) return next(new HttpError(401, 'No scope found'));
-            if (req.authInfo.scope === '*') return next();
-
-            var scopes = req.authInfo.scope.split(',');
-
-            for (var i = 0; i < requestedScopes.length; ++i) {
-                if (scopes.indexOf(requestedScopes[i]) === -1) {
-                    debug('scope: missing scope "%s".', requestedScopes[i]);
-                    return next(new HttpError(401, 'Missing required scope "' + requestedScopes[i] + '"'));
-                }
-            }
+            var error = validateRequestedScopes(req, requestedScopes);
+            if (error) return next(new HttpError(401, error.message));
 
             next();
         }
@@ -511,6 +532,7 @@ exports = module.exports = {
     accountSetup: accountSetup,
     authorization: authorization,
     token: token,
+    validateRequestedScopes: validateRequestedScopes,
     scope: scope,
     csrf: csrf
 };

src/routes/profile.js

@@ -8,12 +8,11 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
-    groups = require('../groups.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     user = require('../user.js'),
-    tokendb = require('../tokendb.js'),
-    UserError = user.UserError;
+    UserError = user.UserError,
+    _ = require('underscore');
 
 function auditSource(req) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -23,26 +22,14 @@ function auditSource(req) {
 function get(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    var result = {};
-    result.id = req.user.id;
-    result.tokenType = req.user.tokenType;
-
-    if (req.user.tokenType === tokendb.TYPE_USER || req.user.tokenType === tokendb.TYPE_DEV) {
-        result.username = req.user.username;
-        result.email = req.user.email;
-        result.displayName = req.user.displayName;
-        result.showTutorial = req.user.showTutorial;
-
-        groups.isMember(groups.ADMIN_GROUP_ID, req.user.id, function (error, isAdmin) {
-            if (error) return next(new HttpError(500, error));
-
-            result.admin = isAdmin;
-
-            next(new HttpSuccess(200, result));
-        });
-    } else {
-        next(new HttpSuccess(200, result));
-    }
+    next(new HttpSuccess(200, {
+        id: req.user.id,
+        username: req.user.username,
+        email: req.user.email,
+        admin: req.user.admin,
+        displayName: req.user.displayName,
+        showTutorial: req.user.showTutorial
+    }));
 }
 
 function update(req, res, next) {
@@ -52,12 +39,11 @@ function update(req, res, next) {
     if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
     if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
 
-    if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
+    var data = _.pick(req.body, 'email', 'displayName');
 
-    user.update(req.user.id, req.user.username, req.body.email || req.user.email, req.body.displayName || req.user.displayName, auditSource(req), function (error) {
-        if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, error.message));
-        if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, error.message));
-        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+    user.update(req.user.id, data, auditSource(req), function (error) {
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
         if (error) return next(new HttpError(500, error));
 
@@ -69,13 +55,10 @@ function changePassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.user, 'object');
 
-    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'API call requires the users old password.'));
-    if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'API call requires the users new password.'));
-
-    if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
+    if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'newPassword must be a string'));
 
     user.setPassword(req.user.id, req.body.newPassword, function (error) {
-        if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Wrong password'));
         if (error) return next(new HttpError(500, error));
 

src/routes/settings.js

@@ -17,6 +17,10 @@ exports = module.exports = {
     setBackupConfig: setBackupConfig,
 
     getTimeZone: getTimeZone,
+    setTimeZone: setTimeZone,
+
+    getAppstoreConfig: getAppstoreConfig,
+    setAppstoreConfig: setAppstoreConfig,
 
     setCertificate: setCertificate,
     setAdminCertificate: setAdminCertificate
@@ -45,7 +49,7 @@ function setAutoupdatePattern(req, res, next) {
     if (typeof req.body.pattern !== 'string') return next(new HttpError(400, 'pattern is required'));
 
     settings.setAutoupdatePattern(req.body.pattern, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, 'Invalid pattern'));
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(200));
@@ -58,15 +62,17 @@ function setCloudronName(req, res, next) {
     if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name is required'));
 
     settings.setCloudronName(req.body.name, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, 'Invalid name'));
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
-        next(new HttpSuccess(200));
+
+        next(new HttpSuccess(202));
     });
 }
 
 function getCloudronName(req, res, next) {
     settings.getCloudronName(function (error, name) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200, { name: name }));
     });
 }
@@ -74,10 +80,24 @@ function getCloudronName(req, res, next) {
 function getTimeZone(req, res, next) {
     settings.getTimeZone(function (error, tz) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200, { timeZone: tz }));
     });
 }
 
+function setTimeZone(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.timeZone !== 'string') return next(new HttpError(400, 'timeZone is required'));
+
+    settings.setTimeZone(req.body.timeZone, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
 function setCloudronAvatar(req, res, next) {
     assert.strictEqual(typeof req.files, 'object');
 
@@ -86,6 +106,7 @@ function setCloudronAvatar(req, res, next) {
 
     settings.setCloudronAvatar(avatar, function (error) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(202, {}));
     });
 }
@@ -144,6 +165,33 @@ function setBackupConfig(req, res, next) {
     });
 }
 
+function getAppstoreConfig(req, res, next) {
+    settings.getAppstoreConfig(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function setAppstoreConfig(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.userId !== 'string') return next(new HttpError(400, 'userId is required'));
+    if (typeof req.body.token !== 'string') return next(new HttpError(400, 'token is required'));
+
+    var options = {
+        userId: req.body.userId,
+        token: req.body.token
+    };
+
+    settings.setAppstoreConfig(options, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
 // default fallback cert
 function setCertificate(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');

src/routes/sysadmin.js

@@ -36,6 +36,7 @@ function update(req, res, next) {
     cloudron.updateToLatest(auditSource, function (error) {
         if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
         if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === CloudronError.SELF_UPGRADE_NOT_SUPPORTED) return next(new HttpError(412, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
@@ -45,12 +46,9 @@ function update(req, res, next) {
 function retire(req, res, next) {
     debug('triggering retire');
 
-    // note that cloudron.backup only waits for backup initiation and not for backup to complete
-    // backup progress can be checked up ny polling the progress api call
-    cloudron.retire(function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(202, {}));
+    cloudron.retire('migrate', { }, function (error) {
+        if (error) console.error('Retire failed.', error);
     });
+
+    next(new HttpSuccess(202, {}));
 }

src/routes/test/apps-test.js

@@ -1,7 +1,7 @@
 'use strict';
 
-/* jslint node:true */
 /* global it:false */
+/* global xit:false */
 /* global describe:false */
 /* global before:false */
 /* global after:false */
@@ -12,7 +12,7 @@ var appdb = require('../../appdb.js'),
     path = require('path'),
     async = require('async'),
     child_process = require('child_process'),
-    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     config = require('../../config.js'),
     constants = require('../../constants.js'),
     database = require('../../database.js'),
@@ -23,18 +23,18 @@ var appdb = require('../../appdb.js'),
     http = require('http'),
     https = require('https'),
     js2xml = require('js2xmlparser'),
+    ldap = require('../../ldap.js'),
     net = require('net'),
     nock = require('nock'),
     paths = require('../../paths.js'),
-    redis = require('redis'),
-    superagent = require('superagent'),
     safe = require('safetydance'),
     server = require('../../server.js'),
     settings = require('../../settings.js'),
+    simpleauth = require('../../simpleauth.js'),
+    superagent = require('superagent'),
     taskmanager = require('../../taskmanager.js'),
     tokendb = require('../../tokendb.js'),
     url = require('url'),
-    util = require('util'),
     uuid = require('node-uuid'),
     _ = require('underscore');
 
@@ -42,9 +42,9 @@ var SERVER_URL = 'http://localhost:' + config.get('port');
 
 // Test image information
 var TEST_IMAGE_REPO = 'cloudron/test';
-var TEST_IMAGE_TAG = '11.0.0';
+var TEST_IMAGE_TAG = '16.0.0';
 var TEST_IMAGE = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
-var TEST_IMAGE_ID = child_process.execSync('docker inspect --format={{.Id}} ' + TEST_IMAGE).toString('utf8').trim();
+// var TEST_IMAGE_ID = child_process.execSync('docker inspect --format={{.Id}} ' + TEST_IMAGE).toString('utf8').trim();
 
 var APP_STORE_ID = 'test', APP_ID;
 var APP_LOCATION = 'appslocation';
@@ -59,7 +59,7 @@ APP_MANIFEST_1.dockerImage = TEST_IMAGE;
 APP_MANIFEST_1.singleUser = true;
 
 var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='admin@me.com';
-var USER_1_ID = null, USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1338', EMAIL_1 ='user@me.com';
+var USER_1_ID = null, USERNAME_1 = 'user', EMAIL_1 ='user@me.com';
 var token = null; // authentication token
 var token_1 = null;
 
@@ -104,52 +104,74 @@ function startDockerProxy(interceptor, callback) {
     }).listen(5687, callback);
 }
 
+function checkAddons(appEntry, done) {
+    async.retry({ times: 15, interval: 3000 }, function (callback) {
+        // this was previously written with superagent but it was getting sporadic EPIPE
+        var req = http.get({ hostname: 'localhost', port: appEntry.httpPort, path: '/check_addons?username=' + USERNAME + '&password=' + PASSWORD });
+        req.on('error', callback);
+        req.on('response', function (res) {
+            if (res.statusCode !== 200) return callback('app returned non-200 status : ' + res.statusCode);
+
+            var d = '';
+            res.on('data', function (chunk) { d += chunk.toString('utf8'); });
+            res.on('end', function () {
+                var body = JSON.parse(d);
+
+                delete body.recvmail; // unclear why dovecot mail delivery won't work
+                delete body.stdenv; // cannot access APP_ORIGIN
+
+                for (var key in body) {
+                    if (body[key] !== 'OK') return callback('Not done yet: ' + JSON.stringify(body));
+                }
+
+                callback();
+            });
+        });
+
+        req.end();
+    }, done);
+}
+
+function checkRedis(containerId, done) {
+    var redisIp, exportedRedisPort;
+
+    docker.getContainer(containerId).inspect(function (error, data) {
+        expect(error).to.not.be.ok();
+        expect(data).to.be.ok();
+
+        redisIp = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
+        expect(redisIp).to.be.ok();
+
+        exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp');
+        expect(exportedRedisPort).to.be(null);
+
+        done();
+    });
+}
+
 describe('Apps', function () {
-    this.timeout(50000);
+    this.timeout(100000);
 
     var dockerProxy;
     var imageDeleted = false;
     var imageCreated = false;
 
     before(function (done) {
-        console.log('Starting addons, this can take 10 seconds');
-
-        dockerProxy = startDockerProxy(function interceptor(req, res) {
-            if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
-                imageCreated = true;
-                res.writeHead(200);
-                res.end();
-                return true;
-            } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE + '?force=false&noprune=false') {
-                imageDeleted = true;
-                res.writeHead(200);
-                res.end();
-                return true;
-            }
-            return false;
-        }, done);
-    });
-
-    after(function (done) {
-        // child_process.exec('docker rm -f mysql; docker rm -f postgresql; docker rm -f mongodb; docker rm -f mail', function (error) {
-        child_process.exec('docker ps | awk \'{print $1}\' | xargs docker rm -f', function () {
-            dockerProxy.close(done);
-        });
-    });
-
-
-    /*
-        Individual sub category setup and cleanup
-    */
-    function setup(done) {
         config._reset();
 
+        process.env.CREATE_INFRA = 1;
+
+        safe.fs.unlinkSync(paths.INFRA_VERSION_FILE);
+        child_process.execSync('docker ps -qa | xargs --no-run-if-empty docker rm -f');
+
         async.series([
             // first clear, then start server. otherwise, taskmanager spins up tasks for obsolete appIds
             database.initialize,
             database._clear,
 
             server.start.bind(server),
+            ldap.start,
+            simpleauth.start,
 
             function (callback) {
                 var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
@@ -188,58 +210,104 @@ describe('Apps', function () {
                 token_1 = tokendb.generateToken();
 
                 // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                tokendb.add(token_1, tokendb.PREFIX_USER + USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+                tokendb.add(token_1, USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+            },
+
+            function (callback) {
+                dockerProxy = startDockerProxy(function interceptor(req, res) {
+                    if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
+                        imageCreated = true;
+                        res.writeHead(200);
+                        res.end();
+                        return true;
+                    } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE + '?force=false&noprune=false') {
+                        imageDeleted = true;
+                        res.writeHead(200);
+                        res.end();
+                        return true;
+                    }
+                    return false;
+                }, callback);
             },
 
             settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
             settings.setTlsConfig.bind(null, { provider: 'caas' }),
             settings.setBackupConfig.bind(null, { provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
-        ], done);
-    }
+        ], function (error) {
+            if (error) return done(error);
+
+            console.log('This test can take ~40 seconds to start as it waits for infra to be ready');
+            setTimeout(done, 40000);
+        });
+    });
+
+    after(function (done) {
+        delete process.env.CREATE_INFRA;
+
+        // child_process.execSync('docker ps -qa | xargs --no-run-if-empty docker rm -f');
+        dockerProxy.close(function () { });
 
-    function cleanup(done) {
         // db is not cleaned up here since it's too late to call it after server.stop. if called before server.stop taskmanager apptasks are unhappy :/
         async.series([
             taskmanager.stopPendingTasks,
             taskmanager.waitForPendingTasks,
             server.stop,
+            ldap.stop,
+            simpleauth.stop,
             config._reset,
         ], done);
-    }
+    });
 
     describe('App API', function () {
-        this.timeout(50000);
-
-        before(setup);
-
         after(function (done) {
-            APP_ID = null;
-            cleanup(done);
+            appdb._clear(done); // TODO: test proper uninstall (requires mock for aws)
         });
 
         it('app install fails - missing manifest', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD })
+                   .send({ password: PASSWORD })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
-                expect(res.body.message).to.eql('manifest is required');
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
                 done();
             });
         });
 
-        it('app install fails - missing appId', function (done) {
+        it('app install fails - null manifest', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ manifest: APP_MANIFEST, password: PASSWORD })
+                   .send({ manifest: null, password: PASSWORD })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
-                expect(res.body.message).to.eql('appStoreId is required');
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
                 done();
             });
         });
 
-        it('app install fails - invalid json', function (done) {
+        it('app install fails - bad manifest format', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ manifest: 'epic', password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(res.body.message).to.eql('manifest must be an object');
+                done();
+            });
+        });
+
+        it('app install fails - empty appStoreId format', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ manifest: null, appStoreId: '', password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
+                done();
+            });
+        });
+
+       it('app install fails - invalid json', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
                    .send('garbage')
@@ -252,7 +320,7 @@ describe('Apps', function () {
         it('app install fails - invalid location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('Hostname can only contain alphanumerics and hyphen');
@@ -263,7 +331,7 @@ describe('Apps', function () {
         it('app install fails - invalid location type', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('location is required');
@@ -274,7 +342,7 @@ describe('Apps', function () {
         it('app install fails - reserved admin location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql(constants.ADMIN_LOCATION + ' is reserved');
@@ -285,7 +353,7 @@ describe('Apps', function () {
         it('app install fails - reserved api location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql(constants.API_LOCATION + ' is reserved');
@@ -296,7 +364,7 @@ describe('Apps', function () {
         it('app install fails - portBindings must be object', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('portBindings must be an object');
@@ -307,7 +375,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction is required', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {} })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {} })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction is required');
@@ -318,7 +386,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction type is wrong', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: '' })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: '' })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction is required');
@@ -329,7 +397,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction no users not allowed', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction must specify one user');
@@ -340,7 +408,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction too many users not allowed', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: { users: [ 'one', 'two' ] } })
+                   .send({ manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: { users: [ 'one', 'two' ] } })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction must specify one user');
@@ -351,50 +419,64 @@ describe('Apps', function () {
         it('app install fails for non admin', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token_1 })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(403);
                 done();
             });
         });
 
-        it('app install fails due to purchase failure', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(402, {});
+        it('app install fails because manifest download fails', function (done) {
+            var fake = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(404, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(fake.isDone()).to.be.ok();
+                done();
+            });
+        });
+
+        it('app install fails due to purchase failure', function (done) {
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(402, {});
+
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(402);
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 done();
             });
         });
 
         it('app install succeeds with purchase', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 expect(res.body.id).to.be.a('string');
                 APP_ID = res.body.id;
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 done();
             });
         });
 
         it('app install fails because of conflicting location', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
-
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(409);
-                expect(fake.isDone()).to.be.ok();
                 done();
             });
         });
@@ -492,23 +574,23 @@ describe('Apps', function () {
         });
 
         it('app install succeeds already purchased', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: null })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 expect(res.body.id).to.be.a('string');
                 APP_ID = res.body.id;
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 done();
             });
         });
 
         it('app install succeeds without password but developer token', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
-
             settings.setDeveloperMode(true, function (error) {
                 expect(error).to.be(null);
 
@@ -517,7 +599,7 @@ describe('Apps', function () {
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(200);
-                    expect(result.body.expiresAt).to.be.a('number');
+                    expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                     expect(result.body.token).to.be.a('string');
 
                     // overwrite non dev token
@@ -525,11 +607,10 @@ describe('Apps', function () {
 
                     superagent.post(SERVER_URL + '/api/v1/apps/install')
                            .query({ access_token: token })
-                           .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: null })
+                           .send({ manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: null })
                            .end(function (err, res) {
                         expect(res.statusCode).to.equal(202);
                         expect(res.body.id).to.be.a('string');
-                        expect(fake.isDone()).to.be.ok();
                         APP_ID = res.body.id;
                         done();
                     });
@@ -560,8 +641,6 @@ describe('Apps', function () {
             imageCreated = false;
 
             async.series([
-                setup,
-
                 function (callback) {
                     apiHockInstance
                         .get('/api/v1/apps/' + APP_STORE_ID + '/versions/' + APP_MANIFEST.version + '/icon')
@@ -590,7 +669,6 @@ describe('Apps', function () {
             APP_ID = null;
 
             async.series([
-                cleanup,
                 apiHockServer.close.bind(apiHockServer),
                 awsHockServer.close.bind(awsHockServer)
             ], done);
@@ -599,7 +677,8 @@ describe('Apps', function () {
         var appResult = null /* the json response */, appEntry = null /* entry from database */;
 
         it('can install test app', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
 
             var count = 0;
             function checkInstallStatus() {
@@ -616,12 +695,13 @@ describe('Apps', function () {
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                  .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 expect(res.body.id).to.be.a('string');
-                expect(res.body.id).to.be.eql(APP_ID);
+                APP_ID = res.body.id;
                 checkInstallStatus();
             });
         });
@@ -650,15 +730,9 @@ describe('Apps', function () {
                 expect(data.Config.Env).to.contain('CLOUDRON=1');
                 expect(data.Config.Env).to.contain('APP_ORIGIN=https://' + config.appFqdn(APP_LOCATION));
                 expect(data.Config.Env).to.contain('APP_DOMAIN=' + config.appFqdn(APP_LOCATION));
-                expect(data.Config.Hostname).to.be(APP_LOCATION);
-                clientdb.getByAppIdAndType(appResult.id, clientdb.TYPE_OAUTH, function (error, client) {
-                    expect(error).to.not.be.ok();
-                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
-                    expect(client.clientSecret.length).to.be(64); // 32 hex chars (256 bits)
-                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_ID=' + client.id);
-                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_SECRET=' + client.clientSecret);
-                    done();
-                });
+                // Hostname must not be set of app fqdn or app location!
+                expect(data.Config.Hostname).to.not.contain(APP_LOCATION);
+                done();
             });
         });
 
@@ -704,144 +778,50 @@ describe('Apps', function () {
             });
         });
 
-        var redisIp, exportedRedisPort;
-
-        it('installation - redis addon created', function (done) {
-            docker.getContainer('redis-' + APP_ID).inspect(function (error, data) {
-                expect(error).to.not.be.ok();
-                expect(data).to.be.ok();
-
-                redisIp = safe.query(data, 'NetworkSettings.IPAddress');
-                expect(redisIp).to.be.ok();
-
-                exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
-                expect(exportedRedisPort).to.be.ok();
-
+        it('installation - app responnds to http request', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort).end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.status).to.be('OK');
                 done();
             });
         });
 
-        it('installation - redis addon config', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
-
-                var urlp = url.parse(redisUrl);
-                var password = urlp.auth.split(':')[1];
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                client.on('error', done);
-                client.set('key', 'value');
-                client.get('key', function (err, reply) {
-                    expect(err).to.not.be.ok();
-                    expect(reply.toString()).to.be('value');
-                    client.end();
-                    done();
-                });
-            });
-        });
-
-        it('installation - mysql addon config', function (done) {
+        it('installation - oauth addon config', function (done) {
             var appContainer = docker.getContainer(appEntry.containerId);
             appContainer.inspect(function (error, data) {
-                var mysqlUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('MYSQL_URL=') === 0) mysqlUrl = env.split('=')[1]; });
-                expect(mysqlUrl).to.be.ok();
+                expect(error).to.not.be.ok();
 
-                var urlp = url.parse(mysqlUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('MYSQL_PORT=3306');
-                expect(data.Config.Env).to.contain('MYSQL_HOST=mysql');
-                expect(data.Config.Env).to.contain('MYSQL_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('MYSQL_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('MYSQL_DATABASE=' + dbname);
-
-                var cmd = util.format('mysql -h %s -u%s -p%s --database=%s -e "CREATE TABLE IF NOT EXISTS foo (id INT);"',
-                    'mysql', username, password, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error, stdout, stderr) {
-                    expect(!error).to.be.ok();
-                    expect(stdout.length).to.be(0);
-                    // expect(stderr.length).to.be(0); // "Warning: Using a password on the command line interface can be insecure."
+                clients.getByAppIdAndType(APP_ID, clients.TYPE_OAUTH, function (error, client) {
+                    expect(error).to.not.be.ok();
+                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
+                    expect(client.clientSecret.length).to.be(256); // 32 hex chars (8 * 256 bits)
+                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_ID=' + client.id);
+                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_SECRET=' + client.clientSecret);
                     done();
                 });
             });
         });
 
-        it('installation - postgresql addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                var postgresqlUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('POSTGRESQL_URL=') === 0) postgresqlUrl = env.split('=')[1]; });
-                expect(postgresqlUrl).to.be.ok();
-
-                var urlp = url.parse(postgresqlUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('POSTGRESQL_PORT=5432');
-                expect(data.Config.Env).to.contain('POSTGRESQL_HOST=postgresql');
-                expect(data.Config.Env).to.contain('POSTGRESQL_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('POSTGRESQL_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('POSTGRESQL_DATABASE=' + dbname);
-
-                var cmd = util.format('bash -c "PGPASSWORD=%s psql -q -h %s -U%s --dbname=%s -e \'CREATE TABLE IF NOT EXISTS foo (id INT);\'"',
-                    password, 'postgresql', username, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error, stdout, stderr) {
-                    expect(!error).to.be.ok();
-                    expect(stdout.length).to.be(0);
-                    expect(stderr.length).to.be(0);
-                    done();
-                });
+        it('installation - app can populate addons', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort + '/populate_addons').end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                for (var key in res.body) {
+                    expect(res.body[key]).to.be('OK');
+                }
+                done();
             });
         });
 
-        it('installation - mongodb addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                var mongodbUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('MONGODB_URL=') === 0) mongodbUrl = env.split('=')[1]; });
-                expect(mongodbUrl).to.be.ok();
-
-                var urlp = url.parse(mongodbUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('MONGODB_PORT=27017');
-                expect(data.Config.Env).to.contain('MONGODB_HOST=mongodb');
-                expect(data.Config.Env).to.contain('MONGODB_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('MONGODB_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('MONGODB_DATABASE=' + dbname);
-
-                var cmd = util.format('mongo --quiet -u %s -p %s %s:%s/%s --eval "db.collection.insert({ item: 34 })"',
-                    username, password, 'mongodb', 27017, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error) {
-                    expect(!error).to.be.ok();
-                    done();
-                });
-            });
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 1');
+            checkAddons(appEntry, done);
         });
 
-        it('installation - scheduler', function (done) {
-            async.retry({ times: 100, interval: 1000 }, function (retryCallback) {
-                if (fs.existsSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env')) return retryCallback();
-
-                retryCallback(new Error('not run yet'));
-            }, done);
+        it('installation - redis addon created', function (done) {
+            checkRedis('redis-' + APP_ID, done);
         });
 
         xit('logs - stdout and stderr', function (done) {
@@ -963,6 +943,12 @@ describe('Apps', function () {
             checkStartState();
         });
 
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 2');
+            checkAddons(appEntry, done);
+        });
+
         it('can uninstall app', function (done) {
             var count = 0;
             function checkUninstallStatus() {
@@ -1046,8 +1032,6 @@ describe('Apps', function () {
             APP_ID = uuid.v4();
 
             async.series([
-                setup,
-
                 function (callback) {
                     config.set('fqdn', 'test.foobar.com');
                     callback();
@@ -1084,7 +1068,6 @@ describe('Apps', function () {
         after(function (done) {
             APP_ID = null;
             async.series([
-                cleanup,
                 apiHockServer.close.bind(apiHockServer),
                 awsHockServer.close.bind(awsHockServer)
             ], done);
@@ -1093,7 +1076,8 @@ describe('Apps', function () {
         var appResult = null, appEntry = null;
 
         it('can install test app', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
 
             var count = 0;
             function checkInstallStatus() {
@@ -1110,11 +1094,12 @@ describe('Apps', function () {
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: null })
+                  .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
-                expect(fake.isDone()).to.be.ok();
-                expect(res.body.id).to.equal(APP_ID);
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
+                APP_ID = res.body.id;
                 checkInstallStatus();
             });
         });
@@ -1202,52 +1187,26 @@ describe('Apps', function () {
             });
         });
 
-        var redisIp, exportedRedisPort;
-
-        it('installation - redis addon created', function (done) {
-            docker.getContainer('redis-' + APP_ID).inspect(function (error, data) {
-                expect(error).to.not.be.ok();
-                expect(data).to.be.ok();
-
-                redisIp = safe.query(data, 'NetworkSettings.IPAddress');
-                expect(redisIp).to.be.ok();
-
-                exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
-                expect(exportedRedisPort).to.be.ok();
 
+        it('installation - app can populate addons', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort + '/populate_addons').end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                for (var key in res.body) {
+                    expect(res.body[key]).to.be('OK');
+                }
                 done();
             });
         });
 
-        it('installation - redis addon config', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 3');
+            checkAddons(appEntry, done);
+        });
 
-                var urlp = url.parse(redisUrl);
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                var password = urlp.auth.split(':')[1];
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                function checkRedis() {
-                    var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                    client.on('error', done);
-                    client.set('key', 'value');
-                    client.get('key', function (err, reply) {
-                        expect(err).to.not.be.ok();
-                        expect(reply.toString()).to.be('value');
-                        client.end();
-                        done();
-                    });
-                }
-
-                setTimeout(checkRedis, 1000); // the bridge network takes time to come up?
-            });
+        it('installation - redis addon created', function (done) {
+            checkRedis('redis-' + APP_ID, done);
         });
 
         function checkConfigureStatus(count, done) {
@@ -1265,20 +1224,20 @@ describe('Apps', function () {
             });
         }
 
-        it('cannot reconfigure app with missing location', function (done) {
+        it('cannot reconfigure app with bad location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: 1234, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
             });
         });
 
-        it('cannot reconfigure app with missing accessRestriction', function (done) {
+        it('cannot reconfigure app with bad accessRestriction', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 } })
+                  .send({ password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: false })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -1288,7 +1247,7 @@ describe('Apps', function () {
         it('cannot reconfigure app with only the cert, no key', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -1298,27 +1257,27 @@ describe('Apps', function () {
         it('cannot reconfigure app with only the key, no cert', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, key: validKey1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
             });
         });
 
-        it('cannot reconfigure app with cert not bein a string', function (done) {
+        it('cannot reconfigure app with cert not being a string', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: 1234, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: 1234, key: validKey1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
             });
         });
 
-        it('cannot reconfigure app with key not bein a string', function (done) {
+        it('cannot reconfigure app with key not being a string', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: 1234 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, cert: validCert1, key: 1234 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -1328,7 +1287,7 @@ describe('Apps', function () {
         it('non admin cannot reconfigure app', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token_1 })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(403);
                 done();
@@ -1338,7 +1297,7 @@ describe('Apps', function () {
         it('can reconfigure app', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 } })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 checkConfigureStatus(0, done);
@@ -1351,7 +1310,7 @@ describe('Apps', function () {
                 expect(!error).to.be.ok();
                 expect(app).to.be.an('object');
                 appEntry = app;
-                expect(appEntry.containerid).to.not.be(oldContainerId);
+                expect(appEntry.containerId).to.not.be(oldContainerId);
                 done();
             });
         });
@@ -1368,61 +1327,19 @@ describe('Apps', function () {
         });
 
         it('reconfiguration - redis addon recreated', function (done) {
-            docker.getContainer('redis-' + APP_ID).inspect(function (error, data) {
-                expect(error).to.not.be.ok();
-                expect(data).to.be.ok();
-
-                redisIp = safe.query(data, 'NetworkSettings.IPAddress');
-                expect(redisIp).to.be.ok();
-
-                exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
-                expect(exportedRedisPort).to.be.ok();
-
-                done();
-            });
+            checkRedis('redis-' + APP_ID, done);
         });
 
-        it('redis addon works after reconfiguration', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
-
-                var urlp = url.parse(redisUrl);
-                var password = urlp.auth.split(':')[1];
-
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                client.on('error', done);
-                client.set('key', 'value');
-                client.get('key', function (err, reply) {
-                    expect(err).to.not.be.ok();
-                    expect(reply.toString()).to.be('value');
-                    client.end();
-                    done();
-                });
-            });
-        });
-
-        it('scheduler works after reconfiguration', function (done) {
-            async.retry({ times: 100, interval: 1000 }, function (callback) {
-                var data = safe.fs.readFileSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env', 'utf8');
-
-                if (data && data.indexOf('ECHO_SERVER_PORT=7172') !== -1) return callback();
-
-                callback(new Error('not run yet'));
-            }, done);
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 4');
+            checkAddons(appEntry, done);
         });
 
         it('can reconfigure app with custom certificate', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: validKey1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 checkConfigureStatus(0, done);

src/routes/test/backups-test.js

@@ -53,7 +53,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, { }, callback);
         },
 
         function createSettings(callback) {

src/routes/test/clients-test.js

@@ -8,7 +8,7 @@
 
 var async = require('async'),
     config = require('../../config.js'),
-    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     database = require('../../database.js'),
     oauth2 = require('../oauth2.js'),
     expect = require('expect.js'),
@@ -163,6 +163,26 @@ describe('OAuth Clients API', function () {
                 });
             });
 
+            it('fails with invalid name', function (done) {
+                superagent.post(SERVER_URL + '/api/v1/oauth/clients')
+                       .query({ access_token: token })
+                       .send({ appId: '$"$%^45asdfasdfadf.adf.', redirectURI: 'http://foobar.com', scope: 'profile' })
+                       .end(function (error, result) {
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+            });
+
+            it('succeeds with dash', function (done) {
+                superagent.post(SERVER_URL + '/api/v1/oauth/clients')
+                       .query({ access_token: token })
+                       .send({ appId: 'fo-1234-bar', redirectURI: 'http://foobar.com', scope: 'profile' })
+                       .end(function (error, result) {
+                    expect(result.statusCode).to.equal(201);
+                    done();
+                });
+            });
+
             it('succeeds', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
                        .query({ access_token: token })
@@ -174,7 +194,7 @@ describe('OAuth Clients API', function () {
                     expect(result.body.redirectURI).to.be.a('string');
                     expect(result.body.clientSecret).to.be.a('string');
                     expect(result.body.scope).to.be.a('string');
-                    expect(result.body.type).to.equal(clientdb.TYPE_EXTERNAL);
+                    expect(result.body.type).to.equal(clients.TYPE_EXTERNAL);
 
                     done();
                 });
@@ -291,6 +311,14 @@ describe('OAuth Clients API', function () {
             scope: 'profile'
         };
 
+        var CLIENT_1 = {
+            id: '',
+            appId: 'someAppId-1',
+            redirectURI: 'http://some.callback1',
+            scope: 'profile',
+            type: clients.TYPE_OAUTH
+        };
+
         before(function (done) {
             async.series([
                 server.start.bind(null),
@@ -387,6 +415,44 @@ describe('OAuth Clients API', function () {
                    });
                 });
             });
+
+            it('fails for cid-webadmin', function (done) {
+                superagent.del(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin')
+                       .query({ access_token: token })
+                       .end(function (error, result) {
+                    expect(result.statusCode).to.equal(405);
+
+                    superagent.get(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin')
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(result.statusCode).to.equal(200);
+
+                        done();
+                   });
+                });
+            });
+
+            it('fails for addon auth client', function (done) {
+                clients.add(CLIENT_1.appId, CLIENT_1.type, CLIENT_1.redirectURI, CLIENT_1.scope, function (error, result) {
+                    expect(error).to.equal(null);
+
+                    CLIENT_1.id = result.id;
+
+                    superagent.del(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_1.id)
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(result.statusCode).to.equal(405);
+
+                        superagent.get(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_1.id)
+                               .query({ access_token: token })
+                               .end(function (error, result) {
+                            expect(result.statusCode).to.equal(200);
+
+                            done();
+                       });
+                    });
+                });
+            });
         });
     });
 });
@@ -489,8 +555,7 @@ describe('Clients', function () {
             .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
 
-                expect(result.body.clients.length).to.eql(1);
-                expect(result.body.clients[0].tokenCount).to.eql(1);
+                expect(result.body.clients.length).to.eql(3);
 
                 done();
             });
@@ -543,7 +608,7 @@ describe('Clients', function () {
                 expect(result.statusCode).to.equal(200);
 
                 expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.id);
+                expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
 
                 done();
             });
@@ -596,7 +661,7 @@ describe('Clients', function () {
                 expect(result.statusCode).to.equal(200);
 
                 expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.id);
+                expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
 
                 superagent.del(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin/tokens')
                 .query({ access_token: token })

src/routes/test/cloudron-test.js

@@ -1,6 +1,5 @@
 'use strict';
 
-/* jslint node:true */
 /* global it:false */
 /* global describe:false */
 /* global before:false */
@@ -10,6 +9,7 @@ var async = require('async'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
+    locker = require('../../locker.js'),
     nock = require('nock'),
     os = require('os'),
     superagent = require('superagent'),
@@ -26,6 +26,7 @@ function setup(done) {
     nock.cleanAll();
     config._reset();
     config.set('version', '0.5.0');
+    config.set('fqdn', 'localhost');
     server.start(done);
 }
 
@@ -262,6 +263,182 @@ describe('Cloudron', function () {
 
     });
 
+    describe('migrate', function () {
+        before(function (done) {
+            async.series([
+                setup,
+
+                function (callback) {
+                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                           .query({ setupToken: 'somesetuptoken' })
+                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                           .end(function (error, result) {
+                        expect(result).to.be.ok();
+                        expect(scope1.isDone()).to.be.ok();
+                        expect(scope2.isDone()).to.be.ok();
+
+                        // stash token for further use
+                        token = result.body.token;
+
+                        callback();
+                    });
+                },
+
+                function setupBackupConfig(callback) {
+                    superagent.post(SERVER_URL + '/api/v1/settings/backup_config')
+                           .send({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(result.statusCode).to.equal(200);
+
+                        callback();
+                    });
+                }
+
+            ], done);
+        });
+
+        after(function (done) {
+            locker.unlock(locker._operation); // migrate never unlocks
+            cleanup(done);
+        });
+
+        it('fails without token', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 'sfo'})
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails without password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 'sfo'})
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds without size', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ region: 'sfo', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
+                done();
+            });
+        });
+
+        it('fails with wrong size type', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 4, region: 'sfo', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds without region', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
+                done();
+            });
+        });
+
+        it('fails with wrong region type', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 4, password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails when in wrong state', function (done) {
+            var scope2 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) {
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope1 = nock(config.apiServerOrigin())
+                .post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
+                    return body.size && body.region && body.restoreKey;
+                }).reply(409, {});
+
+            injectShellMock();
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 'sfo', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
+
+                function checkAppstoreServerCalled() {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
+                        restoreShellMock();
+                        return done();
+                    }
+
+                    setTimeout(checkAppstoreServerCalled, 100);
+                }
+
+                checkAppstoreServerCalled();
+            });
+        });
+
+        it('succeeds', function (done) {
+            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
+                return body.size && body.region && body.restoreKey;
+            }).reply(202, {});
+
+            var scope2 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) {
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+
+            injectShellMock();
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
+                   .send({ size: 'small', region: 'sfo', password: PASSWORD })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
+
+                function checkAppstoreServerCalled() {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
+                        restoreShellMock();
+                        return done();
+                    }
+
+                    setTimeout(checkAppstoreServerCalled, 100);
+                }
+
+                checkAppstoreServerCalled();
+            });
+        });
+    });
+
     describe('feedback', function () {
         before(function (done) {
             async.series([

src/routes/test/developer-test.js

@@ -327,7 +327,7 @@ describe('Developer API', function () {
                    .send({ username: USERNAME, password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
@@ -338,7 +338,7 @@ describe('Developer API', function () {
                    .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
@@ -349,7 +349,7 @@ describe('Developer API', function () {
                    .send({ username: EMAIL, password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
@@ -360,10 +360,66 @@ describe('Developer API', function () {
                    .send({ username: EMAIL.toUpperCase(), password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
         });
     });
+
+    describe('sdk tokens are valid without password checks', function () {
+        var token_normal, token_sdk;
+
+        before(function (done) {
+            async.series([
+                setup,
+
+                settings.setDeveloperMode.bind(null, true),
+
+                function (callback) {
+                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                           .query({ setupToken: 'somesetuptoken' })
+                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                           .end(function (error, result) {
+                        expect(result).to.be.ok();
+                        expect(scope1.isDone()).to.be.ok();
+                        expect(scope2.isDone()).to.be.ok();
+
+                        token_normal = result.body.token;
+
+                        superagent.post(SERVER_URL + '/api/v1/developer/login')
+                          .send({ username: USERNAME, password: PASSWORD })
+                          .end(function (error, result) {
+                            expect(result.statusCode).to.equal(200);
+                            expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
+                            expect(result.body.token).to.be.a('string');
+
+                            token_sdk = result.body.token;
+
+                            callback();
+                        });
+                    });
+                },
+            ], done);
+        });
+
+        after(cleanup);
+
+        it('fails with non sdk token', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password').query({ access_token: token_normal }).send({ newPassword: 'Some?$123' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password').query({ access_token: token_sdk }).send({ newPassword: 'Some?$123' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+    });
 });

src/routes/test/eventlog-test.js

@@ -67,7 +67,7 @@ function setup(done) {
             token_1 = tokendb.generateToken();
 
             // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-            tokendb.add(token_1, tokendb.PREFIX_USER + USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+            tokendb.add(token_1, USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
         }
 
     ], done);

src/routes/test/groups-test.js

@@ -6,18 +6,15 @@
 
 'use strict';
 
-var appdb = require('../../appdb.js'),
-    async = require('async'),
+var async = require('async'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
     groups = require('../../groups.js'),
     superagent = require('superagent'),
     server = require('../../server.js'),
-    settings = require('../../settings.js'),
     tokendb = require('../../tokendb.js'),
-    nock = require('nock'),
-    userdb = require('../../userdb.js');
+    nock = require('nock');
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
@@ -117,6 +114,9 @@ describe('Groups API', function () {
                 expect(res.body.groups).to.be.an(Array);
                 expect(res.body.groups.length).to.be(1);
                 expect(res.body.groups[0].name).to.eql('admin');
+                expect(res.body.groups[0].userIds).to.be.an(Array);
+                expect(res.body.groups[0].userIds.length).to.be(1);
+                expect(res.body.groups[0].userIds[0]).to.be(userId);
                 done();
             });
         });
@@ -224,7 +224,7 @@ describe('Groups API', function () {
         });
 
         it('cannot add user to invalid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'admin', 'something' ]})
                   .end(function (error, result) {
@@ -234,7 +234,7 @@ describe('Groups API', function () {
         });
 
         it('can add user to valid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'admin', 'group0', 'group1' ]})
                   .end(function (error, result) {
@@ -243,8 +243,8 @@ describe('Groups API', function () {
             });
         });
 
-        it('can remove last user from admin', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
+        it('cannot remove self from admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'group0', 'group1' ]})
                   .end(function (error, result) {
@@ -252,5 +252,37 @@ describe('Groups API', function () {
                 done();
             });
         });
+
+        it('can add another user to admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId_1 + '/groups')
+                  .query({ access_token: token })
+                  .send({ groupIds: [ 'admin' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+
+        it('lists members of admin group', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups/admin')
+                  .query({ access_token: token })
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.userIds.length).to.be(2);
+                expect(result.body.userIds[0]).to.be(userId);
+                expect(result.body.userIds[1]).to.be(userId_1);
+                done();
+            });
+        });
+
+        it('remove activation user from admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
+                  .query({ access_token: token_1 })
+                  .send({ groupIds: [ 'group0', 'group1' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204); // user_1 is still admin, so we can remove the other person
+                done();
+            });
+        });
     });
 });

src/routes/test/mailboxes-test.js

@@ -0,0 +1,211 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    superagent = require('superagent'),
+    server = require('../../server.js'),
+    nock = require('nock'),
+    userdb = require('../../userdb.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var MAILBOX_ID = 'mailbox';
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var server;
+function setup(done) {
+    config.set('fqdn', 'foobar.com');
+
+    async.series([
+        server.start.bind(server),
+
+        userdb._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        }
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('Mailbox API', function () {
+    this.timeout(10000);
+
+    before(setup);
+    after(cleanup);
+
+    it('cannot create a mailbox without name param', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot create a mailbox without token', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(401);
+            done();
+        });
+    });
+
+    it('cannot create invalid mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: 'no-reply' })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('can create mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(201);
+            done();
+        });
+    });
+
+    it('can get mailbox', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.name).to.equal(MAILBOX_ID);
+            expect(res.body.creationTime).to.be.ok();
+            done();
+        });
+    });
+
+    it('cannot set with invalid alias', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'a' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot set with invalid type', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'apple', 34 ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('can set aliases of mailbox', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'alias1', 'alias2' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            done();
+        });
+    });
+
+    it('can list mailboxes', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.mailboxes).to.be.an(Array);
+            expect(res.body.mailboxes[0].name).to.be(MAILBOX_ID);
+            done();
+        });
+    });
+
+    it('can get aliases', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.aliases).to.be.an(Array);
+            expect(res.body.aliases[0]).to.be('alias1');
+            expect(res.body.aliases[1]).to.be('alias2');
+            done();
+        });
+    });
+
+    it('can add another mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID + '2' })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(201);
+            done();
+        });
+    });
+
+    it('cannot alias existing mailbox', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ MAILBOX_ID + '2' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(409);
+            done();
+        });
+    });
+
+    it('can delete mailbox', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(204);
+            done();
+        });
+    });
+
+    it('cannot delete random mailbox', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(404);
+            done();
+        });
+    });
+});

src/routes/test/oauth2-test.js

@@ -18,6 +18,7 @@ var expect = require('expect.js'),
     querystring = require('querystring'),
     database = require('../../database.js'),
     clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     userdb = require('../../userdb.js'),
     user = require('../../user.js'),
     appdb = require('../../appdb.js'),
@@ -197,7 +198,7 @@ describe('OAuth2', function () {
         var CLIENT_0 = {
             id: 'cid-client0',
             appId: 'appid-app0',
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret0',
             redirectURI: 'http://redirect0',
             scope: 'profile'
@@ -207,7 +208,7 @@ describe('OAuth2', function () {
         var CLIENT_1 = {
             id: 'cid-client1',
             appId: 'appid-app1',
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret1',
             redirectURI: 'http://redirect1',
             scope: 'profile'
@@ -217,7 +218,7 @@ describe('OAuth2', function () {
         var CLIENT_2 = {
             id: 'cid-client2',
             appId: APP_0.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret2',
             redirectURI: 'http://redirect2',
             scope: 'profile'
@@ -227,7 +228,7 @@ describe('OAuth2', function () {
         var CLIENT_3 = {
             id: 'cid-client3',
             appId: APP_0.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret3',
             redirectURI: 'http://redirect1',
             scope: 'profile'
@@ -237,7 +238,7 @@ describe('OAuth2', function () {
         var CLIENT_4 = {
             id: 'cid-client4',
             appId: 'appid-app4',
-            type: clientdb.TYPE_PROXY,
+            type: clients.TYPE_PROXY,
             clientSecret: 'secret4',
             redirectURI: 'http://redirect4',
             scope: 'profile'
@@ -247,7 +248,7 @@ describe('OAuth2', function () {
         var CLIENT_5 = {
             id: 'cid-client5',
             appId: APP_0.id,
-            type: clientdb.TYPE_PROXY,
+            type: clients.TYPE_PROXY,
             clientSecret: 'secret5',
             redirectURI: 'http://redirect5',
             scope: 'profile'
@@ -257,7 +258,7 @@ describe('OAuth2', function () {
         var CLIENT_6 = {
             id: 'cid-client6',
             appId: APP_1.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret6',
             redirectURI: 'http://redirect6',
             scope: 'profile'
@@ -267,7 +268,7 @@ describe('OAuth2', function () {
         var CLIENT_7 = {
             id: 'cid-client7',
             appId: APP_2.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret7',
             redirectURI: 'http://redirect7',
             scope: 'profile'
@@ -277,7 +278,7 @@ describe('OAuth2', function () {
         var CLIENT_8 = {
             id: 'cid-client8',
             appId: APP_2.id,
-            type: clientdb.TYPE_SIMPLE_AUTH,
+            type: clients.TYPE_SIMPLE_AUTH,
             clientSecret: 'secret8',
             redirectURI: 'http://redirect8',
             scope: 'profile'
@@ -287,7 +288,7 @@ describe('OAuth2', function () {
         var CLIENT_9 = {
             id: 'cid-client9',
             appId: APP_3.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret9',
             redirectURI: 'http://redirect9',
             scope: 'profile'
@@ -313,10 +314,10 @@ describe('OAuth2', function () {
                 clientdb.add.bind(null, CLIENT_7.id, CLIENT_7.appId, CLIENT_7.type, CLIENT_7.clientSecret, CLIENT_7.redirectURI, CLIENT_7.scope),
                 clientdb.add.bind(null, CLIENT_8.id, CLIENT_8.appId, CLIENT_8.type, CLIENT_8.clientSecret, CLIENT_8.redirectURI, CLIENT_8.scope),
                 clientdb.add.bind(null, CLIENT_9.id, CLIENT_9.appId, CLIENT_9.type, CLIENT_9.clientSecret, CLIENT_9.redirectURI, CLIENT_9.scope),
-                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, APP_0.altDomain),
-                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit, APP_1.altDomain),
-                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit, APP_2.altDomain),
-                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit, APP_3.altDomain),
+                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0),
+                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1),
+                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2),
+                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3),
                 function (callback) {
                     user.create(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, null /* source */, function (error, userObject) {
                         expect(error).to.not.be.ok();

src/routes/test/profile-test.js

@@ -115,7 +115,7 @@ describe('Profile API', function () {
             var token = tokendb.generateToken();
             var expires = Date.now() - 2000; // 1 sec
 
-            tokendb.add(token, tokendb.PREFIX_USER + user_0.id, null, expires, '*', function (error) {
+            tokendb.add(token, user_0.id, null, expires, '*', function (error) {
                 expect(error).to.not.be.ok();
 
                 superagent.get(SERVER_URL + '/api/v1/profile').query({ access_token: token }).end(function (error, result) {
@@ -153,7 +153,7 @@ describe('Profile API', function () {
         after(cleanup);
 
         it('change email fails due to missing token', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .send({ email: EMAIL_0_NEW })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(401);
@@ -162,7 +162,7 @@ describe('Profile API', function () {
         });
 
         it('change email fails due to invalid email', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .query({ access_token: token_0 })
                    .send({ email: 'foo@bar' })
                    .end(function (error, result) {
@@ -172,7 +172,7 @@ describe('Profile API', function () {
         });
 
         it('change user succeeds without email nor displayName', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .query({ access_token: token_0 })
                    .send({})
                    .end(function (error, result) {
@@ -182,7 +182,7 @@ describe('Profile API', function () {
         });
 
         it('change email succeeds', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .query({ access_token: token_0 })
                    .send({ email: EMAIL_0_NEW })
                    .end(function (error, result) {
@@ -203,7 +203,7 @@ describe('Profile API', function () {
         });
 
         it('change displayName succeeds', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .query({ access_token: token_0 })
                    .send({ displayName: DISPLAY_NAME_0_NEW })
                    .end(function (error, result) {
@@ -229,7 +229,7 @@ describe('Profile API', function () {
         after(cleanup);
 
         it('fails due to missing current password', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ newPassword: 'some wrong password' })
                    .end(function (err, res) {
@@ -239,7 +239,7 @@ describe('Profile API', function () {
         });
 
         it('fails due to missing new password', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ password: PASSWORD })
                    .end(function (err, res) {
@@ -249,7 +249,7 @@ describe('Profile API', function () {
         });
 
         it('fails due to wrong password', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ password: 'some wrong password', newPassword: 'MOre#$%34' })
                    .end(function (err, res) {
@@ -259,7 +259,7 @@ describe('Profile API', function () {
         });
 
         it('fails due to invalid password', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ password: PASSWORD, newPassword: 'five' })
                    .end(function (err, res) {
@@ -269,7 +269,7 @@ describe('Profile API', function () {
         });
 
         it('succeeds', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ password: PASSWORD, newPassword: 'MOre#$%34' })
                    .end(function (err, res) {
@@ -284,7 +284,7 @@ describe('Profile API', function () {
         after(cleanup);
 
         it('fails due to missing showTutorial', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/tutorial')
+            superagent.post(SERVER_URL + '/api/v1/profile/tutorial')
                    .query({ access_token: token_0 })
                    .send({})
                    .end(function (err, res) {
@@ -294,7 +294,7 @@ describe('Profile API', function () {
         });
 
         it('fails due to wrong showTutorial type', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/tutorial')
+            superagent.post(SERVER_URL + '/api/v1/profile/tutorial')
                    .query({ access_token: token_0 })
                    .send({ showTutorial: 'true' })
                    .end(function (err, res) {
@@ -304,7 +304,7 @@ describe('Profile API', function () {
         });
 
         it('succeeds', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/tutorial')
+            superagent.post(SERVER_URL + '/api/v1/profile/tutorial')
                    .query({ access_token: token_0 })
                    .send({ showTutorial: false })
                    .end(function (err, res) {

src/routes/test/server-test.js

@@ -0,0 +1,88 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    net = require('net'),
+    nock = require('nock'),
+    superagent = require('superagent'),
+    server = require('../../server.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var server;
+function setup(done) {
+    config.setVersion('1.2.3');
+
+    async.series([
+        server.start.bind(server),
+
+        database._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        }
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('REST API', function () {
+    before(setup);
+    after(cleanup);
+
+    it('does not crash with invalid JSON', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+            .query({ access_token: token })
+            .set('content-type', 'application/json')
+            .send("some invalid non-strict json")
+           .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            expect(result.body.message).to.be('Bad JSON');
+            done();
+        });
+    });
+
+    it('does not crash with invalid string', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+            .query({ access_token: token })
+            .set('content-type', 'application/x-www-form-urlencoded')
+            .send("some string")
+           .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            done();
+        });
+    });
+});

src/routes/test/settings-test.js

@@ -56,7 +56,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok' };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, { }, callback);
         }
     ], done);
 }
@@ -175,7 +175,7 @@ describe('Settings API', function () {
                    .query({ access_token: token })
                    .send({ name: name })
                    .end(function (err, res) {
-                expect(res.statusCode).to.equal(200);
+                expect(res.statusCode).to.equal(202);
                 done();
             });
         });

src/routes/test/simpleauth-test.js

@@ -6,9 +6,10 @@
 
 'use strict';
 
-var clientdb = require('../../clientdb.js'),
-    appdb = require('../../appdb.js'),
+var appdb = require('../../appdb.js'),
     async = require('async'),
+    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
@@ -70,7 +71,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_0 = {
         id: 'someclientid',
         appId: 'someappid',
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret',
         redirectURI: '',
         scope: 'user,profile'
@@ -79,7 +80,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_1 = {
         id: 'someclientid1',
         appId: APP_0.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret1',
         redirectURI: '',
         scope: 'user,profile'
@@ -88,7 +89,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_2 = {
         id: 'someclientid2',
         appId: APP_1.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret2',
         redirectURI: '',
         scope: 'user,profile'
@@ -97,7 +98,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_3 = {
         id: 'someclientid3',
         appId: APP_2.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret3',
         redirectURI: '',
         scope: 'user,profile'
@@ -106,7 +107,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_4 = {
         id: 'someclientid4',
         appId: APP_2.id,
-        type: clientdb.TYPE_OAUTH,
+        type: clients.TYPE_OAUTH,
         clientSecret: 'someclientsecret4',
         redirectURI: '',
         scope: 'user,profile'
@@ -115,7 +116,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_5 = {
         id: 'someclientid5',
         appId: APP_3.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret5',
         redirectURI: '',
         scope: 'user,profile'
@@ -159,10 +160,10 @@ describe('SimpleAuth API', function () {
             clientdb.add.bind(null, CLIENT_3.id, CLIENT_3.appId, CLIENT_3.type, CLIENT_3.clientSecret, CLIENT_3.redirectURI, CLIENT_3.scope),
             clientdb.add.bind(null, CLIENT_4.id, CLIENT_4.appId, CLIENT_4.type, CLIENT_4.clientSecret, CLIENT_4.redirectURI, CLIENT_4.scope),
             clientdb.add.bind(null, CLIENT_5.id, CLIENT_5.appId, CLIENT_5.type, CLIENT_5.clientSecret, CLIENT_5.redirectURI, CLIENT_5.scope),
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, APP_0.altDomain),
-            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit, APP_1.altDomain),
-            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit, APP_2.altDomain),
-            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit, APP_3.altDomain)
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2),
+            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3)
         ], done);
     });
 

src/routes/test/start_addons.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
-
-source ${SOURCE_DIR}/src/INFRA_VERSION
-
-readonly mysqldatadir="/tmp/mysqldata-$(date +%s)"
-readonly postgresqldatadir="/tmp/postgresqldata-$(date +%s)"
-readonly mongodbdatadir="/tmp/mongodbdata-$(date +%s)"
-root_password=secret
-
-start_postgresql() {
-    postgresql_vars="POSTGRESQL_ROOT_PASSWORD=${root_password}; POSTGRESQL_ROOT_HOST=172.17.0.0/255.255.0.0"
-
-    rm -rf /tmp/postgresql_vars.sh
-    echo "${postgresql_vars}" > /tmp/postgresql_vars.sh
-
-    docker rm -f postgresql 2>/dev/null 1>&2 || true
-
-    docker run -dtP --name=postgresql -v "${postgresqldatadir}:/var/lib/postgresql" \
-        --read-only -v /tmp -v /run \
-        -v /tmp/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh "${POSTGRESQL_IMAGE}" >/dev/null
-}
-
-start_mysql() {
-    local mysql_vars="MYSQL_ROOT_PASSWORD=${root_password}; MYSQL_ROOT_HOST=172.17.0.0/255.255.0.0"
-
-    rm -rf /tmp/mysql_vars.sh
-    echo "${mysql_vars}" > /tmp/mysql_vars.sh
-
-    docker rm -f mysql 2>/dev/null 1>&2 || true
-
-    docker run -dP --name=mysql -v "${mysqldatadir}:/var/lib/mysql" \
-        --read-only -v /tmp -v /run \
-        -v /tmp/mysql_vars.sh:/etc/mysql/mysql_vars.sh "${MYSQL_IMAGE}" >/dev/null
-}
-
-start_mongodb() {
-    local mongodb_vars="MONGODB_ROOT_PASSWORD=${root_password}"
-
-    rm -rf /tmp/mongodb_vars.sh
-    echo "${mongodb_vars}" > /tmp/mongodb_vars.sh
-
-    docker rm -f mongodb 2>/dev/null 1>&2 || true
-
-    docker run -dP --name=mongodb -v "${mongodbdatadir}:/var/lib/mongodb" \
-        --read-only -v /tmp -v /run \
-        -v /tmp/mongodb_vars.sh:/etc/mongodb_vars.sh "${MONGODB_IMAGE}" >/dev/null
-}
-
-start_mail() {
-    docker rm -f mail 2>/dev/null 1>&2 || true
-
-    docker run -dP --name=mail -e MAIL_SERVER_NAME="server.local" -e MAIL_DOMAIN="server.local" \
-        --read-only -v /tmp -v /run \
-        -v /tmp/maildata:/app/data "${MAIL_IMAGE}" >/dev/null
-}
-
-start_mysql
-start_postgresql
-start_mongodb
-start_mail
-
-echo -n "Waiting for addons to start"
-for i in {1..10}; do
-    echo -n "."
-    sleep 1
-done
-echo ""
-

src/routes/test/sysadmin-test.js

@@ -51,7 +51,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, { }, callback);
         },
 
         function createSettings(callback) {

src/routes/test/user-test.js

@@ -1,4 +1,3 @@
-/* jslint node:true */
 /* global it:false */
 /* global describe:false */
 /* global before:false */
@@ -21,7 +20,7 @@ var SERVER_URL = 'http://localhost:' + config.get('port');
 var USERNAME_0 = 'superaDmIn', PASSWORD = 'Foobar?1337', EMAIL_0 = 'silLY@me.com', EMAIL_0_NEW = 'stupID@me.com', DISPLAY_NAME_0_NEW = 'New Name';
 var USERNAME_1 = 'userTheFirst', EMAIL_1 = 'taO@zen.mac';
 var USERNAME_2 = 'userTheSecond', EMAIL_2 = 'USER@foo.bar', EMAIL_2_NEW = 'happy@ME.com';
-var USERNAME_3 = 'userTheThird', EMAIL_3 = 'user3@FOO.bar';
+var USERNAME_3 = 'ut', EMAIL_3 = 'user3@FOO.bar';
 
 function setup(done) {
     server.start(function (error) {
@@ -161,7 +160,7 @@ describe('User API', function () {
         var token = tokendb.generateToken();
         var expires = Date.now() + 2000; // 1 sec
 
-        tokendb.add(token, tokendb.PREFIX_USER + user_0.id, null, expires, '*', function (error) {
+        tokendb.add(token, user_0.id, null, expires, '*', function (error) {
             expect(error).to.not.be.ok();
 
             setTimeout(function () {
@@ -261,7 +260,7 @@ describe('User API', function () {
 
             checkMails(2, function () {
               // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-              tokendb.add(token_1, tokendb.PREFIX_USER + user_1.id, 'test-client-id',  Date.now() + 10000, '*', done);
+              tokendb.add(token_1, user_1.id, 'test-client-id',  Date.now() + 10000, '*', done);
             });
         });
     });
@@ -293,7 +292,7 @@ describe('User API', function () {
     });
 
     it('set second user as admin succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/set_groups')
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/groups')
                .query({ access_token: token })
                .send({ groupIds: [ groups.ADMIN_GROUP_ID ] })
                .end(function (err, res) {
@@ -310,8 +309,24 @@ describe('User API', function () {
         });
     });
 
+    it('list groupIds when listing users', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/users')
+        .query({ access_token: token })
+        .end(function (error, res) {
+            expect(error).to.be(null);
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.users).to.be.an('array');
+
+            res.body.users.forEach(function (user) {
+                expect(user.admin).to.be(true);
+                expect(user.groupIds).to.eql([ groups.ADMIN_GROUP_ID ]);
+            });
+            done();
+        });
+    });
+
     it('remove itself from admins fails', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id + '/set_groups')
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id + '/groups')
                .query({ access_token: token })
                .send({ groupIds: [ 'somegroupid' ] })
                .end(function (err, res) {
@@ -321,7 +336,7 @@ describe('User API', function () {
     });
 
     it('remove second user from admins succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/set_groups')
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/groups')
                .query({ access_token: token })
                .send({ groupIds: [ 'somegroupid' ] })
                .end(function (err, res) {
@@ -368,6 +383,26 @@ describe('User API', function () {
         });
     });
 
+    it('create user reserved name fails', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+               .query({ access_token: token })
+               .send({ username: 'no-reply' })
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('create user with short name fails', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+               .query({ access_token: token })
+               .send({ username: 'n' })
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            done();
+        });
+    });
+
     it('create second and third user', function (done) {
         mailer._clearMailQueue();
 
@@ -441,12 +476,24 @@ describe('User API', function () {
                 expect(user.email).to.be.ok();
                 expect(user.password).to.not.be.ok();
                 expect(user.salt).to.not.be.ok();
+                expect(user.groupIds).to.be.an(Array);
+                expect(user.admin).to.be.a('boolean');
             });
 
             done();
         });
     });
 
+    it('remove random user fails', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/users/randomid')
+               .query({ access_token: token })
+               .send({ password: PASSWORD })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(404);
+            done();
+        });
+    });
+
     it('user removes himself is not allowed', function (done) {
         superagent.del(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
@@ -508,7 +555,7 @@ describe('User API', function () {
 
     // Change email
     it('change email fails due to missing token', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_0.id)
                .send({ email: EMAIL_0_NEW })
                .end(function (error, result) {
             expect(result.statusCode).to.equal(401);
@@ -517,7 +564,7 @@ describe('User API', function () {
     });
 
     it('change email fails due to invalid email', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({ email: 'foo@bar' })
                .end(function (error, result) {
@@ -527,7 +574,7 @@ describe('User API', function () {
     });
 
     it('change user succeeds without email nor displayName', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({})
                .end(function (error, result) {
@@ -537,7 +584,7 @@ describe('User API', function () {
     });
 
     it('change email succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_2.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_2.id)
                .query({ access_token: token })
                .send({ email: EMAIL_2_NEW })
                .end(function (error, result) {
@@ -558,7 +605,7 @@ describe('User API', function () {
     });
 
     it('change email as admin for other user succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_2.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_2.id)
                .query({ access_token: token })
                .send({ email: EMAIL_2 })
                .end(function (error, result) {
@@ -579,7 +626,7 @@ describe('User API', function () {
     });
 
     it('change displayName succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({ displayName: DISPLAY_NAME_0_NEW })
                .end(function (error, result) {

src/routes/user.js

@@ -13,13 +13,16 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    clients = require('../clients.js'),
     generatePassword = require('../password.js').generate,
     groups = require('../groups.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    oauth2 = require('./oauth2.js'),
     user = require('../user.js'),
     tokendb = require('../tokendb.js'),
-    UserError = user.UserError;
+    UserError = user.UserError,
+    _ = require('underscore');
 
 function auditSource(req) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -41,11 +44,8 @@ function create(req, res, next) {
     var displayName = req.body.displayName || '';
 
     user.create(username, password, email, displayName, auditSource(req), { invitor: req.user, sendInvite: sendInvite }, function (error, user) {
-        if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, 'Invalid username'));
-        if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, 'Invalid email'));
-        if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(400, 'Invalid password'));
         if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
-        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
 
         var userInfo = {
@@ -54,6 +54,7 @@ function create(req, res, next) {
             displayName: user.displayName,
             email: user.email,
             admin: user.admin,
+            groupIds: [ ],
             resetToken: user.resetToken
         };
 
@@ -68,30 +69,29 @@ function update(req, res, next) {
 
     if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
     if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
+    if ('username' in req.body && typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be a string'));
 
-    if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
     if (req.user.id !== req.params.userId && !req.user.admin) return next(new HttpError(403, 'Not allowed'));
 
-    user.get(req.params.userId, function (error, result) {
-        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
+    user.update(req.params.userId, req.body, auditSource(req), function (error) {
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
         if (error) return next(new HttpError(500, error));
 
-        user.update(req.params.userId, result.username, req.body.email || result.email, req.body.displayName || result.displayName, auditSource(req), function (error) {
-            if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, error.message));
-            if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, error.message));
-            if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
-            if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
-            if (error) return next(new HttpError(500, error));
-
-            next(new HttpSuccess(204));
-        });
+        next(new HttpSuccess(204));
     });
 }
 
 function list(req, res, next) {
-    user.list(function (error, result) {
+    user.list(function (error, results) {
         if (error) return next(new HttpError(500, error));
-        next(new HttpSuccess(200, { users: result }));
+
+        var users = results.map(function (result) {
+            return _.pick(result, 'id', 'username', 'email', 'displayName', 'groupIds', 'admin');
+        });
+
+        next(new HttpSuccess(200, { users: users }));
     });
 }
 
@@ -105,17 +105,14 @@ function get(req, res, next) {
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
 
-        groups.isMember(groups.ADMIN_GROUP_ID, req.params.userId, function (error, isAdmin) {
-            if (error) return next(new HttpError(500, error));
-
-            next(new HttpSuccess(200, {
-                id: result.id,
-                username: result.username,
-                email: result.email,
-                admin: isAdmin,
-                displayName: result.displayName
-            }));
-        });
+        next(new HttpSuccess(200, {
+            id: result.id,
+            username: result.username,
+            displayName: result.displayName,
+            email: result.email,
+            admin: result.admin,
+            groupIds: result.groupIds
+        }));
     });
 }
 
@@ -129,24 +126,20 @@ function remove(req, res, next) {
 
     if (req.user.id === req.params.userId) return next(new HttpError(403, 'Not allowed to remove yourself.'));
 
-    user.get(req.params.userId, function (error, userObject) {
+    user.remove(req.params.userId, auditSource(req), function (error) {
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
 
-        user.remove(userObject, auditSource(req), function (error) {
-            if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
-            if (error) return next(new HttpError(500, error));
-
-            next(new HttpSuccess(204));
-        });
+        next(new HttpSuccess(204));
     });
 }
 
 function verifyPassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    // developers are allowed through without password
-    if (req.user.tokenType === tokendb.TYPE_DEV) return next();
+    // using an 'sdk' token we skip password checks
+    var error = oauth2.validateRequestedScopes(req, [ clients.SCOPE_ROLE_SDK ]);
+    if (!error) return next();
 
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'API call requires user password'));
 

src/scripts/backupapp.sh

@@ -12,8 +12,8 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [ $# -lt 8 ]; then
-    echo "Usage: backupapp.sh <appid> <s3 config url> <s3 data url> <access key id> <access key> <session token> <region> <password>"
+if [ $# -lt 7 ]; then
+    echo "Usage: backupapp.sh <appid> <s3 config url> <s3 data url> <access key id> <access key> <region> <password> [session token]"
     exit 1
 fi
 
@@ -25,9 +25,12 @@ readonly s3_config_url="$2"
 readonly s3_data_url="$3"
 export AWS_ACCESS_KEY_ID="$4"
 export AWS_SECRET_ACCESS_KEY="$5"
-export AWS_SESSION_TOKEN="$6"
-export AWS_DEFAULT_REGION="$7"
-readonly password="$8"
+export AWS_DEFAULT_REGION="$6"
+readonly password="$7"
+
+if [ $# -gt 7 ]; then
+    export AWS_SESSION_TOKEN="$8"
+fi
 
 readonly now=$(date "+%Y-%m-%dT%H:%M:%S")
 readonly app_data_dir="${DATA_DIR}/${app_id}"

src/scripts/backupbox.sh

@@ -12,8 +12,8 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [ $# -lt 6 ]; then
-    echo "Usage: backupbox.sh <s3 url> <access key id> <access key> <session token> <region> <password>"
+if [ $# -lt 5 ]; then
+    echo "Usage: backupbox.sh <s3 url> <access key id> <access key> <region> <password> [session token]"
     exit 1
 fi
 
@@ -21,9 +21,13 @@ fi
 s3_url="$1"
 export AWS_ACCESS_KEY_ID="$2"
 export AWS_SECRET_ACCESS_KEY="$3"
-export AWS_SESSION_TOKEN="$4"
-export AWS_DEFAULT_REGION="$5"
-password="$6"
+export AWS_DEFAULT_REGION="$4"
+password="$5"
+
+if [ $# -gt 5 ]; then
+    export AWS_SESSION_TOKEN="$6"
+fi
+
 now=$(date "+%Y-%m-%dT%H:%M:%S")
 BOX_DATA_DIR="${HOME}/data/box"
 box_snapshot_dir="${HOME}/data/snapshots/box-${now}"

src/scripts/retire.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-# This script is called once at the end of a cloudrons lifetime
-
 set -eu -o pipefail
 
 if [[ ${EUID} -ne 0 ]]; then
@@ -22,7 +20,7 @@ if [[ "${BOX_ENV}" != "cloudron" ]]; then
 	exit 0
 fi
 
-"${BOX_SRC_DIR}/setup/splashpage.sh" --retire --data "$1" # show splash
+"${BOX_SRC_DIR}/setup/splashpage.sh" --retire-reason "$1" --retire-info "$2" --data "$3" # show splash
 
 echo "Stopping apps"
 systemctl stop docker # stop the apps

src/scripts/setup_infra.sh

@@ -1,148 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-if [[ $# == 1 && "$1" == "--check" ]]; then
-    echo "OK"
-    exit 0
-fi
-
-readonly DATA_DIR="/home/yellowtent/data"
-
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
-
-readonly fqdn="$1"
-readonly mail_fqdn="$2"
-readonly mail_tls_cert="$3"
-readonly mail_tls_key="$4"
-
-# removing containers ensures containers are launched with latest config updates
-# restore code in appatask does not delete old containers
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
-    echo "Infrastructure is upto date"
-    exit 0
-fi
-
-echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
-
-# TODO: be nice and stop addons cleanly (example, shutdown commands)
-existing_containers=$(docker ps -qa)
-echo "Remove containers: ${existing_containers}"
-if [[ -n "${existing_containers}" ]]; then
-    echo "${existing_containers}" | xargs docker rm -f
-fi
-
-# graphite
-graphite_container_id=$(docker run --restart=always -d --name="graphite" \
-    -m 75m \
-    --memory-swap 150m \
-    -p 127.0.0.1:2003:2003 \
-    -p 127.0.0.1:2004:2004 \
-    -p 127.0.0.1:8000:8000 \
-    -v "${DATA_DIR}/graphite:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${GRAPHITE_IMAGE}")
-echo "Graphite container id: ${graphite_container_id}"
-if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old graphite images"
-fi
-
-# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
-# MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
-# MAIL_DOMAIN is the domain for which this server is relaying mails
-mail_container_id=$(docker run --restart=always -d --name="mail" \
-    -m 75m \
-    --memory-swap 150m \
-    -h "${fqdn}" \
-    -e "MAIL_DOMAIN=${fqdn}" \
-    -e "MAIL_SERVER_NAME=${mail_fqdn}" \
-    -v "${DATA_DIR}/box/mail:/app/data" \
-    -v "${mail_tls_key}:/etc/tls_key.pem:ro" \
-    -v "${mail_tls_cert}:/etc/tls_cert.pem:ro" \
-    --read-only -v /tmp -v /run \
-    "${MAIL_IMAGE}")
-echo "Mail container id: ${mail_container_id}"
-if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mail images"
-fi
-
-# mysql
-mysql_addon_root_password=$(pwgen -1 -s)
-docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
-cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
-readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
-readonly MYSQL_ROOT_HOST='${docker0_ip}'
-EOF
-mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -m 256m \
-    --memory-swap 512m \
-    -h "${fqdn}" \
-    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MYSQL_IMAGE}")
-echo "MySQL container id: ${mysql_container_id}"
-if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mysql images"
-fi
-
-# postgresql
-postgresql_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
-readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
-EOF
-postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${fqdn}" \
-    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
-    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${POSTGRESQL_IMAGE}")
-echo "PostgreSQL container id: ${postgresql_container_id}"
-if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old postgresql images"
-fi
-
-# mongodb
-mongodb_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
-readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
-EOF
-mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${fqdn}" \
-    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
-    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MONGODB_IMAGE}")
-echo "Mongodb container id: ${mongodb_container_id}"
-if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mongodb images"
-fi
-
-# redis
-if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old redis images"
-fi
-
-# only touch apps in installed state. any other state is just resumed by the taskmanager
-if [[ "${infra_version}" == "none" ]]; then
-    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
-    echo "Marking installed apps for restore"
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
-else
-    # if existing infra was found, just mark apps for reconfiguration
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
-fi
-
-echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"